diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index b43cf4fd..f1ef884d 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -61,6 +61,7 @@ No kernel works in place. |---|---|---| | `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` | | `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef` | +| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon` (HDR) | | `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` | All are built with `clean/runner/build.py`'s `device_command`. The build's `-cuid` comes from the output file name, @@ -117,10 +118,27 @@ the frame parity, as Model M's. and filters them with no decode and no exposure (`recon.output_filter`). - The output filter is Model M's (the project's own taps). - The reconstruction kernel is **planned**. Its arguments and buffers will be Model M's `post` kernels' (Model M's - `clean/FORMATS.md`). Its summation order (taps, candidates) is not fixed yet: the oracle sums the gather columns - first, while Model M's reference sums rows first. The kernel's order decides, and the kernel-order model follows - it. + The reconstruction kernel `l_post_recon` (`l_post.co`, HDR; exists, GPU-checked by + `clean/l/runner/l_post_check.py`): + - arguments, buffers, layouts and grid are Model M's `post_recon` (Model M's `clean/FORMATS.md`), with `w` = the + post blob (`w[0]` = Model L's s); + - its colour gather runs in the model's precision, as confirmed by the spec side (the oracle's `recon.gather_half`): + - fused target positions; + - FP16 offsets and exponent with the fused cross term; + - float32 weights flushed to +0 below 2^-126; + - FP16 weights into half accumulators; + - a float32 weight sum; + - tap order i (columns) outer, j (rows) inner. + + So the kernel and the oracle agree on which weights underflow. Two regimes follow: total weight 0, the + zero-weight values; and total weight > 0 with every FP16 weight 0, where the output is the candidate history. + - its candidate combination is one fused multiply-add w1 L1 + (w2 L2), then times e; + - its blend is the fused form beta' = fused(alpha (1 - beta-bar) + beta-bar), H^ = fused((C - Hl) beta' + Hl); + - the decode passes a NaN through. + + The LDR kernel is planned, with the same arguments. Still open for the kernel-order model: the hardware exp2, + reciprocal and divisions, and the Catmull-Rom weight forms, which the kernel takes from Model M's `post.hip` + (part of the Model M fidelity decision). ## Checks (GPU, synthetic data) diff --git a/clean/l/kernels/l_post.hip b/clean/l/kernels/l_post.hip new file mode 100644 index 00000000..a6d747f1 --- /dev/null +++ b/clean/l/kernels/l_post.hip @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): output reconstruction (stage "post", HDR) as Model M's post_recon (clean/kernels/post.hip, +// included for its functions; its own kernels come along unused) with Model L's three changes +// (clean/l/reference/recon.py, the oracle): +// 1. a 4 x 4 colour gather: columns ax - 1 ... ax + 2 and rows b - 1 ... b + 2 (b from the even row of the pair); +// 2. each history candidate decoded and divided by the previous exposure before the two are combined: +// H_lin = (w1 L1 + w2 L2) e, with L_i = D(H_i) / e_prev (L2 only when w2 > 0); +// 3. an invalid or reset sample falls back to the render pixel's linear colour (no exposure, no encode). +// The colour gather runs in the model's own precision as confirmed by the spec side (oracle: recon.gather_half): +// fused target positions, FP16 offsets and exponent with the fused cross term, float32 weights flushed to +0 below +// 2^-126, FP16 weights into half accumulators, a float32 weight sum. The blend is item 3's fused form. The decode +// passes a NaN through (a zero total gather weight gives C = D(NaN) = NaN; the NaN guard then substitutes it, as the +// oracle). Arguments, layouts and grid are post_recon's; w[0] = Model L's post scale s. +#include "post.hip" + +namespace { + +constexpr int kLWin = 4; // taps -1 ... 2 +typedef _Float16 h16; + +// D(.) with a NaN passed through. +WG_FN float l_dec(float q) { return q != q ? q : dec_e(q); } + +// One render pixel's gather inputs in the model's precision: the nine features in FP16 (h(E_in(e colour)), the B and +// A values k / 255), and A = h(a -0.721325), B = h(b -0.721325), C = h(c -1.44265) (float32 product, one FP16 +// rounding). +struct LTexel { + h16 f[9]; + h16 kA, kB, kC; +}; + +WG_FN LTexel l_load_texel(const v4f* color, const u32* A, const u32* B, const v4h* C, int idx, float e) { + LTexel t; + const v4f c = color[idx]; + t.f[0] = (h16)enc_in(e * c[0]); + t.f[1] = (h16)enc_in(e * c[1]); + t.f[2] = (h16)enc_in(e * c[2]); + const u32 b = B[idx], a = A[idx]; +#pragma unroll + for (int i = 0; i < 3; ++i) { + t.f[3 + i] = (h16)(byte_f(b, i) / 255.0f); + t.f[6 + i] = (h16)(byte_f(a, i) / 255.0f); + } + const v4h k = C[idx]; + t.kA = (h16)((float)k[0] * -0.721325f); + t.kB = (h16)((float)k[1] * -0.721325f); + t.kC = (h16)((float)k[2] * -1.44265f); + return t; +} + +// Gathered sums of one D pixel: W = float32 sum of the unrounded weights; F = the half accumulators. +struct LGather { + float W; + h16 F[9]; +}; + +WG_FN void l_gather_zero(LGather& g) { + g.W = 0.0f; +#pragma unroll + for (int k = 0; k < 9; ++k) g.F[k] = (h16)0.0f; +} + +// One tap: eps = fused_half(h(u v), C, h(h(h(u u) A) + h(h(v v) B))), w = 2^eps (float32, +0 below 2^-126), +// W += w, F_k = fused_half(h(w), f_k, F_k). One FP16 operation per statement (contraction is off in post.hip). +WG_FN void l_gather_add(LGather& g, const LTexel& t, h16 u, h16 v) { + const h16 uu = u * u, vv = v * v, uv = u * v; + const h16 ta = uu * t.kA, tb = vv * t.kB; + const h16 s = ta + tb; + const h16 eps = __builtin_fmaf16(uv, t.kC, s); + float w = wg_exp2((float)eps); + if (!(w >= 0x1p-126f)) w = 0.0f; // flush to zero (also a NaN exponent gives 0, as 2^NaN never occurs here) + g.W += w; + const h16 hw = (h16)w; +#pragma unroll + for (int k = 0; k < 9; ++k) g.F[k] = __builtin_fmaf16(hw, t.f[k], g.F[k]); +} + +// Normalized features and the candidate choice (Model M's rules in value units). +struct LSel { + float fbar[9]; + int first, second; + float w1, w2; +}; + +WG_FN LSel l_select(const LGather& g) { + LSel r; + const float inv = wg_rcp(g.W); // rcp(0) = inf: 0 * inf = NaN features (zero total weight) +#pragma unroll + for (int k = 0; k < 9; ++k) r.fbar[k] = inv * (float)g.F[k]; + const float* sig = r.fbar + 3; + int first = 0; + float best = sig[0]; + if (sig[1] > best) { first = 1; best = sig[1]; } + if (sig[2] > best) first = 2; + const int o1 = first == 0 ? 1 : 0, o2 = first == 2 ? 1 : 2; + const int second = sig[o2] > sig[o1] ? o2 : o1; + const float q1 = clampf(1.1f * sig[first] - 0.05f, 0.0f, 1.0f); + const float q2 = clampf(1.1f * sig[second] - 0.05f, 0.0f, 1.0f); + const float rden = wg_rcp(wg_fmax(q1 + q2, 1e-10f)); + float w2 = q2 * rden, w1; + if (w2 < 1.0f / 255.0f) { + w2 = 0.0f; + w1 = 1.0f; + } else { + w1 = q1 * rden; + } + r.first = first; + r.second = second; + r.w1 = w1; + r.w2 = w2; + return r; +} + +// Steps 3-5 for one D pixel (Model L). Returns the linear new history H^ in hh and the linear display value O^ in oh. +WG_FN void l_post_core(const LGather& g, int x, int y, int fx, int fy, const float mx[3], const float my[3], + const WgPostParams& p, float s, const v4f* color, const void* hist_prev, float hh[3], + float oh[3]) { + const LSel sl = l_select(g); + const float X = (float)x + 0.5f, Y = (float)y + 0.5f; + const int ncand = sl.w2 != 0.0f ? 2 : 1; + float L[2][3]; + for (int n = 0; n < ncand; ++n) { + const int k = n == 0 ? sl.first : sl.second; + const float qx = X + mx[k], qy = Y + my[k]; + const bool valid = !p.reset && qx >= 0.0f && qx <= (float)p.dw && qy >= 0.0f && qy <= (float)p.dh; + if (valid) { + float h[3]; + history_cr(hist_prev, p.dw, p.dh, qx, qy, h); +#pragma unroll + for (int i = 0; i < 3; ++i) L[n][i] = l_dec(h[i]) / p.e_prev; + } else { + const v4f c = color[fy * p.rw + fx]; +#pragma unroll + for (int i = 0; i < 3; ++i) L[n][i] = c[i]; + } + } + float Hl[3], Cl[3]; + bool hl_inf = false; +#pragma unroll + for (int i = 0; i < 3; ++i) { + const float comb = ncand == 2 ? __builtin_fmaf(sl.w1, L[0][i], sl.w2 * L[1][i]) : sl.w1 * L[0][i]; + Hl[i] = comb * p.e; + hl_inf |= __builtin_fabsf(Hl[i]) == __builtin_inff(); + Cl[i] = l_dec(sl.fbar[i]); + } + // 5. Blend: alpha = sat((s W) g-bar), beta' = fused(alpha (1 - beta-bar) + beta-bar), H^ = fused((C - Hl) beta' + Hl), + // O^ likewise (item 3). sat(NaN) = 0 (zero total weight). + const float alpha = clampf((s * g.W) * sl.fbar[6], 0.0f, 1.0f); + const float bh2 = __builtin_fmaf(alpha, 1.0f - sl.fbar[7], sl.fbar[7]); + const float bo2 = __builtin_fmaf(alpha, 1.0f - sl.fbar[8], sl.fbar[8]); +#pragma unroll + for (int i = 0; i < 3; ++i) { + hh[i] = __builtin_fmaf(Cl[i] - Hl[i], bh2, Hl[i]); + oh[i] = __builtin_fmaf(Cl[i] - Hl[i], bo2, Hl[i]); + } + const float sh = hh[0] + hh[1] + hh[2], so = oh[0] + oh[1] + oh[2]; + if (sh != sh || hl_inf) { +#pragma unroll + for (int i = 0; i < 3; ++i) hh[i] = Cl[i]; + } + if (so != so || hl_inf) { +#pragma unroll + for (int i = 0; i < 3; ++i) oh[i] = Cl[i]; + } +} + +// post_recon's finish_pixel with l_post_core. +WG_FN void l_finish_pixel(const LGather& g, int x, int y, float px, float py, bool store, const WgPostParams& p, + float s, const v4f* color, const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, v4h* hist_out, _Float16* display, float hh[3]) { + const int fx = clampi(floor_i(px), 0, p.rw - 1), fy = clampi(floor_i(py), 0, p.rh - 1); + float mx[3], my[3]; + unpack_h2(m_near[fy * p.rw + fx], mx[0], my[0]); + unpack_h2(m_far[fy * p.rw + fx], mx[2], my[2]); + motion_bilinear(px, py, + [&](int c, int r) { return m_center[clampi(r, 0, p.rh - 1) * p.rw + clampi(c, 0, p.rw - 1)]; }, + mx[1], my[1]); + float oh[3]; + l_post_core(g, x, y, fx, fy, mx, my, p, s, color, hist_prev, hh, oh); + if (store) hist_store(hist_out, y * p.dw + x, enc_e(hh[0]), enc_e(hh[1]), enc_e(hh[2])); + if (store) { + const int pitch = display_pitch(p.dw), plane = pitch * p.dh, di = y * pitch + x; + display[di] = display_value(oh[0]); + display[plane + di] = display_value(oh[1]); + display[2 * plane + di] = display_value(oh[2]); + } +} + +} // namespace + +// Stage "post" for Model L, HDR. Arguments, layouts and grid as post_recon (clean/kernels/post.hip): grid +// ceil(dw / 32) * ceil(ceil(dh / 2) / 8) workgroups of 256 lanes; w = the post blob (w[0] = s). +WG_KERNEL_N(l_post_recon, kPostThreads)(const v4f* color, const u32* A, const u32* B, const v4h* C, + const v4h* hist_prev, const u32* m_near, const u32* m_center, + const u32* m_far, const float* w, v4h* hist_out, u32* reduced, + _Float16* display, WgPostParams p, const float* expo) { + if (expo) { + p.e = expo[0] == 0.0f ? 1.0f : expo[0]; + p.e_prev = expo[1] == 0.0f ? 1.0f : expo[1]; + p.ratio = (float)((double)p.e / (double)p.e_prev); + } + WG_LDS LTexel sh_tex[kStageCap]; + WG_LDS float sh_red[kPostThreads * 3]; + // Target positions in the model's precision: px = fused(x + 1/2, rx, jx), py = fused(ry, y + 1/2, jy). + const auto lpx = [&](int xx) { return __builtin_fmaf((float)xx + 0.5f, p.rx, p.jx); }; + const auto lpy = [&](int yy) { return __builtin_fmaf(p.ry, (float)yy + 0.5f, p.jy); }; + + const int lane = (int)wg_lane_id(); + const int tiles_x = (p.dw + kTileX - 1) / kTileX; + const int tile = (int)wg_block_id(); + const int x0 = (tile % tiles_x) * kTileX; + const int k0 = (tile / tiles_x) * kTilePairs; + const int npairs = (p.dh + 1) / 2; + const float s = w[0]; + + // Render footprint of the tile: window columns ax - 1 ... ax + 2 of its first and last column, window rows of its + // first and last row pair. + const int xl = mini(x0 + kTileX - 1, p.dw - 1), kl = mini(k0 + kTilePairs - 1, npairs - 1); + const int c0 = floor_i(lpx(x0) - 0.5f) - 1, c1 = floor_i(lpx(xl) - 0.5f) + (kLWin - 2); + const int r0 = floor_i(lpy(2 * k0) - 0.5f) - 1, r1 = floor_i(lpy(2 * kl) - 0.5f) + (kLWin - 2); + const int ncol = c1 - c0 + 1, nrow = r1 - r0 + 1; + const bool staged = ncol > 0 && nrow > 0 && ncol <= kStageCap && nrow <= kStageCap && ncol * nrow <= kStageCap; + if (staged) { + for (int i = lane; i < ncol * nrow; i += kPostThreads) { + const int r = i / ncol, c = i - r * ncol; + const int idx = clampi(r0 + r, 0, p.rh - 1) * p.rw + clampi(c0 + c, 0, p.rw - 1); + sh_tex[i] = l_load_texel(color, A, B, C, idx, p.e); + } + } + wg_barrier(); + + const bool active = x0 + (lane & 31) < p.dw && k0 + (lane >> 5) < npairs; + const int x = mini(x0 + (lane & 31), p.dw - 1); + const int y0 = 2 * mini(k0 + (lane >> 5), npairs - 1); + const bool has1 = y0 + 1 < p.dh; + const float px = lpx(x), py0 = lpy(y0), py1 = lpy(y0 + 1); + const int ax = floor_i(px - 0.5f), b = floor_i(py0 - 0.5f); + + // 2. Gather in the model's precision. Offsets: dx = h((ax + 1/2) - px); d_y = h((b_y + 1/2) - py(y)), minus 1 (half) + // when the row's own b_y differs from the pair's b; u = h(dx + i), v = h(d_y + j). Taps i = -1 ... 2 (outer), + // j = -1 ... 2 (inner). + const h16 dx = (h16)(((float)ax + 0.5f) - px); + const int b1 = floor_i(py1 - 0.5f); + const h16 one = (h16)1.0f; + h16 dy0 = (h16)(((float)b + 0.5f) - py0); // b_y of the even row is b + h16 dy1 = (h16)(((float)b1 + 0.5f) - py1); + if (b1 != b) dy1 = dy1 - one; + LGather g0, g1; + l_gather_zero(g0); + l_gather_zero(g1); +#pragma unroll + for (int i = -1; i <= kLWin - 2; ++i) { + const h16 u = dx + (h16)(float)i; +#pragma unroll + for (int j = -1; j <= kLWin - 2; ++j) { + const h16 v0 = dy0 + (h16)(float)j, v1 = dy1 + (h16)(float)j; + LTexel t; + if (staged) { + t = sh_tex[(b + j - r0) * ncol + (ax + i - c0)]; + } else { + const int idx = clampi(b + j, 0, p.rh - 1) * p.rw + clampi(ax + i, 0, p.rw - 1); + t = l_load_texel(color, A, B, C, idx, p.e); + } + l_gather_add(g0, t, u, v0); + l_gather_add(g1, t, u, v1); + } + } + + float h0[3], h1[3] = {0.0f, 0.0f, 0.0f}; + l_finish_pixel(g0, x, y0, px, py0, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, display, h0); + if (has1) + l_finish_pixel(g1, x, y0 + 1, px, py1, active, p, s, color, hist_prev, m_near, m_center, m_far, hist_out, + display, h1); + + // Reduced history: mean of H^ over the 2 x 2 block (x, x + 1) x (y0, y0 + 1); lane x + 1 is lane ^ 1. +#pragma unroll + for (int i = 0; i < 3; ++i) sh_red[lane * 3 + i] = h0[i] + h1[i]; + wg_barrier(); + if (active && has1 && (x & 1) == 0 && x + 1 < p.dw) { + u32 code = 0; +#pragma unroll + for (int i = 0; i < 3; ++i) { + const float m = (sh_red[lane * 3 + i] + sh_red[(lane ^ 1) * 3 + i]) * 0.25f; + const float v = clampf(enc_e(m) * (1.0f / 2.45f), 0.0f, 1.0f); + code |= (u32)wg_rint(v * 1023.0f) << (10 * i); + } + reduced[(y0 / 2) * (p.dw / 2) + x / 2] = code; + } +} diff --git a/clean/l/reference/recon.py b/clean/l/reference/recon.py index 9fb0d5b1..19c0832e 100644 --- a/clean/l/reference/recon.py +++ b/clean/l/reference/recon.py @@ -11,8 +11,11 @@ The learned scale s is the post stage's tensor "s". Everything else (positions, the score and candidate rules, the 12-tap Catmull-Rom, the blend, the NaN guard, the stored outputs) is Model M's, by import. -Arithmetic is float64 with only the stored outputs rounded, as Model M's oracle: the reconstruction's half-precision -and fused-rounding forms (e.g. the gather exponent) are a matter of the kernel-order model, not of this oracle. +The colour gather ("gather": "half", Model L) runs in the model's own precision as the spec side confirmed it (target +positions, FP16 offsets and exponent with the fused cross term, float32 weights flushed to zero below 2^-126, FP16 +weights and half accumulators, float32 weight sum: gather_half). Everything after it is float64 with only the stored +outputs rounded, as Model M's oracle; the remaining kernel-order details (the hardware exp2, reciprocal and +divisions, the Catmull-Rom weight forms) are a matter of the kernel-order model. ldr=True gives the LDR form: identity transfer, no exposure (e = e_prev = 1), codes from the values directly; the display is the 11 / 11 / 10-bit codes of the plain colour (ldr_display_codes), which the output stage unpacks with code RN32(1/N) and filters without a decode or an exposure (output_filter). Both confirmed by the spec side. @@ -22,8 +25,8 @@ NaN (the decode passes NaN through: dec), alpha = 0, and the NaN guard substitutes C, which is NaN. The stored history is then h(E(NaN)) = -1.4404296875 (HDR) or NaN (LDR), the display and the reduced code 0. -The reconstruction kernel's summation order (taps, candidates) is not fixed yet; this oracle sums in float64 and is -the tolerance reference, the kernel's order decides the kernel-order model. +The gather's tap order is i (columns) outer, j (rows) inner; the other sums of this oracle are float64 (the tolerance +reference). The reconstruction variant (3/2 or 3/1) only decides how a kernel tiles the work; both forms compute this function (table.recon_variant). @@ -110,6 +113,70 @@ def gather(feat, C, px, py, rw, rh, window): return W, fbar +def _h(v): + """Round to FP16 (from float64, one rounding).""" + with np.errstate(over="ignore", invalid="ignore"): + return np.asarray(v, np.float64).astype(np.float16).astype(np.float64) + + +def _f32(v): + with np.errstate(over="ignore", invalid="ignore"): + return np.asarray(v, np.float64).astype(np.float32).astype(np.float64) + + +def target_positions_f32(rw, rh, dw, dh, jx, jy): + """Render-space positions in the model's precision: rx = float32(rw) / float32(dw), px = fused(x + 1/2, rx, jx) + (one float32 rounding), py = fused(ry, y + 1/2, jy).""" + rx, ry = _f32(np.float32(rw) / np.float32(dw)), _f32(np.float32(rh) / np.float32(dh)) + px = _f32((np.arange(dw) + 0.5) * rx + _f32(jx)) + py = _f32((np.arange(dh) + 0.5) * ry + _f32(jy)) + return px, py + + +def gather_half(feat, C, px, py, rw, rh, window): + """Step 2 in the model's precision (spec side, confirmed for Model L and Model M): + - per render pixel: A = h(f32(a -0.721325)), B = h(f32(b -0.721325)), C = h(f32(c -1.44265)); the nine features + in FP16 (feat must already be FP16 values); + - offsets: dx = h(f32((ax + 1/2) - px)); d_y = h(f32((b_y + 1/2) - py)), minus 1 (half) when the row's own + b_y = floor(py - 1/2) is not the pair's b; u = h(dx + i), v = h(d_y + j); + - exponent: eps = fused_half(h(u v), C, h(h(h(u u) A) + h(h(v v) B))) (C1); + - w = 2^eps in float32, flushed to +0 below 2^-126; W = float32 sum of the unrounded w in tap order; + - F_k = fused_half(h(w), f_k, F_k), from +0; r = 1 / W, f-bar_k = float32(r F_k). + Tap order: i (columns) outer, j (rows) inner. Fused half multiply-adds are evaluated in float64 and rounded once + to FP16 (exact unless the addends' exponents differ by more than about 30, far outside these values).""" + ax, b = _post.kernel_window(px, py) + Cc = np.asarray(C, np.float64) + kA = _h(_f32(Cc[..., 0] * _f32(-0.721325))) + kB = _h(_f32(Cc[..., 1] * _f32(-0.721325))) + kC = _h(_f32(Cc[..., 2] * _f32(-1.44265))) + dh, dw = py.size, px.size + by = np.floor(py - 0.5).astype(np.int64) + dx = _h(_f32((ax + 0.5) - px))[None, :] + dyv = _h(_f32((by + 0.5) - py)) + dyv = np.where(by != b, _h(dyv - 1.0), dyv)[:, None] + W = np.zeros((dh, dw)) + F = np.zeros((dh, dw, feat.shape[2])) + offs = range(-1, window - 1) + with np.errstate(invalid="ignore", over="ignore", under="ignore"): + for i in offs: + sx = np.clip(ax + i, 0, rw - 1) + u = _h(dx + i) + for j in offs: + sy = np.clip(b + j, 0, rh - 1) + v = _h(dyv + j) + a_, b_, c_ = kA[sy][:, sx], kB[sy][:, sx], kC[sy][:, sx] + s_ = _h(_h(_h(u * u) * a_) + _h(_h(v * v) * b_)) + eps = _h(_h(u * v) * c_ + s_) + w = _f32(np.exp2(eps)) + w = np.where(w < 2.0 ** -126, 0.0, w) + W = _f32(W + w) + hw = _h(w)[..., None] + F = _h(hw * feat[sy][:, sx] + F) + r = _f32(1.0 / W) + fbar = _f32(r[..., None] * F) + return W, fbar + + def reconstruct(x, s, t=_table.TABLE, ldr=False): """Stage "post" for Model L. x: Model M's reconstruction input dict (clean/reference/post.py reconstruct); s: the post stage's learned scale. Returns Model M's output dict ("history", "reduced", "display", "_lin").""" @@ -131,11 +198,15 @@ def reconstruct(x, s, t=_table.TABLE, ldr=False): "reset)") s = float(np.asarray(s, np.float64).reshape(-1)[0]) - px, py = _post.target_positions(rw, rh, dw, dh, jx, jy) with np.errstate(invalid="ignore", over="ignore"): ec = enc_in(e * color) feat = np.concatenate([ec, np.asarray(x["B"], np.float64), np.asarray(x["A"], np.float64)], axis=2) - W, fbar = gather(feat, x["C"], px, py, rw, rh, window) + if rc.get("gather", "float64") == "half": + px, py = target_positions_f32(rw, rh, dw, dh, jx, jy) + W, fbar = gather_half(_h(feat), x["C"], px, py, rw, rh, window) + else: + px, py = _post.target_positions(rw, rh, dw, dh, jx, jy) + W, fbar = gather(feat, x["C"], px, py, rw, rh, window) cbar, sig, gbar, bh, bo = fbar[..., 0:3], fbar[..., 3:6], fbar[..., 6], fbar[..., 7], fbar[..., 8] # 3. motion candidates (Model M's rules) diff --git a/clean/l/reference/table.py b/clean/l/reference/table.py index 6cc540d4..39ec847f 100644 --- a/clean/l/reference/table.py +++ b/clean/l/reference/table.py @@ -79,7 +79,8 @@ "dec_pixel": "l", # The reconstruction (clean/l/reference/recon.py): Model M's (spec 06) with a "window" x "window" colour gather, # history candidates combined in linear colour ("history_combine"), and the plain linear colour as the fallback - # of an invalid or reset sample ("fallback"). Model M: window 3, "encoded", "encoded_colour". The output stage + # of an invalid or reset sample ("fallback"); "gather": "half" = the colour gather in the model's precision + # (recon.gather_half). Model M: window 3, "encoded", "encoded_colour", "float64". The output stage # (D -> output filter) is Model M's unchanged. "post_variant" names Model L's kernel set (one function, both # tilings). "post_variant": "unfolded", @@ -88,7 +89,7 @@ # ratio (single-precision division and comparison), else "3/2" (recon_variant). The ratio is D / render, not # output / render: 3.0 here is about output >= 2 x render. "recon": {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 4, - "history_combine": "linear", "fallback": "linear_colour"}, + "history_combine": "linear", "fallback": "linear_colour", "gather": "half"}, # Window offsets (ox, oy) of an L block for frame parity 0 and 1, in pixels; the blocks run on the padded grid F # and keep every grid position's output. "window_offsets": {"first": ((2, 2), (0, 2)), "second": ((0, 0), (2, 0))}, @@ -102,7 +103,7 @@ # Model M's reconstruction in the same terms (clean/reference/post.py reconstruct): for reductions and tests. M_RECON = {"rule": "d_over_render_at_least", "ratio": 3.0, "variants": ("3/2", "3/1"), "window": 3, - "history_combine": "encoded", "fallback": "encoded_colour"} + "history_combine": "encoded", "fallback": "encoded_colour", "gather": "float64"} def recon_size(output_w, output_h): diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 118dd0da..d42d00c2 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -50,7 +50,7 @@ def compile_kernels(build, rocm, with_swin=False, clang=None): clean_process_environment(rocm) clang = clang or B.find_clang(rocm) kdir = os.path.join(ROOT, "clean", "kernels") - for name in ("l_swin", "l_ops"): + for name in ("l_swin", "l_ops", "l_post"): B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", name + ".hip"), os.path.join(build, name + ".co")) + ["-I", kdir]) B.run(B.device_command(clang, rocm, os.path.join(kdir, "encoder.hip"), os.path.join(build, "l_enc_values.co"), diff --git a/clean/l/runner/l_post_check.py b/clean/l/runner/l_post_check.py new file mode 100644 index 00000000..97708844 --- /dev/null +++ b/clean/l/runner/l_post_check.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the reconstruction kernel l_post_recon (clean/l/kernels/l_post.hip, HDR) against +the oracle (clean/l/reference/recon.py), synthetic data only. + + python clean/l/runner/l_post_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--seeds N] + --json out.json + +/l_post.co comes from clean/l/runner/l_c32_check.py --compile-only. The GPU part runs in a GPU slot. Output: +counts only, no timing. + +Harness: every output buffer starts filled with a non-zero sentinel byte and is followed by sentinel slack. Every +defined output byte is compared: the full history's RGB (FP16) and alpha (must be 0), the display planes' pixels, and +the reduced codes' three 10-bit fields and top 2 bits (must be 0); no byte of any slack may change. A negative +control (the same case with the jitter moved by half a render pixel) must fail the criteria. + +Criteria. Values: Model M's acceptance for its own reconstruction kernel (clean/tests/test_post_emu.py +test_matches_reference): no non-finite value, at most 0.01 % of FP16 values more than one ulp off, at most 1 % differ, +mean absolute error at most 0.1 % of the reference RMS; reduced codes at most 1 off, at most 1 % differ. These are +Model M's thresholds (float32 kernel against a float64 oracle); they are to be tightened when a kernel-order model of +the reconstruction exists. The oracle's colour gather runs in the model's precision (confirmed by the spec side), so +the kernel and the oracle agree on which weights underflow: no pixel is excluded. +Known answers (C = (65504, 65504, 0), D = R, every target offset from a texel centre by the jitter): offset 0.1, total +weight 0 everywhere: history -1.4404296875, display 0, reduced code 0; offset 0.05, total weight > 0 but every FP16 +weight 0: the output is the candidate history (here the reset fallback: the linear colour times e). +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import recon as R # noqa: E402 +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.reference import post as MP # noqa: E402 +from clean.runner import post as RP # noqa: E402 + +S = 2.9796233 +SENTINEL = 0xA5 +SLACK = 64 +# (rw, rh, dw, dh, reset, kind) +CASES = [(24, 16, 54, 36, False, "random"), (31, 17, 70, 38, False, "random"), (20, 12, 60, 36, False, "random"), + (33, 19, 99, 57, True, "random"), (17, 11, 38, 25, False, "extreme_c"), + (16, 12, 16, 12, False, "offset_0.1"), (16, 12, 16, 12, True, "offset_0.05")] + + +def make_case(rng, rw, rh, dw, dh, reset, kind): + x = MP.random_inputs(rng, rw, rh, dw, dh, reset=reset) + x["exposure"], x["exposure_prev"] = np.float32(rng.uniform(0.5, 2.0)), np.float32(rng.uniform(0.5, 2.0)) + if kind != "random": + x["C"] = np.broadcast_to(np.array([65504.0, 65504.0, 0.0], np.float32), np.shape(x["C"])).copy() + if kind.startswith("offset_"): + x["jitter"] = np.array([float(kind.split("_")[1]), 0.0], np.float32) + return x + + +def run_kernel(hip, mod, x, s): + """Launch l_post_recon with sentinel-filled outputs; returns the raw output buffers (with slack) and sizes.""" + prm = RP.post_params(x) + data = RP.post_device_inputs(x) + dw, dh = prm.dw, prm.dh + bufs = {} + for k, v in data.items(): + bufs[k] = hip.malloc(len(v)) + hip.upload(bufs[k], v) + blob = RP.pack({"s": np.array(s)}) + pw = hip.malloc(len(blob)) + hip.upload(pw, blob) + sizes = {"history": dw * dh * 8, "reduced": max(4, (dw // 2) * (dh // 2) * 4), "display": RP.display_bytes(dw, dh)} + outs = {} + for k, n in sizes.items(): + outs[k] = hip.malloc(n + SLACK) + hip.memset(outs[k], SENTINEL, n + SLACK) + args = [ctypes.c_void_p(bufs[k]) for k in ("color", "A", "B", "C", "hist_prev", "m_near", "m_center", "m_far")] + args += [ctypes.c_void_p(pw), ctypes.c_void_p(outs["history"]), ctypes.c_void_p(outs["reduced"]), + ctypes.c_void_p(outs["display"]), prm, ctypes.c_void_p(0)] + hip.launch(hip.get_function(mod, "l_post_recon"), RP.recon_grid(dw, dh), 256, args) + hip.sync() + raw = {k: np.frombuffer(hip.download(outs[k], n + SLACK), np.uint8) for k, n in sizes.items()} + for ptr in list(bufs.values()) + [pw] + list(outs.values()): + hip.free(ptr) + return raw, sizes, (dw, dh) + + +def decode(raw, sizes, dims): + """Every defined byte: history RGB + alpha, display pixels, reduced fields + top bits; slack integrity.""" + dw, dh = dims + hist = raw["history"][:sizes["history"]].view(np.float16).reshape(dh, dw, 4) + red = raw["reduced"][:sizes["reduced"]].view(np.uint32)[:(dw // 2) * (dh // 2)].reshape(dh // 2, dw // 2) + disp = RP.decode_planes(raw["display"][:sizes["display"]].tobytes(), (dh, dw)) + return {"history": hist[..., :3].astype(np.float32), + "alpha_zero": bool((hist[..., 3].view(np.uint16) == 0).all()), + "reduced_codes": np.stack([red & 1023, (red >> 10) & 1023, (red >> 20) & 1023], -1).astype(np.int64), + "reduced_top_zero": bool(((red >> 30) == 0).all()), + "display": disp, + "slack_untouched": all(bool((raw[k][sizes[k]:] == SENTINEL).all()) for k in sizes)} + + +def m_rule(got, ref): + res = {} + for k in ("history", "display"): + g, r = np.asarray(got[k], np.float32), np.asarray(ref[k], np.float32) + fin = np.isfinite(g) + s = C32.f16_steps(np.where(fin, g, 0), r) + d = np.abs(np.where(fin, g, 0.0).astype(np.float64) - r) + res[k] = {"values": int(s.size), "nonfinite": int(np.sum(~fin)), "frac_over_1": float(np.mean(s > 1)), + "differ": int(np.sum(s > 0)), "frac_diff": float(np.mean(s > 0)), "max_steps": int(s.max()), + "mean_rel_rms": float(d.mean() / max(np.sqrt(np.mean(np.square(r, dtype=np.float64))), 1e-30))} + res[k]["pass"] = (res[k]["nonfinite"] == 0 and res[k]["frac_over_1"] <= 1e-4 and res[k]["frac_diff"] <= 0.01 + and res[k]["mean_rel_rms"] <= 1e-3) + rc = np.rint(np.asarray(ref["reduced"], np.float64) * 1023.0).astype(np.int64) + dc = np.abs(got["reduced_codes"] - rc) + res["reduced"] = {"values": int(dc.size), "differ": int(np.sum(dc > 0)), "max": int(dc.max(initial=0))} + res["reduced"]["pass"] = res["reduced"]["max"] <= 1 and res["reduced"]["differ"] <= 0.01 * max(dc.size, 1) + res["pass"] = all(res[k]["pass"] for k in ("history", "display", "reduced")) + return res + + +def check_case(hip, mod, x, kind, s=S): + raw, sizes, dims = run_kernel(hip, mod, x, s) + got = decode(raw, sizes, dims) + ref = R.reconstruct(x, s) + m = m_rule(got, ref) + m.update({"alpha_zero": got["alpha_zero"], "reduced_top_zero": got["reduced_top_zero"], + "slack_untouched": got["slack_untouched"]}) + ok = m["pass"] and got["alpha_zero"] and got["reduced_top_zero"] and got["slack_untouched"] + W = ref["_lin"]["W"] + m["weight_zero_pixels"] = int((W == 0).sum()) + if kind == "offset_0.1": + known = ((got["history"] == np.float32(-1.4404296875)).all() and (got["display"] == 0).all() + and (got["reduced_codes"] == 0).all() and m["weight_zero_pixels"] == W.size) + m["known_answer"] = bool(known) + ok = ok and known + if kind == "offset_0.05": + Hl = np.asarray(x["color"], np.float64) * float(np.asarray(x["exposure"])) + known = bool((W > 0).all()) and np.array_equal(got["history"], MP.f16(MP.enc(Hl))) + m["known_answer"] = bool(known) + ok = ok and known + m["pass"] = bool(ok) + return m + + +def run(build, rocm, seeds=(89,)): + from clean.runner.hip import Hip + hip = Hip(rocm) + with open(os.path.join(build, "l_post.co"), "rb") as f: + mod = hip.load_module(f.read()) + res = {"cases": []} + for seed in seeds: + rng = np.random.default_rng(seed) + for rw, rh, dw, dh, reset, kind in CASES: + x = make_case(rng, rw, rh, dw, dh, reset, kind) + m = check_case(hip, mod, x, kind) + m["case"] = [rw, rh, dw, dh, reset, kind] + res["cases"].append(m) + # Negative control: the first random case with the jitter moved by half a render pixel must fail. + rng = np.random.default_rng(seeds[0]) + x = make_case(rng, *CASES[0]) + raw, sizes, dims = run_kernel(hip, mod, dict(x, jitter=np.asarray(x["jitter"], np.float32) + 0.5), S) + neg = m_rule(decode(raw, sizes, dims), R.reconstruct(x, S)) + res["negative_control_fails"] = not neg["pass"] + res["pass"] = all(c["pass"] for c in res["cases"]) and res["negative_control_fails"] + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + for c in res["cases"]: + print(json.dumps({k: c[k] for k in c if k not in ("history", "display", "reduced")}), + {k: (c[k]["differ"], c[k].get("max_steps", c[k].get("max"))) for k in ("history", "display", "reduced")}) + print({"negative_control_fails": res["negative_control_fails"], "pass": res["pass"]}) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/tests/test_l_recon.py b/clean/l/tests/test_l_recon.py index 229deebc..100ca9b2 100644 --- a/clean/l/tests/test_l_recon.py +++ b/clean/l/tests/test_l_recon.py @@ -95,7 +95,7 @@ def test_reset_uses_the_linear_colour(self): color = np.asarray(x["color"], np.float64) rh, rw = color.shape[:2] dw, dh = x["recon_size"] - px, py = MP.target_positions(rw, rh, dw, dh, *np.asarray(x["jitter"], np.float64)) + px, py = R.target_positions_f32(rw, rh, dw, dh, *np.asarray(x["jitter"], np.float64)) fy, fx = MP.point_index(px, py, rw, rh) w = lin["w1"] + np.where(lin["w2"] == 0, 0.0, lin["w2"]) H_lin = w[..., None] * color[fy, fx] * float(x["exposure"]) @@ -122,7 +122,7 @@ def test_linear_combination(self): rh, rw = color.shape[:2] dw, dh = x["recon_size"] e, ep = float(x["exposure"]), float(x["exposure_prev"]) - px, py = MP.target_positions(rw, rh, dw, dh, *np.asarray(x["jitter"], np.float64)) + px, py = R.target_positions_f32(rw, rh, dw, dh, *np.asarray(x["jitter"], np.float64)) fy, fx = MP.point_index(px, py, rw, rh) X = np.arange(dw)[None, :] + 0.5 Y = np.arange(dh)[:, None] + 0.5 @@ -182,6 +182,7 @@ def test_ldr_constant_image_end_to_end(self): x = _inputs(rng, rw=20, rh=12) x["color"] = np.full(np.shape(x["color"]), 0.5, np.float32) x["history"] = np.full(np.shape(x["history"]), 0.5, np.float32) + x["C"] = np.zeros(np.shape(x["C"]), np.float32) # every weight exactly 1: the half gather is exact dw, dh = x["recon_size"] ow, oh = (2 * dw) // 3, (2 * dh) // 3 self.assertEqual(T.recon_size(ow, oh), (dw, dh)) @@ -198,6 +199,53 @@ def test_ldr_constant_image_end_to_end(self): self.assertEqual(np.float32(1 / 2047).view(np.uint32), 0x3A001002) self.assertEqual(np.float32(1 / 1023).view(np.uint32), 0x3A802008) + def _extreme(self, offset, reset): + """D = R, every target `offset` render pixels right of a texel centre (jitter), C = (65504, 65504, 0).""" + rng = np.random.default_rng(12) + x = MP.random_inputs(rng, 16, 12, 16, 12, reset=reset) + x["jitter"] = np.array([offset, 0.0], np.float32) + x["exposure"], x["exposure_prev"] = np.float32(1.5), np.float32(0.75) + x["C"] = np.broadcast_to(np.array([65504.0, 65504.0, 0.0], np.float32), np.shape(x["C"])).copy() + for k in ("m_near", "m_center", "m_far"): + x[k] = np.zeros(np.shape(x[k]), np.float32) + x["history"] = np.full(np.shape(x["history"]), 0.75, np.float32) # E-encoded, constant + return x + + def test_gather_precision_constants(self): + """A = B = h(f32(65504 -0.721325)) = -47264; at offset 0.1: u = h(0.1) = 0.0999755859375, h(u u) = + 0.0099945068359375, eps = -472.5.""" + self.assertEqual(R._h(R._f32(65504.0 * R._f32(-0.721325))), -47264.0) + u = R._h(0.1) + self.assertEqual(u, 0.0999755859375) + self.assertEqual(R._h(u * u), 0.0099945068359375) + self.assertEqual(R._h(R._h(u * u) * -47264.0), -472.5) + + def test_offset_005_is_the_candidate_history(self): + """Offset 0.05 (between the h(w) = 0 and w = 0 thresholds): W > 0, every FP16 weight 0, all features 0, so the + output is candidate 1's history: with reset the linear colour times e; without, D(history) / e_prev times e + (the history is constant and every position valid).""" + for reset in (True, False): + x = self._extreme(0.05, reset) + out = R.reconstruct(x, 2.9796233) + W = out["_lin"]["W"] + self.assertTrue((W > 0).all() and np.isfinite(W).all()) + e, ep = 1.5, 0.75 + if reset: + Hl = np.asarray(x["color"], np.float64) * e # D = R, floor p = the pixel itself + else: + Hl = np.full(out["history"].shape, float(MP.dec(0.75)) / ep * e) + np.testing.assert_allclose(out["_lin"]["H_hat"], Hl, rtol=1e-12) + np.testing.assert_allclose(out["_lin"]["O_hat"], Hl, rtol=1e-12) + + def test_offset_01_is_zero_weight(self): + """Offset 0.1 (beyond 0.0516): W = 0 everywhere, the confirmed zero-weight values.""" + x = self._extreme(0.1, False) + out = R.reconstruct(x, 2.9796233) + self.assertTrue((out["_lin"]["W"] == 0).all()) + self.assertTrue((out["history"] == np.float32(-1.4404296875)).all()) + self.assertTrue((out["display"] == 0).all()) + self.assertTrue((out["reduced"] == 0).all()) + def test_history_of_another_size_needs_reset(self): rng = np.random.default_rng(11) x = _inputs(rng)