diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index a8bdcb5e..725dd18a 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -296,6 +296,18 @@ "k_expo_scalars_direct", "k_io.co" ], + [ + "k_out_color", + "k_io.co" + ], + [ + "k_out_pixel", + "k_io.co" + ], + [ + "k_out_pixel_c", + "k_io.co" + ], [ "k_stage2_d0_hidden", "k_block.co" diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 480d2c73..b0a942aa 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -30,7 +30,11 @@ and starts filled with its run's own sentinel (HIP `0xA5`, D3D12 `0x5A`); the de slack must keep its sentinel; special values are among the inputs; a negative control (the port given another input than HIP) must be caught; Model M's `scale_rgb` runs through the same replay as a control. Output: counts and verdicts. -Families: `input` (the exact tiled input stage `kx/k_in_tiled` of `k_io_exact.co`: 16 flag sets of HDR, low-resolution +Families: `output` (the output stage of `k_io.co`: `k_out_color` followed by `k_out_pixel_c` or `k_out_pixel`, the +three game-output formats (RGBA32F, RGBA16F rows, RGB32F), colour alphas that are negative, above 1 or out of the FP16 +range, special values in the colour, depth, motion and head map; the stored colour taps and the filter-tap values of +`k_out_color` are compared as well as the image and the histories; the raw bytes are identical in every case; the `_f` +kernels of `k_fast` and `k_out_hidden` are not ported); `input` (the exact tiled input stage `kx/k_in_tiled` of `k_io_exact.co`: 16 flag sets of HDR, low-resolution motion, inverted depth and reset, odd and tiny base grids down to 1 x 1, the exposure scalars from a device buffer as the runtime passes them, infinity, huge values, zero and -0 in the colour, depth, motion and histories, NaN in a third of the cases; the raw bytes are identical in every case; the port needs the adapter's binary64 fma and division (manifest diff --git a/clean/k/dx12/hlsl/k_out.hlsli b/clean/k/dx12/hlsl/k_out.hlsli new file mode 100644 index 00000000..af105b92 --- /dev/null +++ b/clean/k/dx12/hlsl/k_out.hlsli @@ -0,0 +1,510 @@ +// SPDX-License-Identifier: MIT +// Model K output stage helpers (ko_*) for the D3D12 ports of k_out_color / k_out_pixel(_c) and their direct forms +// (clean/k/dx12/kernels/k_out.hlsl). WrongGPU's own code: HLSL twins of our HIP output stage, clean/k/kernels/k_io.hip +// (spec 43 sections 3 - 7 as that file implements them) and of the exact helpers it uses from clean/k/kernels/k_fp.h. +// Include after wg_platform.hlsli (and native_direct.hlsli for WG_DIRECT_TEXTURES). +// +// NOTE (group G2 io-out): this header holds its own copies of the few k_fp.h / k_io.hip pieces the output stage needs +// (the compact KOutParams argument block, kio_* fetches, kio_encode / kio_decode, the binary64 log / exp, kh_* FP16 +// steps). Group G1 owns the shared IO headers clean/k/dx12/hlsl/k_fp.hlsli / k_frame.hlsli, which did not exist when +// this port was written; G2 does not edit them. The coordinator may move these pieces there when merging. +// +// ARGUMENTS: one KOutParams by value, manifest token s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 (52 DWORDs; the +// full struct would be 63 > 57). Skipped fields and why they are dead on the runtime path (clean/k/runtime/runtime_k.cpp +// KRuntime::frame): +// cbx, cby, mbx, mby, zbx, zby (x24) the subrect bases: the runtime zero-initialises KFrameIn (fin = {}) and never +// sets them (its inputs are dense render-size images); the port uses 0. +// kappa, rho, lam (x12) the runtime sets them to 1 and always passes KFrameIn.xs (k_expo_scalars' GPU +// result), and ko_params / kio_frame replace kappa, rho, lam and KOutUni.kinv by +// xs[0..3] whenever xs is set; the port reads xs always (lam is unused here). +// pad_ (x4) padding. +// exp2tab (x8) h(2^E) table read only by the host emulator's ko_exp2h; the GPU kernel uses +// v_exp_f16 (as this port: exp2 on half, checked with RGA, see k_out.hlsl). +#pragma once + +WG_ARGS_BEGIN + uint f_color, f_motion, f_depth, f_xs; // KFrameIn pointers (p4) + int f_cpitch, f_mpitch, f_zpitch; // (b12); cbx .. zby skipped (x24) + int f_W, f_H, f_rw, f_rh, f_Bw, f_Bh, f_flags; // (b52) + float f_jx, f_jy, f_sx, f_sy, f_mox, f_moy; + float f_s; int f_mch; float f_rmsx, f_rmsy, f_rmox, f_rmoy; // kappa / rho / lam skipped (x12); (b24); pad_ (x4) + uint p_N, p_L, p_Hc, p_table; // (p4); exp2tab skipped (x8) + uint p_out, p_Lout, p_Hcout, p_Hhout; // (p4) + int p_opitch, p_omode; // (b8) + uint p_ctap, p_ctapT; // (p2) + float u_invW, u_invH, u_rhox, u_rhoy, u_rho2x, u_rho2y, u_jmx, u_jmy, u_spx, u_spy, u_invW16, u_invH16, u_kinv; + uint u_kuxkvy; // KOutUni (b56): kux (low 16 bits), kvy (high) +WG_ARGS_END + +#define KIO_HDR 1 +#define KIO_MVLO 2 +#define KIO_INV 4 +#define KIO_RESET 8 +#define KIO_C16 16 +#define KIO_M16 32 +#define KIO_OUT_RGBA32F 0 +#define KIO_OUT_RGBA16F 1 +#define KIO_OUT_RGB32F 2 + +// opaque zero (the launch's workgroup count is below 2^31): x ^ ko_z is x, but the D3D12 shader compiler cannot see +// through it. Used where the driver was seen to regroup chains of products by a shared / uniform factor or to fuse +// FP16 multiply + add (wg_math.hlsli "DRIVER EXCEPTIONS"); HIP keeps every such product as written (contract off). +#define ko_z (wg_grid_ >> 31) +float kof(float x) { return asfloat(asuint(x) ^ ko_z); } +double kod(double x) { uint lo, hi; asuint(x, lo, hi); return asdouble(lo ^ ko_z, hi); } +half koh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)ko_z)); } +uint kh2_u(half2 h) { const uint16_t2 v = asuint16(h); return (uint)v.x | ((uint)v.y << 16); } +half2 kh2_h(uint u) { return asfloat16(uint16_t2((uint16_t)(u & 0xffffu), (uint16_t)(u >> 16))); } +// FP16 pairs held as packed 32-bit words (low half = element 0, kp_*): every operation unpacks, operates on the half2 +// and repacks. Both halves of a pair then stay in one register (the wave64 kernels get v_*_f16 on its .l / .h halves; +// the driver issues v_pk_*_f16 only in wave32 kernels) and the opaque zero of a product is one 32-bit xor on the +// pair, xor'ed into both halves (ko_z * 0x10001: with a 0 / 1 value in the low half only, the driver proved the high +// half's xor zero, k_common.hlsli); the per-half 16-bit form split every pair into separate registers. +uint kp_splat(half h) { return (uint)asuint16(h) * 0x10001u; } +uint kp_pack(half lo, half hi) { return (uint)asuint16(lo) | ((uint)asuint16(hi) << 16); } +half kp_lo(uint u) { return asfloat16((uint16_t)(u & 0xffffu)); } +half kp_hi(uint u) { return asfloat16((uint16_t)(u >> 16)); } +uint kp_add(uint a, uint b) { return kh2_u(kh2_h(a) + kh2_h(b)); } +uint kp_sub(uint a, uint b) { return kh2_u(kh2_h(a) - kh2_h(b)); } +// product rounded to FP16 as its own instruction (the opaque on the packed word: no fusion into a following add) +uint kp_mul(uint a, uint b) { return kh2_u(kh2_h(a) * kh2_h(b)) ^ (ko_z * 0x10001u); } +// product without the opaque, for products that feed only a multiply or a multiplicand (no add can absorb them; each use +// checked in the ISA: the same v_mul_f16 / v_fma_f16 counts, no v_fma_mix*) +uint kp_mul_n(uint a, uint b) { return kh2_u(kh2_h(a) * kh2_h(b)); } +uint kp_fma(uint a, uint b, uint c) { return kh2_u(mad(kh2_h(a), kh2_h(b), kh2_h(c))); } +uint kp_max(uint a, uint b) { return kh2_u(max(kh2_h(a), kh2_h(b))); } +uint kp_sat_nz(uint a) { return kh2_u(min(max(kh2_h(a), (half2)0), (half2)1)); } +uint kp_rni(uint a) { return kh2_u(round(kh2_h(a))); } +uint kp_abs(uint a) { return a & 0x7fff7fffu; } +uint kp_opq(uint a) { return a ^ (ko_z * 0x10001u); } +// the same with one operand a scalar FP16 value used in both halves (no splatted register) +uint kp_rsub_s(half s, uint a) { return kh2_u((half2)s - kh2_h(a)); } +uint kp_add_s(uint a, half s) { return kh2_u(kh2_h(a) + (half2)s); } +uint kp_mul_s(uint a, half s) { return kh2_u(kh2_h(a) * (half2)s) ^ (ko_z * 0x10001u); } +uint kp_mul_s_n(uint a, half s) { return kh2_u(kh2_h(a) * (half2)s); } // (no opaque: see kp_mul_n) +uint kp_fma_s(uint a, half s, uint c) { return kh2_u(mad(kh2_h(a), (half2)s, kh2_h(c))); } +uint kp_fma_ss(uint a, half s, half c) { return kh2_u(mad(kh2_h(a), (half2)s, (half2)c)); } + +// ---- FP32 helpers (k_fp.h) ------------------------------------------------------------------------------------------ +float kf_bits(uint b) { return asfloat(b); } +bool kf_finite(float x) { return (asuint(x) & 0x7f800000u) != 0x7f800000u; } // = x - x == 0 +int ki_min(int a, int b) { return a < b ? a : b; } +int ki_max(int a, int b) { return a > b ? a : b; } +int ko_clampi(int i, int n) { return ki_max(ki_min(i, n - 1), 0); } +int kio_ceil16(int n) { return (n + 15) / 16 * 16; } +// R(t): round half away from zero +float kf_R(float t) +{ + const float a = abs(t); + const float fl = floor(a); + const float r = (a - fl) >= 0.5f ? fl + 1.0f : fl; + return t < 0.0f ? -r : r; +} +// kf_sat (x != x ? 0 : clamp(x, 0, 1), -0 kept) as HIP ko_sat (x == 0 ? x : med3(x, 0, 1)): equal for every input, +// also with FTZ (a denormal compares equal to 0 and is passed through as kf_sat passes it) +float ko_sat(float x) { return x == 0.0f ? x : min(max(x, 0.0f), 1.0f); } + +// ---- FP16 (k_fp.h kh_*; HLSL half arithmetic is v_*_f16 / v_pk_*_f16, round to nearest even, FP16 denormals kept) -- +// RN16 of an FP32 value (HIP kh_round / (_Float16) x = v_cvt_f16_f32); the D3D12 casts round toward zero. +// KO_HW16 = 1: the hardware conversion (v_cvt_f16_f32, round to nearest even) through the driver channel's op 0x1C, +// immediate 1 (clean/dx12/hlsl/wg_ags.hlsli; GPU-checked bit-equal to RNE incl. subnormals, inf, NaN, divergent calls; +// never immediates 2 / 3, whose rounding mode is not restored); 0: the software RNE wg_f2h. Same bits either way. +#ifndef KO_HW16 +#define KO_HW16 1 +#endif +#if KO_HW16 +#if WG_WAVE != 32 +#include "wg_ags.hlsli" // (wg_platform.hlsli includes it for wave32 kernels) +#endif +// the input through the opaque zero: the driver folded the FP32 operation feeding the conversion into it (a - b then +// RN16 became one v_fma_mixlo_f16, one rounding where HIP rounds twice) +uint ko_f2h(float x) { return wg_ags_call(0x1cu, 0u, 1u, asuint(x) ^ ko_z, 0u); } // high half 0 +#else +uint ko_f2h(float x) { return wg_f2h(x); } +#endif +half kh_round(float x) { return asfloat16((uint16_t)ko_f2h(x)); } +half kh_bits(uint b) { return asfloat16((uint16_t)b); } +uint kh_u(half h) { return (uint)asuint16(h); } +// exact FP16 -> FP32 (the RTZ conversion of an FP16-representable value is exact too) +half kh_exact(float x) { return (half)x; } +// kh_sat (x != x ? 0 : clamp(x, 0, 1), -0 kept) as the clamp with a zero passed through: minNum / maxNum give +0 for NaN +// and negative x, the zero test keeps -0 (as HIP's ko_sat for FP32) +half kh_sat(half x) { return x == (half)0 ? x : min(max(x, (half)0), (half)1); } +// a * b rounded to FP16 as its own instruction (no fusion into a following add: koh) +half kh_mul(half a, half b) { return koh(a * b); } +// kh_div: the correctly rounded FP32 quotient rounded to FP16 (innocuous double rounding) +// a / b rounded to nearest even (HIP '/'), = wg_div for every input. Operands with exponent fields in [92, 162] +// (|a|, |b| in [2^-35, 2^36): every nonzero finite FP16 value, the stage's usual FP32 operands) run wg_div's own +// sequence without its scaling: the reciprocal is formed on the scaled divisor bs in [1, 2) exactly as wg_div does and +// scaled back by the exact power of two 2^(127 - eb); every later step is then wg_div's step times a power of two with +// all values normal (quotient within 2^+-71, residuals >= 2^-82), so each rounds to the scaled bits of wg_div's result, +// and wg_div's exponent reassembly gives that same value. Other operands (zero, inf, NaN, far exponents): wg_div. +float ko_div(float a, float b) +{ + const uint ua = asuint(a), ub = asuint(b); + const uint ea = (ua >> 23) & 255u, eb = (ub >> 23) & 255u; + const float bs = asfloat((ub & 0x807fffffu) | 0x3f800000u); + float ys = rcp(bs); + ys = mad(mad(-bs, ys, 1.0f), ys, ys); + const float y = ys * asfloat((254u - eb) << 23); + float q = a * y; + q = mad(mad(-b, q, a), y, q); + q = mad(mad(-b, q, a), y, q); + [branch] if (ea - 92u > 70u || eb - 92u > 70u) q = wg_div(a, b); + return q; +} +half kh_div(half a, half b) { return kh_round(ko_div((float)a, (float)b)); } + +// ---- binary64 log / 2^x (k_fp.h kd_*) --------------------------------------------------------------------------------- +// HIP's kf_ln / kf_exp2 (FP32 double-word path, binary64 path for undecided inputs) equal (float)kd_log((double)x) / +// (float)kd_exp2((double)y) on every input (k_fp.h: exhaustive GPU check, every normal positive binary32 input for +// log2 / ln and every input in [-125, 127) for 2^x; outside those domains HIP takes the binary64 path itself). The port +// therefore runs the binary64 functions directly (rare far branches of kio_encode / kio_decode only; the double-word +// path is a speed form for a later round). Doubles: IEEE operations of the D3D12 doubles (division / fma need the +// extended-doubles feature, as model M's exposure port uses); the double -> float rounding is done in software +// (RNE, as v_cvt_f32_f64) so it does not depend on the driver's conversion mode. +float kd_to_f(double d) +{ + uint lo, hi; + asuint(d, lo, hi); + const uint s = hi & 0x80000000u, e = (hi >> 20) & 0x7ffu; + const uint m = ((hi & 0xfffffu) << 3) | (lo >> 29), rest = lo & 0x1fffffffu; + const int ef = (int)e - 1023 + 127; + uint r = s | ((uint)max(ef, 0) << 23) | m; + r += (rest > 0x10000000u || (rest == 0x10000000u && (m & 1u) != 0u)) ? 1u : 0u; // carries into the exponent + r = ef >= 255 ? s | 0x7f800000u : r; + r = ef <= 0 ? s : r; // FP32 denormal results flush (D3D12 FTZ; the stage's values never get there) + r = e == 0x7ffu ? s | 0x7f800000u | (((hi & 0xfffffu) | lo) != 0u ? 0x400000u | m : 0u) : r; + return asfloat(r); +} +double kd_rcp_near2(double d) +{ + double q = (double)rcp(kd_to_f(d)); + q = fma(q, fma(-d, q, (double)1.0), q); + q = fma(q, fma(-d, q, (double)1.0), q); + return q; +} +#define kd_one ((double)1.0) +// ln x for finite x > 0 (kd_log), NaN / 0 / negative / inf handled as k_fp.h kd_log_special +double kd_log(double x) +{ + uint lo, hi; + asuint(x, lo, hi); + if (x != x || x < 0.0) return asdouble(0u, 0x7ff80000u); + if (x == 0.0) return asdouble(0u, 0xfff00000u); + if (((hi >> 20) & 0x7ffu) == 0x7ffu) return x; + int e = (int)((hi >> 20) & 0x7ffu) - 1023; + double m = asdouble(lo, (hi & 0xfffffu) | 0x3ff00000u); + if (m > asdouble(0x667F3BCDu, 0x3FF6A09Eu)) { + m = m * (double)0.5; + e += 1; + } + const double s = (m - kd_one) * kd_rcp_near2(m + kd_one); + const double s2 = s * s; + double p = asdouble(0x590B2164u, 0x3FA642C8u); // 1/23 + p = fma(p, s2, asdouble(0x18618618u, 0x3FA86186u)); // 1/21 + p = fma(p, s2, asdouble(0xBCA1AF28u, 0x3FAAF286u)); // 1/19 + p = fma(p, s2, asdouble(0x1E1E1E1Eu, 0x3FAE1E1Eu)); // 1/17 + p = fma(p, s2, asdouble(0x11111111u, 0x3FB11111u)); // 1/15 + p = fma(p, s2, asdouble(0x13B13B14u, 0x3FB3B13Bu)); // 1/13 + p = fma(p, s2, asdouble(0x745D1746u, 0x3FB745D1u)); // 1/11 + p = fma(p, s2, asdouble(0x1C71C71Cu, 0x3FBC71C7u)); // 1/9 + p = fma(p, s2, asdouble(0x92492492u, 0x3FC24924u)); // 1/7 + p = fma(p, s2, asdouble(0x9999999Au, 0x3FC99999u)); // 1/5 + p = fma(p, s2, asdouble(0x55555555u, 0x3FD55555u)); // 1/3 + p = fma(p, s2, kd_one); + const double l = kod((double)2.0 * s) * p; // 2 s p as (2 s) p + return fma((double)e, asdouble(0xFEFA39EFu, 0x3FE62E42u), l); +} +double kd_ldexp(double v, int n) +{ + [unroll] for (int i = 0; i < 2; ++i) { + const int k = n > 1023 ? 1023 : (n < -1022 ? -1022 : n); + v = v * asdouble(0u, (uint)(k + 1023) << 20); + n -= k; + } + return v; +} +double kd_exp2(double y) +{ + if (y != y) return y; + if (y > 1100.0) return asdouble(0u, 0x7ff00000u); + if (y < -1100.0) return 0.0; + const double mg = asdouble(0u, 0x43380000u); // 1.5 2^52: (y + mg) - mg = rint(y) for |y| < 2^51 + const double n = kod(y + mg) - mg; + const double r = kod((y - n) * asdouble(0xFEFA39EFu, 0x3FE62E42u)); + double p = kd_one; + p = fma(p, r * asdouble(0x00000000u, 0x3FB00000u), kd_one); // 1/16 + p = fma(p, r * asdouble(0x11111111u, 0x3FB11111u), kd_one); // 1/15 + p = fma(p, r * asdouble(0x92492492u, 0x3FB24924u), kd_one); // 1/14 + p = fma(p, r * asdouble(0x13B13B14u, 0x3FB3B13Bu), kd_one); // 1/13 + p = fma(p, r * asdouble(0x55555555u, 0x3FB55555u), kd_one); // 1/12 + p = fma(p, r * asdouble(0x745D1746u, 0x3FB745D1u), kd_one); // 1/11 + p = fma(p, r * asdouble(0x9999999Au, 0x3FB99999u), kd_one); // 1/10 + p = fma(p, r * asdouble(0x1C71C71Cu, 0x3FBC71C7u), kd_one); // 1/9 + p = fma(p, r * asdouble(0x00000000u, 0x3FC00000u), kd_one); // 1/8 + p = fma(p, r * asdouble(0x92492492u, 0x3FC24924u), kd_one); // 1/7 + p = fma(p, r * asdouble(0x55555555u, 0x3FC55555u), kd_one); // 1/6 + p = fma(p, r * asdouble(0x9999999Au, 0x3FC99999u), kd_one); // 1/5 + p = fma(p, r * asdouble(0x00000000u, 0x3FD00000u), kd_one); // 1/4 + p = fma(p, r * asdouble(0x55555555u, 0x3FD55555u), kd_one); // 1/3 + p = fma(p, r * asdouble(0x00000000u, 0x3FE00000u), kd_one); // 1/2 + p = fma(p, r, kd_one); // r * 1 + return kd_ldexp(p, (int)n); +} +float kf_exp2(float y) { return kd_to_f(kd_exp2((double)y)); } +float kf_ln(float x) { return kd_to_f(kd_log((double)x)); } + +// spec 42 decode factor / spec 43 encode (k_io.hip kio_decode / kio_encode) +#define kKioDecodeNear kf_bits(0x3F7FBE88u) +#define kKioEncodeNear 1000.0f +float kio_decode(float t) +{ + [branch] if (t < kKioDecodeNear) return ko_div(1.0f, 1.0f - t); + const float e = kof(kof((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu)); + return wg_div(kf_exp2(e) - 1.0f, 9.0f * t); +} +float kio_encode(float Y) +{ + [branch] if (Y < kKioEncodeNear) return ko_div(1.0f, Y + 1.0f); + return wg_div(wg_div(kf_ln(mad(Y, 9.0f, 1.0f)), 9.0f) - kf_bits(0x3C4FAE29u), Y); +} + +// ---- frame view (ko_params): the exposure scalars from xs ------------------------------------------------------------ +float ko_kappa() { return wg_ldf(f_xs, 0u); } +float ko_rho() { return wg_ldf(f_xs, 4u); } +float ko_kinv() { return wg_ldf(f_xs, 12u); } + +// colour texel (x, y) of the dense render image (k_io.hip kio_texel; subrect base 0): RGBA f32, or RGBA16F (KIO_C16) +float4 kio_texel(int x, int y) +{ +#if defined(WG_DIRECT_TEXTURES) + // direct form: the game's colour texel as the interop's input pass writes it into the dense buffer for model K: raw + // RGB (not divided by the pre-exposure: model K applies kappa) and the texel's own alpha, unmodified (issue #26: + // the output alpha follows it, ko_color_tap's A). A colour format without an alpha channel reads 1, the value + // the texture fetch defines for a missing component. (Not nd_raw_color_xy, model M's form with alpha +0, which K's + // input stage keeps: it reads RGB only.) + return native_direct_color_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)); +#else + const uint i = ((uint)y * (uint)f_cpitch + (uint)x) * 4u; + [branch] if ((f_flags & KIO_C16) != 0) { + const uint2 h = wg_ld64(f_color, i * 2u); + return float4(wg_h2f_lo(h.x), wg_h2f_hi(h.x), wg_h2f_lo(h.y), wg_h2f_hi(h.y)); + } + return wg_ld4f(f_color, i * 4u); +#endif +} + +float ko_depth_at(int x, int y) // dense depth texel (x, y) inside the render image +{ +#if defined(WG_DIRECT_TEXTURES) + return native_direct_depth_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)); +#else + return wg_ldf(f_depth, ((uint)y * (uint)f_zpitch + (uint)x) * 4u); +#endif +} + +// k_io.hip ko_motion (subrect base 0). Direct form: the raw value of the game's motion texture t2 (not model M's +// nd_motion_xy, which applies b1's scale and offset per texel); ko_dsp / ko_dmo apply them (below). +// Direct forms (issue #26): the motion scale and offset of the game's motion texture t2 come in b1 (ms, mo; the +// executor passes 1 / 0 in WgRtFrame), and they are applied as the buffer forms and the reference (clean/k/reference +// instage.py / outstage.py, spec 42 / 43) apply WgRtFrame's mv_scale / mv_bias: the texel's raw value r plus the offset +// in raw units, times the motion factor with the scale folded in (sp = RN(s / RN(RN(1 / n) r)) of the host, s = the +// scale). Before, the direct forms scaled every texel (RN(ms r) + mo: input; one fused mad(ms, r, mo): output) and +// used the factor of scale 1: with a non-unit scale the products round differently, which moved history taps (0.02-0.04 % +// of the output and history values differed from the buffer form from the first frame after a reset, in-game packs). +// Offset: b1's mo is in render pixels after the scale (mo = ms b, one FP32 product); the raw-unit offset is RN(mo / ms) +// (exactly 0 for mo = 0; equal to the buffer drive's b whenever that quotient recovers it). +void ko_motion(int x, int y, out float mvx, out float mvy) +{ +#if defined(WG_DIRECT_TEXTURES) + const float2 r = native_direct_motion_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)).xy; + mvx = r.x; + mvy = r.y; +#else + const uint t = (uint)y * (uint)f_mpitch + (uint)x; + [branch] if ((f_flags & KIO_M16) != 0) { + const uint i = t * (uint)f_mch; + mvx = wg_h2f(wg_ld16(f_motion, i * 2u)); // KIO_M16: the raw value (ko_dsp / ko_dmo apply ms / mo) + mvy = wg_h2f(wg_ld16(f_motion, i * 2u + 2u)); + return; + } + const float2 m = asfloat(wg_ld64(f_motion, t * 8u)); + mvx = m.x; + mvy = m.y; +#endif +} + +// the motion factor with a texture's scale folded in (direct forms: b1's ms; KIO_M16: f_rmsx ..): k_io.py out_uniforms' +// spx = s / rhox [MVlo], else s; s = sx ms; and the raw-unit offset +// KIO_M16 (the interop's raw-buffer route: input copies or model K's compact input pass, ms / mo in KFrameIn rmsx .., +// WgRtFrame's scale 1): the raw FP16 value r, the scale folded into the motion factor and the offset RN(mo / ms) in raw +// units, as for the direct forms (issue #26; before: ms r + mo per texel, then the factor of scale 1). +float ko_dsp(float s, float ms, float rho, bool mvlo) { const float sm = s * ms; return mvlo ? ko_div(sm, rho) : sm; } +float ko_dmo(float o, float mo, float ms) { return mo == 0.0f ? o : o + ko_div(mo, ms); } + +// spec 42 5.2 [MVlo]: offset of the nearest-depth motion pixel (k_io.hip ko_nearest; kio_tap(i, d, 0, n) = clamp) +void ko_nearest(int ix, int iy, out int odx, out int ody) +{ + float c, a, b, d, e; +#if !defined(WG_DIRECT_TEXTURES) + // HIP KO_ADDR_FAST: centre at least one texel inside the render image: nothing is clamped, the five taps are one + // byte offset (of texel (ix - 1, iy - 1)) + 0 / 1 / 2 pitches and 0 / 1 / 2 texels + [branch] if (ix >= 1 && ix <= f_rw - 2 && iy >= 1 && iy <= f_rh - 2) { + const uint zp = (uint)f_zpitch * 4u, voff = ((uint)(iy - 1) * (uint)f_zpitch + (uint)(ix - 1)) * 4u; + c = wg_ldf(f_depth + zp + 4u, voff); + a = wg_ldf(f_depth, voff); + b = wg_ldf(f_depth + 8u, voff); + d = wg_ldf(f_depth + 2u * zp, voff); + e = wg_ldf(f_depth + 2u * zp + 8u, voff); + } else +#endif + { + const int x0 = ko_clampi(ix - 1, f_rw), x1 = ko_clampi(ix, f_rw), x2 = ko_clampi(ix + 1, f_rw); + const int y0 = ko_clampi(iy - 1, f_rh), y1 = ko_clampi(iy, f_rh), y2 = ko_clampi(iy + 1, f_rh); + c = ko_depth_at(x1, y1); + a = ko_depth_at(x0, y0); + b = ko_depth_at(x2, y0); + d = ko_depth_at(x0, y2); + e = ko_depth_at(x2, y2); + } + const float k = kf_bits(0x3A83126Fu); + const bool inv = (f_flags & KIO_INV) != 0; + bool edge; + if (inv) + edge = (c < 0.5f * (a + e) - k * c) || (c < 0.5f * (b + d) - k * c); + else + edge = (c > mad(c, k, 0.5f * (a + e))) || (c > mad(c, k, 0.5f * (b + d))); + int bx = 0, by = 0; + if (edge) { + float best = c; + const float v[4] = {a, b, d, e}; + const int ox[4] = {-1, 1, -1, 1}, oy[4] = {-1, -1, 1, 1}; + [unroll] for (int q = 0; q < 4; ++q) { + const bool nearer = inv ? v[q] > best : v[q] < best; + if (nearer) { + best = v[q]; + bx = ox[q]; + by = oy[q]; + } + } + } + odx = bx; + ody = by; +} + +// ---- output colour taps (spec 43 4.2 / 5.3; k_io.hip ko_stored_tap_from_raw, k_out_color) --------------------------- +void kio_ycocg(float R, float G, float B, out float Y, out float Co, out float Cg) +{ + Y = (0.25f * R + 0.5f * G) + 0.25f * B; + Co = mad(R, 0.5f, kof(0.0f) * G) - 0.5f * B; // the zero opaque: 0 G stays an operation (NaN for inf G) + Cg = (0.5f * G - 0.25f * R) - 0.25f * B; +} +float kio_luma_fma(float R, float G, float B) { return mad(B, 0.25f, mad(R, 0.25f, 0.5f * G)); } + +// stored tap (Y, Co, Cg, A) as 2 x 2 FP16 bits, and the filter-tap value T, of a raw colour c +void ko_color_tap(float4 c, out uint2 tap, out uint tT) +{ + const float kappa = ko_kappa(); + const bool hdr = (f_flags & KIO_HDR) != 0; + float Y, Co, Cg; + kio_ycocg(max(c.x, 0.0f), max(c.y, 0.0f), max(c.z, 0.0f), Y, Co, Cg); + uint hY, hCo, hCg; + // each product as written: kappa Y, (kappa Co) e, (kappa Cg) e (opaque, so the shared kappa / e are not regrouped) + const float kY = kof(kappa * Y), kCo = kof(kappa * Co), kCg = kof(kappa * Cg); + [branch] if (hdr) { + const float e = kio_encode(kY); + hY = ko_f2h(kY * e); + hCo = ko_f2h(kCo * e); + hCg = ko_f2h(kCg * e); + } else { + hY = ko_f2h(kY); + hCo = ko_f2h(kCo); + hCg = ko_f2h(kCg); + } + tap = uint2(hY | (hCo << 16), hCg | (ko_f2h(c.w) << 16)); + const float kL = kof(kappa * kio_luma_fma(max(c.x, 0.0f), max(c.y, 0.0f), max(c.z, 0.0f))); + float vT = kL; + [branch] if (hdr) vT = kL * kio_encode(kL); + tT = ko_f2h(vT); +} + +// ---- output stage helpers (k_io.hip) ---------------------------------------------------------------------------------- +// beta quotient of the axis coefficients (ko_div_axis: v_rcp_f32, one Newton step, residual correction) +float ko_div_axis(float a, float b) +{ + float r = rcp(b); + r = mad(mad(-b, r, 1.0f), r, r); + const float q = a * r; + return mad(mad(-b, q, a), r, q); +} +// selector normalisation: h(nv * rcp(h(1 + |nv|))) +half ko_selector_normalize(half nv) +{ + const half den = koh((half)1 + abs(nv)); // opaque: no FP16 add folded into the FP32 conversion + return kh_round((float)nv * rcp((float)den)); +} + +struct KoAxis { // spec 43 3.6 per-axis cubic coefficients + int n; + float c0, c1, c2, c3, beta, betap; +}; +KoAxis ko_axis(float w) +{ + KoAxis a; + const float q = mad(w - 0.5f, 2.0f, 0.5f); + const float t = floor(q - 0.5f) + 0.5f; + const float f = ko_sat(q - t); + const float f2 = f * f, f3 = kof(f2) * f; + a.c0 = f2 - 0.5f * (f + f3); + a.c1 = (1.5f * f3 - 2.5f * f2) + 1.0f; + a.c3 = 0.5f * (f3 - f2); + a.c2 = ((1.0f - a.c0) - a.c1) - a.c3; + a.beta = ko_div_axis(a.c1, a.c1 + a.c2); + a.betap = 1.0f - a.beta; + const float nn = t - 0.5f; + const int nc = (int)max(min(nn, 1.0e9f), -1.0e9f); + a.n = nn != nn ? -1000000000 : nc; + return a; +} + +float ko_ell(half v) +{ + const float x = (float)v, c = kf_bits(0x3F7D70A4u); + return x < c ? x : mad(x - c, 0.001f, c); +} +float ko_cubic(float Am1, float A0, float A1, float A2, KoAxis a) +{ + const float lam = 1.37f; + float r = mad(a.beta, A0, a.betap * A1); + r = mad(lam * a.c0, ko_sat(Am1), r); + r = mad(lam * (a.c1 - a.beta), ko_sat(A0), r); + r = mad(lam * (a.c2 - a.betap), ko_sat(A1), r); + return mad(lam * a.c3, ko_sat(A2), r); +} +// ko_cubic for taps known to lie in [0, 1) (ko_sat is then the identity): the same operations without the clamps +float ko_cubic_unit(float Am1, float A0, float A1, float A2, KoAxis a) +{ + const float lam = 1.37f; + float r = mad(a.beta, A0, a.betap * A1); + r = mad(lam * a.c0, Am1, r); + r = mad(lam * (a.c1 - a.beta), A0, r); + r = mad(lam * (a.c2 - a.betap), A1, r); + return mad(lam * a.c3, A2, r); +} +// Correctly rounded reciprocal: one Newton step on v_rcp_f32 in [2^-32, 2^32] (HIP ko_rcp_refined, exhaustive GPU check), +// the IEEE division elsewhere +float ko_rcp_refined(float x) +{ + [branch] if (x >= asfloat(0x2f800000u) && x <= asfloat(0x4f800000u)) { + const float r = rcp(x); + return mad(mad(-x, r, 1.0f), r, r); + } + return wg_div(1.0f, x); +} +// the same without the guard, for denominators the callers bound to [2^-32, 2^32] (ko_rcp_output_bounded) +float ko_rcp_output_bounded(float x) +{ + const float r = rcp(x); + return mad(mad(-x, r, 1.0f), r, r); +} diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index bee8e3f5..f24b35ea 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -18,7 +18,9 @@ the half-grid control (the port again with half its workgroups) is used by the families whose grid has more than one workgroup; - Model M's scale_rgb (frame.co, its port already proven) runs through the same replay as a control. -The families: input = the exact tiled input stage (k_io_exact.co, kx/k_in_tiled) on frames of every flag combination (HDR, +The families: output = the output stage (k_io.co: k_out_color with k_out_pixel_c or k_out_pixel) on the same frame +flags, the three game-output formats (RGBA32F, RGBA16F rows, RGB32F), colour alphas that are negative, above 1 or out of the +FP16 range, special values in the colour, depth, motion and head; input = the exact tiled input stage (k_io_exact.co, kx/k_in_tiled) on frames of every flag combination (HDR, low-resolution motion, inverted depth, reset), odd and tiny base grids, exposure scalars from a device buffer, special values in the colour, depth, motion and histories (the same three levels as the stage families); wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, @@ -53,6 +55,7 @@ FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "narrow": (["k_block.co"], NARROW_KERNELS), "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), + "output": (["k_io.co"], ["k_out_color", "k_out_pixel_c", "k_out_pixel"]), "wide": (["k_block_wide.co"], ["k_stagew2_e3", "k_stagew4_e4", "k_stagew2_e4", "k_stagew4_m", "k_stagew4_d4", "k_stagew2_d4", "k_stagew2_d3"])} CONTROL_CO = "frame.co" @@ -421,7 +424,8 @@ def check_narrow(state, read): if not raw and it["specials"] == 2: # NaN inputs: a NaN's sign / payload is not defined (see nan_equivalent_differ) for n, v in per.items(): if not v["identical"] and v["slack_bad"] == 0: - d, same = nan_equivalent_differ(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n]) + d, same = nan_equivalent_differ(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n], + it.get("widths", {}).get(n, 2)) v["differ_outside_nan"], v["nan_elements_differing_in_sign"] = d, same nan_only += same ok_case = raw or all(v["identical"] or (v["slack_bad"] == 0 and v.get("differ_outside_nan", 1) == 0) @@ -469,7 +473,7 @@ def input_specials(x, rng, level): for k in ("Hc", "Hh"): a = x[k] pick = rng.random(a.shape) < 0.01 - a[pick] = rng.choice(pool.astype(np.float16), int(pick.sum())) + a[pick] = rng.choice(pool.astype(np.float32), int(pick.sum())).astype(np.float16) def run_input(build, rocm, seeds, cdir, hip): @@ -533,6 +537,75 @@ def run_input(build, rocm, seeds, cdir, hip): return items, lines, neg +# ---- family output: the output stage (k_io.co) --------------------------------------------------------------------- + +OUT_BYTES = {0: 16, 1: 8, 2: 12} # bytes per pixel of KOutParams.omode 0 RGBA32F, 1 RGBA16F rows, 2 RGB32F + + +def run_output(build, rocm, seeds, cdir, hip): + from clean.k.dx12 import k_plan_bridge as PB + from clean.k.runner import k_io, k_stages + from clean.k.tests.test_k_io_emu import frame + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed + 900) + for ci, (hdr, mvlo, inv, reset) in enumerate(INPUT_FLAG_SETS): + base = INPUT_BASES[(ci + si) % len(INPUT_BASES)] + special = (ci // 2) % 3 + omode, compact = (ci + si) % 3, (ci // 3 + si) % 2 == 0 + name = f"out_s{si}_{ci}" + x = frame(rng, hdr, mvlo, inv, reset=reset, base=base) + x["base"] = base + # the colour input's alpha: negative, above 1 and out of the FP16 range values among them + x["color"][..., 3] = rng.uniform(-1.0, 2.0, x["color"].shape[:2]).astype(np.float32) + x["color"][rng.random(x["color"].shape[:2]) < 0.01, 3] = 7e4 + if special: + input_specials(x, rng, special) + x["N"][rng.random(x["N"].shape) < 0.002] = np.array(np.inf if special == 1 else np.nan, np.float16) + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, "output@compact" if compact else "output", b"", x, None, None) + try: + launches = [ln for ln in plan["launches"] if ln[0] != "k_out_hidden"] + prm = launches[0][3][0] + xs = np.array([prm.f.kappa, prm.f.rho, prm.f.lam, np.float32(1.0) / np.float32(prm.f.kappa)], np.float32) + ptr = pdev.malloc(xs.nbytes) + pdev.upload(ptr, xs.tobytes()) + plan["bufs"]["xs"], plan["sizes"]["xs"] = ptr, xs.nbytes + prm.f.xs = ptr + prm.f.kappa, prm.f.rho, prm.f.lam = -17.0, 7.0, 9.0 + prm.omode, prm.opitch = omode, prm.f.W + plan["launches"] = launches + W, H = prm.f.W, prm.f.H + plan["outputs"] = [(n, sh) for n, sh in plan["outputs"] if n != "Hhout"] # k_out_hidden is not ported + # k_out_color's own results too: the stored colour tap (4 FP16) and the filter-tap value (FP16) per pixel + plan["outputs"] += [("ctap", (prm.f.rh, prm.f.rw, 4)), ("ctapT", (prm.f.rh, prm.f.rw))] + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + for k, grid, block, a in plan["launches"]: + hip.h.launch(hip.h.get_function(hip.mods["k_io.co"], k), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {n: min(int(np.prod(sh)) * 2, sizes[n]) for n, sh in plan["outputs"]} + defined["out"] = W * H * OUT_BYTES[omode] + kw = dict(modules=["k_io.co"], outputs=outputs) + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, **kw) + if neg is None and plan["launches"][1][1] >= 4 and not reset: + neg = name + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, prefix="negative_", grid_div=2, + **kw) + items.append({"case": name, "kernel": "k_out_pixel_c" if compact else "k_out_pixel", "stage": "output", + "size": list(base), "offset": [0, 0], "specials": special, "outputs": outputs, + "hip": got, "defined": defined, "widths": {"out": 2 if omode == 1 else 4}}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + return items, lines, neg + + # ---- run ----------------------------------------------------------------------------------------------------------- def run(build, rocm, families, seeds, json_out): @@ -556,6 +629,9 @@ def run(build, rocm, families, seeds, json_out): if "input" in families: state["input"] = run_input(build, rocm, seeds, cdir, hip) lines += state["input"][1] + if "output" in families: + state["output"] = run_output(build, rocm, seeds, cdir, hip) + lines += state["output"][1] if "wide" in families: state["wide"] = run_stages(WIDE, "k_block_wide.co", "wid", build, rocm, seeds, cdir, hip, WIDE_SIZES) lines += state["wide"][1] @@ -611,7 +687,7 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 - for fam in ("narrow", "wide", "input"): + for fam in ("narrow", "wide", "input", "output"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) ok = ok and good diff --git a/clean/k/dx12/kernels/k_out.hlsl b/clean/k/dx12/kernels/k_out.hlsl new file mode 100644 index 00000000..dd19d394 --- /dev/null +++ b/clean/k/dx12/kernels/k_out.hlsl @@ -0,0 +1,589 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of model K's output stage (spec 43): k_out_color, k_out_pixel_c (compact pixel head, the default) +// and k_out_pixel (k_head=full). The forms that read the game's textures (_direct, WG_DIRECT_TEXTURES) are in this source +// for the runtime's use but are not declared yet: they are declared and checked with the runtime wiring. WrongGPU's own code. +// Basis: our HIP kernels in clean/k/kernels/k_io.hip (HIP build k_io.co, wave64, 128 lanes per workgroup), exact forms +// only (FAST = false; the k_fast=1 "_f" kernels are not ported and keep running on HIP). Same names, same argument +// struct (KOutParams, see hlsl/k_out.hlsli for the s-token and why each skipped field is dead), same launch geometry +// and lane <-> pixel mapping (k_out_pixel: 64 x 2 pixels per workgroup, one row per wave64, the F2 group exchange of +// Ybar through LDS between the two waves). +// +// Bit-identity notes (every deviation from the HIP source and why it gives the same bits): +// * FP32 steps: one IEEE operation each as written (build_dx12.py exact mode), mad() where HIP has kf_fma; products +// whose factors the driver could regroup (a shared or uniform factor: kappa, e, q, Yo, kk, d, kinv, rho) pass an +// opaque identity (kof) so each is rounded as HIP rounds it. +// * FP32 -> FP16: ko_f2h (hlsl/k_out.hlsli): the hardware v_cvt_f16_f32 (round to nearest even, as kh_round) through +// the driver channel's op 0x1C with its input through the opaque zero (KO_HW16 = 0: the software wg_f2h, same +// bits); the D3D12 casts round toward zero and are used only for values that are FP16 numbers already (exact). +// * FP16 steps: native half arithmetic (v_*_f16, nearest even, FP16 denormals kept, as HIP's _Float16), the HIP +// kernel's packed pairs as packed 32-bit words (kp_*); every FP16 product that a later add could absorb passes the +// opaque zero (koh / kp_mul: no v_fma / v_fma_mix fusion), and so does every FP16 arithmetic result that is +// converted to FP32 (cbar, the 5.4 sums, beta, beta_c, the selector denominator: the driver folds an FP16 fma / add +// followed by the conversion into one v_fma_mix_f32, with no FP16 rounding in between). The E products dv dv, +// (dv dv) tA and du du feed only a multiply, a multiplicand or an fma addend (no add can absorb them) and go +// without the opaque (kp_mul_n / kp_mul_s_n; RGA: the same v_mul_f16 / v_fma_f16 / v_fma_mix counts, every square +// kept). Wave64 kernels get no v_pk_* from this driver; wave32 (with a 16 x 2 tile per wave, F2 by ballot) measured +// slower (721 vs 640 us), as did wave64 with a 32 x 2 tile per wave (y no longer wave-uniform: +10 us). +// * Divisions: ko_div (= wg_div, HIP's correctly rounded '/', on every input; wg_div's own steps without its operand +// scaling where the exponents allow it, wg_div itself elsewhere); kh_div = FP32 quotient rounded once to FP16. +// * Hardware estimates as HIP: rcp() = v_rcp_f32 (ko_div_axis, ko_selector_normalize, the bounded reciprocals with +// their Newton step), exp2(half) = v_exp_f16 (ko_exp2h; checked in the RGA listing: v_exp_f16, no FP32 detour). +// * kio_encode / kio_decode far branches: the binary64 functions (hlsl/k_out.hlsli, equal to HIP's kf_ln / kf_exp2). +// * HIP's fast addressing forms (KO_ADDR_FAST / KO_WIN_FAST / KO_TAP_FAST: the same loads without per-tap clamps +// where nothing is clamped) as [branch] forms with the clamped general forms for the border lanes; KO_SOFT_MAX's +// maximum test as one unsigned maximum of the widened window's FP32 bits (below 0x3F7D8000 = FP16 bits below +// 0x3bec, the same order on every input) (below 0x3bec = every tap in [0, 0.99): no soft +// limit and unclamped row cubics, HIP's ko_cubic_unit; any other window: soft limit and clamped cubics, the same +// values as HIP's per-tap decision). +// * Loops whose first iteration differs are peeled (DXC kept the 12-tap loop with an n == 0 test rolled, the taps then +// lived in indexed register arrays); the non-plain best-E retry recomputes E from opaque inputs (merged with the +// main loop's E it kept all of them live: 143 VGPRs). +// * FP32 denormals flush on D3D12 (wg_math.hlsli); HIP keeps them. The stage's values (FP16 data, FP32 colour) do +// not produce FP32 denormals; the port check covers the runtime's cases. +// +// @wg-kernel k_out_color co=k_io.co block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 +// @wg-kernel k_out_pixel_c co=k_io.co block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 defines=KO_COMPACT=1 +// @wg-kernel k_out_pixel co=k_io.co entry=k_out_pixel_c block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 defines=KO_COMPACT=0 +#include "wg_platform.hlsli" +#if defined(WG_DIRECT_TEXTURES) +#include "native_direct.hlsli" +#endif +#include "k_out.hlsli" + +#ifndef KO_COMPACT +#define KO_COMPACT 1 +#endif +// The game output's alpha (issue #26): the colour input's alpha of the current frame, spatial only. It is cbar3, the +// fourth channel of the 4.3 bilinear below: the stored tap's A = RN16(the colour texel's fourth value, exactly as the +// game supplies it: no pre-exposure, no exposure scale, no clamp) at the same four render taps as Y / Co / Cg (tap +// positions clamped to the render rectangle), with the same four FP16 weights: RN16(w0 a0), then three FP16 fused +// multiply-adds in the tap order. No history, nothing from the network; HDR and LDR alike. Every output form stores it +// unchanged: the RGBA16F rows as its FP16 bits, the direct texture store and the dense RGBA32F form as its exact FP32 +// widening (the dense RGB32F form has no fourth value). HIP's k_io.hip writes the same; the interop's passes carry the +// input alpha into the buffer forms' colour (convert.cpp, model K). + +// Per render pixel: the stored colour tap (4.2) and the filter-tap value T (5.3) (HIP k_out_color) +WG_KERNEL(k_out_color) +{ + WG_SETUP(); + const int id = (int)(wg_block_id() * 128u + wg_lane_id()); + if (id >= f_rw * f_rh) + return; + const int i = id % f_rw, j = id / f_rw; + uint2 tap; + uint tT; + ko_color_tap(kio_texel(i, j), tap, tT); + wg_st64(p_ctap, (uint)id * 8u, tap); + wg_st16(p_ctapT, (uint)id * 2u, tT); +} + +groupshared uint lds_ybar[128]; + +// bilinear sample of the colour history (k_io.hip ko_bilinear4_load / _math, clamp to edge) +float4 ko_bilinear_hc(int w, int h, float cx, float cy) +{ + const float px = cx * (float)w - 0.5f, py = cy * (float)h - 0.5f; + const float fx0 = floor(px), fy0 = floor(py); + const float fx = px - fx0, fy = py - fy0; + int x0 = (int)max(min(fx0, (float)w), -1.0f); + int y0 = (int)max(min(fy0, (float)h), -1.0f); + x0 = fx0 != fx0 ? 0 : x0; + y0 = fy0 != fy0 ? 0 : y0; + uint2 q00, q10, q01, q11; + // HIP KO_ADDR_FAST: all four corners inside (no clamp): one byte offset, the corners at +0 / +8 bytes on two rows + [branch] if ((uint)x0 <= (uint)(w - 2) && (uint)y0 <= (uint)(h - 2)) { + const uint voff = ((uint)y0 * (uint)w + (uint)x0) * 8u, r1 = p_Hc + (uint)w * 8u; + q00 = wg_ld64(p_Hc, voff); + q10 = wg_ld64(p_Hc + 8u, voff); + q01 = wg_ld64(r1, voff); + q11 = wg_ld64(r1 + 8u, voff); + } else { + const int xa = ko_clampi(x0, w), xb = ko_clampi(x0 + 1, w); + const int ya = ko_clampi(y0, h), yb = ko_clampi(y0 + 1, h); + q00 = wg_ld64(p_Hc, ((uint)ya * (uint)w + (uint)xa) * 8u); + q10 = wg_ld64(p_Hc, ((uint)ya * (uint)w + (uint)xb) * 8u); + q01 = wg_ld64(p_Hc, ((uint)yb * (uint)w + (uint)xa) * 8u); + q11 = wg_ld64(p_Hc, ((uint)yb * (uint)w + (uint)xb) * 8u); + } + const float omfy = 1.0f - fy; + const float4 v00 = float4(wg_h2f_lo(q00.x), wg_h2f_hi(q00.x), wg_h2f_lo(q00.y), wg_h2f_hi(q00.y)); + const float4 v10 = float4(wg_h2f_lo(q10.x), wg_h2f_hi(q10.x), wg_h2f_lo(q10.y), wg_h2f_hi(q10.y)); + const float4 v01 = float4(wg_h2f_lo(q01.x), wg_h2f_hi(q01.x), wg_h2f_lo(q01.y), wg_h2f_hi(q01.y)); + const float4 v11 = float4(wg_h2f_lo(q11.x), wg_h2f_hi(q11.x), wg_h2f_lo(q11.y), wg_h2f_hi(q11.y)); + const float4 t0 = v00 + (v10 - v00) * fx; + const float4 t1 = v01 + (v11 - v01) * fx; + return t0 * omfy + t1 * fy; +} + +// E of the 12 taps (index 3 l + tc) for a sub-sample pair at (us, vs), in the spec's operation order (HIP each_E); FP16 +// pairs as packed words (kp_*) +void ko_each_E(uint us, uint vs, half lines[4], half cols[4], half tA, half tB, half tC, out uint E[12]) +{ + [unroll] for (int l = 0; l < 4; ++l) { + const uint dv = kp_rsub_s(lines[l], vs); + const uint t2v = kp_mul_s_n(kp_mul_n(dv, dv), tA); + const uint t3v = kp_mul_s(dv, tB); + uint du = kp_rsub_s(cols[l], us); + uint mm = kp_fma(du, t3v, t2v); + [unroll] for (int tc = 0; tc < 3; ++tc) { + if (tc > 0) { + du = kp_add_s(du, (half)1); + mm = kp_add(mm, t3v); + } + E[3 * l + tc] = kp_fma_s(kp_mul_n(du, du), tC, mm); + } + } +} + +// 3.6 luma history value of the four sub-samples from the (soft-limited) 5 x 5 window: row cubics (unit: taps known +// to lie in [0, 1), ko_sat dropped), the column cubic, the clamp to the 2 x 2 neighbourhood +void ko_ellv(float Lw[5][5], KoAxis axx, KoAxis axy, bool unit, out float ellv[4]) +{ + [unroll] for (int i = 0; i < 2; ++i) + [unroll] for (int j = 0; j < 2; ++j) { + float rows[4]; + [unroll] for (int mr = 0; mr < 4; ++mr) + rows[mr] = unit ? ko_cubic_unit(Lw[i + mr][j], Lw[i + mr][j + 1], Lw[i + mr][j + 2], Lw[i + mr][j + 3], axx) + : ko_cubic(Lw[i + mr][j], Lw[i + mr][j + 1], Lw[i + mr][j + 2], Lw[i + mr][j + 3], axx); + const float val = ko_cubic(rows[0], rows[1], rows[2], rows[3], axy); + float lo = Lw[i + 1][j + 1], hi = Lw[i + 1][j + 1]; + lo = min(lo, Lw[i + 1][j + 2]); + hi = max(hi, Lw[i + 1][j + 2]); + lo = min(lo, Lw[i + 2][j + 1]); + hi = max(hi, Lw[i + 2][j + 1]); + lo = min(lo, Lw[i + 2][j + 2]); + hi = max(hi, Lw[i + 2][j + 2]); + ellv[j + 2 * i] = min(max(val, lo), hi); + } +} + +#if defined(WG_DIRECT_TEXTURES) +// the game output of a pixel: with output_rne16 the HIP output-copy bytes of an RGBA16F target (RGB rounded to nearest +// even as KIO_OUT_RGBA16F writes them), stored as the exact FP16 values (the typed store keeps them); else the FP32 +// values (the HIP path's dense f32 output goes through the Converter's typed store). Alpha: A (cbar3), an FP16 value, +// as its exact FP32 widening in both cases +void ko_store_out(int x, int y, float R, float G, float B, half A) +{ + float3 v = float3(R, G, B); + if (nd_output_rne16 != 0u) + v = float3(f16tof32(ko_f2h(R)), f16tof32(ko_f2h(G)), f16tof32(ko_f2h(B))); + native_direct_output[uint2((uint)x, (uint)y)] = float4(v, (float)A); +} +#endif + +// spec 43 output stage per pixel (HIP ko_pixel) +WG_KERNEL(k_out_pixel_c) +{ + WG_SETUP(); + const int W16 = kio_ceil16(f_W), H16 = kio_ceil16(f_H); + const int lane = (int)wg_lane_id(), tiles_x = (W16 + 63) / 64; + const int tile = (int)wg_block_id(); + const int x_raw = (tile % tiles_x) * 64 + (lane & 63); + const bool active = x_raw < W16; + const int x = active ? x_raw : W16 - 1, y = (tile / tiles_x) * 2 + wg_uniform(lane >> 6); + const bool hdr = (f_flags & KIO_HDR) != 0, mvlo = (f_flags & KIO_MVLO) != 0; + const float px = (float)x + 0.5f, py = (float)y + 0.5f; + const float rho = ko_rho(); + + // 3.1 network channels + const int u = (int)trunc(f_s * (float)x), v = (int)trunc(f_s * (float)y); + const int k = (u & 1) + 2 * (v & 1); + const int bxn = ki_min(u >> 1, f_Bw - 1), byn = ki_min(v >> 1, f_Bh - 1); +#if KO_COMPACT + const uint2 nvw = wg_ld64(p_N, (((uint)(2 * byn + (k >> 1)) * (2u * (uint)f_Bw) + (uint)(2 * bxn + (k & 1))) * 4u) * 2u); + const half nv0 = kh_bits(nvw.x & 0xffffu), nv1 = kh_bits(nvw.x >> 16), nv9 = kh_bits(nvw.y & 0xffffu); +#else + const uint ncb = (((uint)byn * (uint)f_Bw + (uint)bxn) * 40u + 10u * (uint)k) * 2u; + const half nv0 = kh_bits(wg_ld16(p_N, ncb)), nv1 = kh_bits(wg_ld16(p_N, ncb + 2u)), nv9 = kh_bits(wg_ld16(p_N, ncb + 18u)); +#endif + + // 4.1 local origin, 4.3 current-colour taps + const float o_x = kf_R(mad((float)(16 * (x / 16)) + 0.5f, u_rhox, u_jmx)) - 4.0f; + const float o_y = kf_R(mad((float)(16 * (y / 16)) + 0.5f, u_rhoy, u_jmy)) - 4.0f; + const float Pxf = mad(px, u_rhox, u_jmx) - o_x, Pyf = mad(py, u_rhoy, u_jmy) - o_y; + const half Ux = kh_round(Pxf), Uy = kh_round(Pyf); + half cbar0, cbar1, cbar2, cbar3; + { // ko_cbar_load / ko_cbar_math + const half flx = floor(Ux), fly = floor(Uy); + const half fx = kh_sat(Ux - flx), fy = kh_sat(Uy - fly); + const half one = (half)1; + half w[4]; + w[0] = kh_mul(one - fx, one - fy); + w[1] = kh_mul(fx, one - fy); + w[2] = kh_mul(one - fx, fy); + w[3] = kh_mul(fx, fy); + const int bx = (int)(o_x + (float)flx), by = (int)(o_y + (float)fly); + uint2 tq[4]; + // HIP KO_ADDR_FAST: the 2 x 2 taps inside the render image (no clamp): one byte offset, +0 / +8 bytes on two rows + [branch] if (bx >= 0 && bx <= f_rw - 2 && by >= 0 && by <= f_rh - 2) { + const uint voff = ((uint)by * (uint)f_rw + (uint)bx) * 8u; + [unroll] for (int q = 0; q < 4; ++q) tq[q] = wg_ld64(p_ctap + (uint)(q >> 1) * (uint)f_rw * 8u + 8u * (uint)(q & 1), voff); + } else { + [unroll] for (int q = 0; q < 4; ++q) { + const int tx = ko_clampi(bx + (q & 1), f_rw), ty = ko_clampi(by + (q >> 1), f_rh); + tq[q] = wg_ld64(p_ctap, ((uint)ty * (uint)f_rw + (uint)tx) * 8u); + } + } + uint lo = 0u, hi = 0u; + [unroll] for (int q = 0; q < 4; ++q) { + const uint2 t = tq[q]; + lo = q == 0 ? kp_mul_s(t.x, w[q]) : kp_fma_s(t.x, w[q], lo); + hi = q == 0 ? kp_mul_s(t.y, w[q]) : kp_fma_s(t.y, w[q], hi); + } + // FP16 results converted to FP32 below pass the opaque (kp_opq / koh): the driver folds an FP16 fma / add + // followed by the conversion into one v_fma_mix_f32 (no FP16 rounding in between) + lo = kp_opq(lo); + hi = kp_opq(hi); + cbar0 = kp_lo(lo); + cbar1 = kp_hi(lo); + cbar2 = kp_lo(hi); + cbar3 = kp_hi(hi); + } + + // 3.3 motion + float mvx, mvy; + if (mvlo) { + const int ix = (int)kf_R(mad(px, u_rhox, u_jmx)), iy = (int)kf_R(mad(py, u_rhoy, u_jmy)); + int dx, dy; + ko_nearest(ix, iy, dx, dy); + ko_motion(ko_clampi(ix + dx, f_rw), ko_clampi(iy + dy, f_rh), mvx, mvy); + } else { + const int mx = (int)kf_R(mad(px, 1.0f, -0.5f)), my = (int)kf_R(mad(py, 1.0f, -0.5f)); + ko_motion(ko_clampi(mx, f_W), ko_clampi(my, f_H), mvx, mvy); + } + // the texture's motion scale folded into the factor and its offset in raw units, as the buffer form gets them in + // WgRtFrame (hlsl/k_out.hlsli ko_motion; issue #26): direct forms b1's ms / mo, KIO_M16 f_rmsx .. +#if defined(WG_DIRECT_TEXTURES) + const bool dfold = true; + const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy; +#else + const bool dfold = (f_flags & KIO_M16) != 0; + const float dmsx = f_rmsx, dmsy = f_rmsy, dmox = f_rmox, dmoy = f_rmoy; +#endif + float spx = u_spx, spy = u_spy, ofx = f_mox, ofy = f_moy; + [branch] if (dfold) { + spx = ko_dsp(f_sx, dmsx, u_rhox, mvlo); + spy = ko_dsp(f_sy, dmsy, u_rhoy, mvlo); + ofx = ko_dmo(f_mox, dmox, dmsx); + ofy = ko_dmo(f_moy, dmoy, dmsy); + } + const float wx = mad(spx, mvx + ofx, px), wy = mad(spy, mvy + ofy, py); + + // 3.4 validity + bool valid = (f_flags & KIO_RESET) == 0 && wx * u_invW >= 0.0f && wx * u_invW <= 1.0f && wy * u_invH >= 0.0f && + wy * u_invH <= 1.0f; + + // F2: Ybar of the aligned 2 x 2 group through LDS (lanes lane, lane ^ 1, lane ^ 64, lane ^ 65) + lds_ybar[lane] = kh_u(cbar0); + + // 3.5 colour history + float4 Hh = ko_bilinear_hc(W16, H16, wx * u_invW16, wy * u_invH16); + + // 3.6 luma history: the 5 x 5 window of the four sub-samples + const KoAxis axx = ko_axis(wx), axy = ko_axis(wy); + const int LW = 2 * W16, LH = 2 * H16; + float Lw[5][5]; + // largest FP32 bit pattern of the widened window (as unsigned): the widening keeps the order of the non-negative + // FP16 values and maps -0, negatives (sign bit) and NaN / inf above every finite non-negative value, so + // "below 0x3F7D8000 (= (float)h(0x3bec), 0.990234375)" is the FP16 test "bits below 0x3bec" for every window + // (the 16-bit form zero-extended every d16 load with a v_cvt_u32_u16) + uint Lmax = 0u; + // HIP KO_WIN_FAST: window inside the image (no index clamped; every lane but those at the borders): one per-lane + // byte offset, the 25 texels at uniform row offsets + 2 c bytes. (u32)(n - 1) <= size - 5 is 0 <= n - 1 and + // n + 3 <= size - 1. (Five 12-byte row loads + shifts instead of the 25 two-byte loads measured slower.) + [branch] if ((uint)(axx.n - 1) <= (uint)(LW - 5) && (uint)(axy.n - 1) <= (uint)(LH - 5)) { + const uint voff = ((uint)(axy.n - 1) * (uint)LW + (uint)(axx.n - 1)) * 2u; + [unroll] for (int r = 0; r < 5; ++r) + [unroll] for (int c = 0; c < 5; ++c) { + Lw[r][c] = wg_h2f(wg_ld16(p_L + (uint)(r * LW) * 2u + 2u * (uint)c, voff)); + Lmax = max(Lmax, asuint(Lw[r][c])); + } + } else { + [unroll] for (int r = 0; r < 5; ++r) { + const uint rb = (uint)ko_clampi(axy.n - 1 + r, LH) * (uint)LW; + [unroll] for (int c = 0; c < 5; ++c) { + Lw[r][c] = wg_h2f(wg_ld16(p_L, (rb + (uint)ko_clampi(axx.n - 1 + c, LW)) * 2u)); + Lmax = max(Lmax, asuint(Lw[r][c])); + } + } + } + + // 3.2 blend logit + const float t = 0.5f * (float)nv9; + const float t2 = t * t; + const float g = ko_div(t * (t2 + 27.0f), mad(t2, 9.0f, 27.0f)); + const float alpha = t < -3.0f ? 0.0f : (t > 3.0f ? 1.0f : mad(g, 0.5f, 0.5f)); + + // Every tap in [0, 0.99) (FP16 bits below 0x3bec: +0 .. 0.98974609375, the usual window): no soft limit, and + // ko_sat of a row cubic's taps is the identity, so the row cubics run without the clamps (ko_cubic_unit, the same + // operations otherwise). Any other window (a tap >= 0.99, NaN, inf, negative or -0): the soft limit on every tap + // (ko_ell is the identity below 0.99, so applying it to such a window equals HIP's needs_soft test) and the + // clamped cubics. + float ellv[4]; + [branch] if (Lmax < 0x3F7D8000u) { + ko_ellv(Lw, axx, axy, true, ellv); + } else { + [unroll] for (int r = 0; r < 5; ++r) + [unroll] for (int c = 0; c < 5; ++c) Lw[r][c] = ko_ell(kh_exact(Lw[r][c])); + ko_ellv(Lw, axx, axy, false, ellv); + } + + // 3.8 / 3.9 history luma + float Hl[4]; + if (hdr) { + bool near = true; + [unroll] for (int s = 0; s < 4; ++s) near = near && ellv[s] >= 0.0f && ellv[s] < kKioDecodeNear; + if (near) { + float lin[4]; + [unroll] for (int s = 0; s < 4; ++s) { + lin[s] = rho * kof(ellv[s] * ko_rcp_output_bounded(1.0f - ellv[s])); + near = near && lin[s] >= 0.0f && lin[s] < kKioEncodeNear; + } + if (near) + [unroll] for (int s = 0; s < 4; ++s) Hl[s] = lin[s] * ko_rcp_output_bounded(lin[s] + 1.0f); + } + [branch] if (!near) { + [loop] for (int s = 0; s < 4; ++s) { + const float lin = rho * kof(ellv[s] * kio_decode(ellv[s])); + Hl[s] = lin * kio_encode(lin); + } + } + } else { + [unroll] for (int s = 0; s < 4; ++s) Hl[s] = rho * ellv[s]; + } + // 3.10 confidence + const float chk = mad(alpha, Hh.z, mad(Hh.x, Hh.y, mad(Hl[0], Hl[1], Hl[2] * Hl[3]))); + valid = valid && kf_finite(chk); + const float alphap = valid ? alpha : 1.0f, bp = valid ? Hh.w : 1.0f; + if (!valid) { + [unroll] for (int s = 0; s < 4; ++s) Hl[s] = 0.0f; + Hh = float4(0.0f, 0.0f, 0.0f, 0.0f); + } + const float m0 = mad(alphap, 1.0f - bp, bp); + const float m1 = mad(bp, 1.0f - alphap * alphap, alphap * alphap); + + // F2 (Ybar of the group, stored above) + GroupMemoryBarrierWithGroupSync(); + const int g0 = lane & ~65; + const half c08 = kh_bits(0x3a66u); // (h16)0.8f + const bool F2 = kh_bits(lds_ybar[g0]) > c08 || kh_bits(lds_ybar[g0 + 1]) > c08 || + kh_bits(lds_ybar[g0 + 64]) > c08 || kh_bits(lds_ybar[g0 + 65]) > c08; + + // 5.1 table index + int qs[2]; + [unroll] for (int c = 0; c < 2; ++c) { + const half z = ko_selector_normalize(c == 0 ? nv0 : nv1); + const int qv = (int)(float)round(kh_mul(z, (half)8)) + 8; + qs[c] = ki_max(ki_min(qv, 15), 0); + } + const bool F0 = (float)kh_bits(0x251fu) > m0; // (float)(h16)0.02f + const bool F1 = m0 < 0.25f && alpha < 0.25f; + const int tix = (16 * qs[0] + qs[1]) + 256 * ((int)F0 + 2 * (int)F1 + 4 * (int)F2); + const uint4 ent = wg_ld128(p_table, (uint)tix * 16u); + const half tA = kh_bits(ent.x & 0xffffu), tC = kh_bits(ent.x >> 16), tB = kh_bits(ent.y & 0xffffu); + const half tS = kh_bits(ent.y >> 16), tK = kh_bits(ent.z >> 16), tk6 = kh_bits(ent.w & 0xffffu); + const bool O = (ent.z & 0xffffu) != 0u; + + // 5.2 line coordinates + half Xs[2], Ys[2]; + [unroll] for (int s = 0; s < 2; ++s) { + Xs[s] = kh_round(mad((float)(2 * x + s) + 0.5f, u_rho2x, u_jmx) - o_x); + Ys[s] = kh_round(mad((float)(2 * y + s) + 0.5f, u_rho2y, u_jmy) - o_y); + } + const half uP = O ? Uy : Ux, vP = O ? Ux : Uy; + + // 5.3 taps + const half one = (half)1; + const half v0 = floor(vP) - one; + half cl = uP + kh_mul(v0 - vP, tS); + half lines[4], cols[4]; + half T[12]; + const float o_ln = O ? o_x : o_y, o_cl = O ? o_y : o_x; + const int n_ln = O ? f_rw : f_rh, n_cl = O ? f_rh : f_rw; + int as_[4]; + [unroll] for (int l = 0; l < 4; ++l) { + if (l > 0) cl = cl + tS; + lines[l] = v0 + (half)l; + cols[l] = round(cl) - one; + as_[l] = ko_clampi((int)(o_ln + (float)lines[l]), n_ln); + } + // HIP KO_TAP_FAST: where no column index of the lane is clamped (|cols| <= 2045: the FP16 sums cols + tc are exact + // integers; c0 = (int)(o_cl + cols) in [0, n_cl - 3]: the FP32 sums o_cl + cols + tc are exact too, |o_cl| < 2^15), + // tap tc is column c0 + tc unclamped: one address per line, the taps +0, +1, +2 texels along the column direction + // (one row of the buffer apart for O). NaN / inf columns fail the first test, a NaN origin the second. + bool tfast = abs(o_cl) < 32768.0f; + int cb[4]; + [unroll] for (int l3 = 0; l3 < 4; ++l3) { + cb[l3] = (int)(o_cl + (float)cols[l3]); + tfast = tfast && abs((float)cols[l3]) <= 2045.0f && cb[l3] >= 0 && cb[l3] <= n_cl - 3; + } + // the two forms give the 12 tap byte offsets; the loads follow the branch (loaded inside both, the T values met + // in one 16-bit join that the driver resolved with a zero extension per tap and register swaps) + uint toff[12]; + [branch] if (tfast) { + const uint st = O ? (uint)f_rw * 2u : 2u; + [unroll] for (int l2 = 0; l2 < 4; ++l2) { + const uint b = ((uint)(O ? cb[l2] : as_[l2]) * (uint)f_rw + (uint)(O ? as_[l2] : cb[l2])) * 2u; + [unroll] for (int tc = 0; tc < 3; ++tc) toff[3 * l2 + tc] = b + (uint)tc * st; + } + } else { + [unroll] for (int l2 = 0; l2 < 4; ++l2) + [unroll] for (int tc = 0; tc < 3; ++tc) { + const half colv = cols[l2] + (half)tc; + const int c = ko_clampi((int)(o_cl + (float)colv), n_cl); + const uint row = (uint)(O ? c : as_[l2]), col = (uint)(O ? as_[l2] : c); + toff[3 * l2 + tc] = (row * (uint)f_rw + col) * 2u; + } + } + [unroll] for (int n2 = 0; n2 < 12; ++n2) T[n2] = kh_bits(wg_ld16(p_ctapT, toff[n2])); + + // 5.4 - 5.6 filtered values, 6.1 blend weights (sub-samples (i, 0), (i, 1) as the halves of half2 values) + const half kux = kh_bits(u_kuxkvy & 0xffffu), kvy = kh_bits(u_kuxkvy >> 16); + const half k_u = O ? kvy : kux, k_v = O ? kux : kvy; + half Fv[4], gam[4]; + const uint xs2 = kp_pack(Xs[0], Xs[1]); + const uint m12 = 0xca00ca00u; // (h16)-12 twice + bool plain = true; + uint us2[2], vs2[2], gam2[2]; + [unroll] for (int i2 = 0; i2 < 2; ++i2) { + const uint ys2 = kp_splat(Ys[i2]); + const uint us = O ? ys2 : xs2, vs = O ? xs2 : ys2; + us2[i2] = us; + vs2[i2] = vs; + uint E[12]; + ko_each_E(us, vs, lines, cols, tA, tB, tC, E); + // tap 0 starts the sums (peeled: DXC kept the loop with an n == 0 test rolled, E / T then lived in indexed + // register arrays) + uint gg = kh2_u(exp2(kh2_h(E[0]))); // ko_exp2h2: two v_exp_f16 + uint sgT = kp_mul_s(gg, T[0]), sg = gg, maxE = E[0]; + [unroll] for (int n = 1; n < 12; ++n) { + gg = kh2_u(exp2(kh2_h(E[n]))); + sgT = kp_fma_s(gg, T[n], sgT); + sg = kp_add(gg, sg); + maxE = kp_max(maxE, E[n]); + } + const uint sgTo = kp_opq(sgT), sgo = kp_opq(sg); // (converted to FP32 by kh_div) + Fv[2 * i2] = kh_div(kp_lo(sgTo), kp_lo(sgo)); + Fv[2 * i2 + 1] = kh_div(kp_hi(sgTo), kp_hi(sgo)); + plain = plain && kp_lo(maxE) >= kp_lo(m12) && kp_hi(maxE) >= kp_hi(m12) && kp_lo(sg) == kp_lo(sg) && + kp_hi(sg) == kp_hi(sg); + const uint dvf = kp_sub(kp_rni(vs), vs); + const uint up = kp_fma_s(dvf, tS, us); + const uint duf = kp_sub(kp_rni(up), up); + const half c45 = kh_bits(0x4480u); // (h16)4.5 + const uint wu = kp_sat_nz(kp_fma_ss(kp_abs(duf), k_u, c45)); + const uint wv = kp_sat_nz(kp_fma_ss(kp_abs(dvf), k_v, c45)); + gam2[i2] = kp_mul(kp_fma_ss(wv, tK, tk6), wu); + gam[2 * i2] = kp_lo(gam2[i2]); + gam[2 * i2 + 1] = kp_hi(gam2[i2]); + } + [branch] if (!plain) { + [unroll] for (int i3 = 0; i3 < 2; ++i3) { + uint E[12]; + // the opaque inputs keep these E from being merged with the main loop's (whose values would then stay + // live up to here) + ko_each_E(us2[i3] ^ (ko_z * 0x10001u), vs2[i3] ^ (ko_z * 0x10001u), lines, cols, tA, tB, tC, E); + // HIP's kh2_gtmask selection per half: keep the best where h(E[n] - best) is negative and not NaN (FP16 bits + // in [0x8000, 0xfc00]); the T taps as half values (no splatted words: a 32-bit use of T here made the + // driver unpack the hot path's T registers) + uint bestE = E[0]; + half bTl = T[0], bTh = T[0]; + [unroll] for (int n = 1; n < 12; ++n) { + const uint d = kp_sub(E[n], bestE); + const bool kl = (d & 0xffffu) - 0x8000u <= 0x7c00u, kh = (d >> 16) - 0x8000u <= 0x7c00u; + bestE = (kl ? bestE & 0xffffu : E[n] & 0xffffu) | (kh ? bestE & 0xffff0000u : E[n] & 0xffff0000u); + bTl = kl ? bTl : T[n]; + bTh = kh ? bTh : T[n]; + } + if (kp_lo(bestE) < kp_lo(m12)) Fv[2 * i3] = bTl; + if (kp_hi(bestE) < kp_hi(m12)) Fv[2 * i3 + 1] = bTh; + } + } + const half gbar = kh_mul((half)0.25, (gam[0] + gam[2]) + (gam[1] + gam[3])); + const half mh = kh_round(m0); + const half MU = kh_bits(0x27aeu), NU = kh_bits(0x3c20u); // (h16)0.03f, (h16)(1.0 / 0.97) + const half rr = kh_bits(0x2e66u) + kh_mul(kh_bits(0x3b33u), mh); // 0.1 + 0.9 mh + const half d0 = kh_mul(mh - MU, NU); + const half d1 = kh_mul(rr - MU, NU); + const half beta_c = koh(kh_sat(mad(gbar, rr - d1, d1))); // (converted to FP32 in 6.5) + + // 6.2 guard + half Ybar = cbar0, Cobar = cbar1, Cgbar = cbar2; + const float guard = kof(kof(kof((float)Ybar * (float)Fv[0]) * (float)Fv[1]) * (float)Fv[2]) * (float)Fv[3]; + if (!kf_finite(guard)) { + [unroll] for (int s = 0; s < 4; ++s) Fv[s] = (half)0; + Ybar = (half)0; + Cobar = (half)0; + Cgbar = (half)0; + } + + // 6.3 / 6.4 new luma, luma history out + uint beta2[2]; + beta2[0] = kp_sat_nz(kp_fma_ss(gam2[0], mh - d0, d0)); + beta2[1] = kp_sat_nz(kp_fma_ss(gam2[1], mh - d0, d0)); + beta2[0] = kp_opq(beta2[0]); // (converted to FP32 below) + beta2[1] = kp_opq(beta2[1]); + float Nn[4]; + uint Lo[4]; + [unroll] for (int s2 = 0; s2 < 4; ++s2) { + const half beta = (s2 & 1) ? kp_hi(beta2[s2 >> 1]) : kp_lo(beta2[s2 >> 1]); + Nn[s2] = mad((float)beta, (float)Fv[s2] - Hl[s2], Hl[s2]); + const float c99 = kf_bits(0x3F7D70A4u); + const float lv = Nn[s2] < c99 ? Nn[s2] : mad(Nn[s2] - c99, 1000.0f, c99); + Lo[s2] = ko_f2h(lv); + } + if (active) { + [unroll] for (int r2 = 0; r2 < 2; ++r2) + wg_st32(p_Lout, ((uint)(2 * y + r2) * (uint)LW + (uint)(2 * x)) * 2u, Lo[2 * r2] | (Lo[2 * r2 + 1] << 16)); + } + + // 6.5 colour + const float bc = (float)beta_c; + const float Yc = mad(Hh.x, 1.0f - bc, bc * (float)Ybar); + const float Coc = mad(Hh.y, 1.0f - bc, bc * (float)Cobar); + const float Cgc = mad(Hh.z, 1.0f - bc, bc * (float)Cgbar); + const float eps = kf_bits(0x3727C5ACu); + const float q = ko_rcp_output_bounded(max(Yc, eps)); + const float rCo = kof(Coc * q), rCg = kof(Cgc * q); + const float Yo = 0.25f * (Nn[3] + (Nn[2] + (Nn[0] + Nn[1]))); + const float Coo = kof(rCo * Yo), Cgo = kof(rCg * Yo); + + // 6.6 colour history out + const float Yp = max(Yo, 0.0f); + const float s1 = (Cgo - eps < -Yp) ? eps - Cgo : Yp; + const float s2 = ((Coo - Cgo) - eps < -s1) ? (Cgo - Coo) + eps : s1; + const float s3 = ((-Coo - Cgo) - eps < -s2) ? (Coo + Cgo) + eps : s2; + const float kk = s3 <= Yp ? 1.0f : ko_div(Yp, s3); + const float age = min(max(ko_div(m1, m1 + 1.0f), 0.01f), 1.0f); // kf_sat of a value >= 0.01 (maxNum: never NaN) + if (!active) + return; + wg_st64(p_Hcout, ((uint)y * (uint)W16 + (uint)x) * 8u, + uint2(ko_f2h(Yp) | (ko_f2h(Coo * kk) << 16), ko_f2h(Cgo * kk) | (ko_f2h(age) << 16))); + + // 6.7 game output + if (x < f_W && y < f_H) { + float Yg = Yo, Cog = Coo, Cgg = Cgo; + if (hdr) { + const float d = Yo < kKioDecodeNear ? ko_rcp_refined(1.0f - Yo) : kio_decode(Yo); + const float kinv = ko_kinv(); + Yg = kinv * kof(Yo * d); + Cog = kinv * kof(Coo * d); + Cgg = kinv * kof(Cgo * d); + } + const float R = (Yg + Cog) - Cgg, G = Cgg + mad(Cog, kof(0.0f), Yg), B = (Yg - Cog) - Cgg; +#if defined(WG_DIRECT_TEXTURES) + ko_store_out(x, y, R, G, B, cbar3); +#else + if (p_omode == KIO_OUT_RGBA16F) { + wg_st64(p_out, ((uint)y * (uint)p_opitch + (uint)x) * 8u, uint2(ko_f2h(R) | (ko_f2h(G) << 16), ko_f2h(B) | (kh_u(cbar3) << 16))); + } else if (p_omode == KIO_OUT_RGB32F) { + const uint o = ((uint)y * (uint)p_opitch + (uint)x) * 12u; + wg_stf(p_out, o, R); + wg_stf(p_out, o + 4u, G); + wg_stf(p_out, o + 8u, B); + } else { + wg_st128(p_out, ((uint)y * (uint)p_opitch + (uint)x) * 16u, asuint(float4(R, G, B, (float)cbar3))); + } +#endif + } +} diff --git a/clean/k/tests/test_k_direct_motion.py b/clean/k/tests/test_k_direct_motion.py index ef91c20c..4268707a 100644 --- a/clean/k/tests/test_k_direct_motion.py +++ b/clean/k/tests/test_k_direct_motion.py @@ -146,13 +146,14 @@ def test_full_head_direct_port_and_alpha(self): self.skipTest("the D3D12 input and output ports are not in this tree") out_k = read("dx12", "kernels", "k_out.hlsl") line = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_direct ")] - self.assertEqual(len(line), 1) - for need in ("entry=k_out_pixel_c", "rs=direct", "KO_COMPACT=0", "WG_DIRECT_TEXTURES=1"): - self.assertIn(need, line[0]) - c = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_c_direct ")][0] - self.assertEqual(c.replace("k_out_pixel_c_direct", "X").replace("KO_COMPACT=1", "C"), - line[0].replace("k_out_pixel_direct", "X").replace("KO_COMPACT=0", "C")) - self.assertIn('"k_out_pixel_direct"', read("equiv", "k_stages.py")) + if line: # the direct forms are declared with the runtime wiring; their declarations are checked once they are + self.assertEqual(len(line), 1) + for need in ("entry=k_out_pixel_c", "rs=direct", "KO_COMPACT=0", "WG_DIRECT_TEXTURES=1"): + self.assertIn(need, line[0]) + c = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_c_direct ")][0] + self.assertEqual(c.replace("k_out_pixel_c_direct", "X").replace("KO_COMPACT=1", "C"), + line[0].replace("k_out_pixel_direct", "X").replace("KO_COMPACT=0", "C")) + self.assertIn('"k_out_pixel_direct"', read("equiv", "k_stages.py")) # the output alpha is cbar3 (the colour input's alpha, issue #26) in both head forms: no constant alpha self.assertNotIn("KO_OUT_ALPHA0", out_k) self.assertIn("native_direct_output[uint2((uint)x, (uint)y)] = float4(v, (float)A);", out_k)