From 69f43157d5b4938968f14195a1ea0fead200d32d Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 17:03:28 -0500 Subject: [PATCH] Model K on D3D12: the exact tiled input stage (k_io_exact.co), byte-identical to HIP Refs #15. Port of kx/k_in_tiled (KInParams; the exact path: correctly rounded log2, 2^x, divisions and reciprocals, FP16 rounding to nearest even; binary64 fma and division from the D3D12 extended doubles, marked fp64=ext so an adapter without them gets no port). Shared headers k_fp.hlsli and k_frame.hlsli; the FP constants and tables are generated at every build by gen_k_fp_tables.py (nothing stored). The forms with KiHost arguments and the forms reading the game's textures are in the source but not declared yet. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 8 +- clean/dx12/kernel_sets.json | 4 + clean/k/dx12/README.md | 6 +- clean/k/dx12/gen_k_fp_tables.py | 108 +++++++++ clean/k/dx12/hlsl/k_fp.hlsli | 393 ++++++++++++++++++++++++++++++++ clean/k/dx12/hlsl/k_frame.hlsli | 290 +++++++++++++++++++++++ clean/k/dx12/k_plan_bridge.py | 8 +- clean/k/dx12/k_port_check.py | 99 +++++++- clean/k/dx12/kernels/k_in.hlsl | 310 +++++++++++++++++++++++++ 9 files changed, 1217 insertions(+), 9 deletions(-) create mode 100644 clean/k/dx12/gen_k_fp_tables.py create mode 100644 clean/k/dx12/hlsl/k_fp.hlsli create mode 100644 clean/k/dx12/hlsl/k_frame.hlsli create mode 100644 clean/k/dx12/kernels/k_in.hlsl diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 05dd92fa..0ac9e2aa 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -383,6 +383,12 @@ def main(argv=None): for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: if fn.startswith("fused_") and fn.endswith("_gen.py"): subprocess.run([sys.executable, os.path.join(KERNEL_DIR, fn)], check=True) + # Model K's FP constants and tables (clean/k/dx12/gen_k_fp_tables.py: computed from their series, not stored in the + # repository) into /dx12/gen, the K ports' second include folder; only when a K kernel can be built + k_gen = os.path.join(dst, "gen") + if (a.model_k or only) and os.path.exists(os.path.join(K_DX12_DIR, "gen_k_fp_tables.py")): + subprocess.run([sys.executable, os.path.join(K_DX12_DIR, "gen_k_fp_tables.py"), + os.path.join(k_gen, "k_fp_tables.hlsli")], check=True) jobs = [] # (source, declaration, output path) in declaration order (sorted file name, then line) for src, k_source in source_files(a.model_k or bool(only)): for d in kernel_decls(src): @@ -410,7 +416,7 @@ def build_one(job): if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] info = {} - compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR,) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm, + compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR, k_gen) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm, keep_listing=out[:-5] + ".ll" if a.listing else None, info=info) with open(out, "rb") as f: sha = hashlib.sha256(f.read()).hexdigest() diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 5af75100..a8bdcb5e 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -367,6 +367,10 @@ [ "k_warm", "k_block_wide.co" + ], + [ + "kx/k_in_tiled", + "k_io_exact.co" ] ] } diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 63978019..480d2c73 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -30,7 +30,11 @@ and starts filled with its run's own sentinel (HIP `0xA5`, D3D12 `0x5A`); the de slack must keep its sentinel; special values are among the inputs; a negative control (the port given another input than HIP) must be caught; Model M's `scale_rgb` runs through the same replay as a control. Output: counts and verdicts. -Families: `narrow` (the narrow network stages of `k_block.co`: `k_stage_e2`, `k_stage_e0`, `k_stage_e1`, `k_stage_d2`, +Families: `input` (the exact tiled input stage `kx/k_in_tiled` of `k_io_exact.co`: 16 flag sets of HDR, low-resolution +motion, inverted depth and reset, odd and tiny base grids down to 1 x 1, the exposure scalars from a device buffer as the +runtime passes them, infinity, huge values, zero and -0 in the colour, depth, motion and histories, NaN in a third of +the cases; the raw bytes are identical in every case; the port needs the adapter's binary64 fma and division (manifest +`fp64: ext`, dropped on an adapter without them), and flushes FP32 denormals); `narrow` (the narrow network stages of `k_block.co`: `k_stage_e2`, `k_stage_e0`, `k_stage_e1`, `k_stage_d2`, `k_stage_d1`, `k_stage2_e0`, `k_stage2_e1`, `k_stage2_d1`, `k_stage2_d0_hp`, `k_stage2_d0_hidden`) on random stage weights and inputs, the stage runner's launch plans run on HIP and replayed on D3D12 (`k_plan_bridge.py`); the inputs of every third case carry special values (infinity, +-65504, zero, -0, subnormals) and of every third case NaN as well; the diff --git a/clean/k/dx12/gen_k_fp_tables.py b/clean/k/dx12/gen_k_fp_tables.py new file mode 100644 index 00000000..c02b76d3 --- /dev/null +++ b/clean/k/dx12/gen_k_fp_tables.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: MIT +"""Generates k_fp_tables.hlsli: the constants of model K's exact FP32 / FP64 helpers (k_fp.h) as bit +patterns for the D3D12 port (k_fp.hlsli). WrongGPU's own code. + +k_fp.h builds its tables at compile time (constexpr binary64 series KfExpTab / KfLogTab) and folds constants such as +1.0 / 23.0 or (float)(1.0 / 3.0). HLSL has no constexpr evaluation in binary64 and its float literals are parsed by +DXC, so the port takes every such value as bits computed here with the same IEEE operations in the same order (Python +floats are binary64 with round to nearest even; numpy.float32 rounds binary64 to binary32 to nearest even, as the C +casts do). clean/dx12/build_dx12.py runs it into /dx12/gen (an include folder of the K ports), so the tables are +computed at every build and never stored in the repository: python clean/k/dx12/gen_k_fp_tables.py +""" +import os +import struct +import sys + +import numpy as np + + + +def f32(v): + return struct.unpack("> 32 + + +def kc_exp(x): # k_fp.h kc_exp, |x| <= 0.4 + t, s = 1.0, 1.0 + for k in range(1, 30): + t = t * x / k + s = s + t + return s + + +def kc_log(x): # k_fp.h kc_log, x in [0.5, 2] + s = (x - 1.0) / (x + 1.0) + s2 = s * s + t, r = s, 0.0 + for k in range(40): + r = r + t / (2 * k + 1) + t = t * s2 + return 2.0 * r + + +KC_LN2 = 0.69314718055994530942 + + +def main(out_path): + exp_h, exp_l = [], [] + for i in range(65): # KfExpTab: 2^(j/64), j = -32 .. 32 + v = kc_exp((i - 32) / 64.0 * KC_LN2) + h = float(np.float32(v)) + exp_h.append(f32(h)) + exp_l.append(f32(v - h)) + log_inv, log_h, log_l = [], [], [] + for i in range(97): # KfLogTab: c_j = 1 + j/128, inv_j = RN32(1/c_j), -log2(inv_j) as hi + lo + c = 1.0 + (i - 32) / 128.0 + inv = float(np.float32(1.0 / c)) + log_inv.append(f32(inv)) + v = -kc_log(inv) / KC_LN2 + h = float(np.float32(v)) + log_h.append(f32(h)) + log_l.append(f32(v - h)) + + def arr(name, vals): + rows = [", ".join(f"0x{v:08X}u" for v in vals[i:i + 8]) for i in range(0, len(vals), 8)] + return f"static const uint {name}[{len(vals)}] = {{\n " + ",\n ".join(rows) + "\n};\n" + + def dconst(name, v): + lo, hi = f64(v) + return f"#define {name} asdouble(0x{lo:08X}u, 0x{hi:08X}u) // {v!r}\n" + + def fconst(name, v): + return f"#define {name} asfloat(0x{f32(v):08X}u) // {float(np.float32(v))!r}\n" + + third = 1.0 / 3.0 + third_h = float(np.float32(third)) + inv_ln2 = 1.4426950408889634074 + ln2 = 0.69314718055994530942 + out = ["// Generated by clean/k/dx12/gen_k_fp_tables.py at build time: the constants of k_fp.h as bits. WrongGPU's own code.\n", + "#pragma once\n\n", + "// KfExpTab: 2^(j/64), j = -32 .. 32, hi + lo\n", arr("KF_EXP_H", exp_h), arr("KF_EXP_L", exp_l), + "// KfLogTab: inv_j = RN32(1 / (1 + j/128)), -log2(inv_j) as hi + lo, j = -32 .. 64\n", + arr("KF_LOG_INV", log_inv), arr("KF_LOG_H", log_h), arr("KF_LOG_L", log_l), "\n", + "// binary64 constants of kd_log_core / kd_log / kd_log2 / kd_exp2 (k_fp.h; folded there at compile time)\n", + dconst("KD_LN2", 0.6931471805599453094), dconst("KD_INV_LN2", 1.4426950408889634074), + dconst("KD_SQRT2", 1.4142135623730951), dconst("KD_INV23", 1.0 / 23.0)] + out.append("// 1 / (2k + 1), k = 0 .. 10 (kd_log_core's series)\n") + out.append("static const uint2 KD_INV_ODD[11] = {\n " + ",\n ".join( + "uint2(0x%08Xu, 0x%08Xu)" % f64(1.0 / (2 * k + 1)) for k in range(11)) + "\n};\n") + out.append("// 1 / k, k = 0 .. 16 (kd_exp2's series; entry 0 unused)\n") + out.append("static const uint2 KD_INV_K[17] = {\n uint2(0u, 0u),\n " + ",\n ".join( + "uint2(0x%08Xu, 0x%08Xu)" % f64(1.0 / k) for k in range(1, 17)) + "\n};\n\n") + out.append("// binary32 constants of kf2_log2 / kf2_exp2 / kf_ln (k_fp.h)\n") + out += [fconst("KF_THIRD_H", third), fconst("KF_THIRD_L", third - third_h), + fconst("KF_SIXTH", np.float32(1.0) / np.float32(6.0)), fconst("KF_1_120", np.float32(1.0) / np.float32(120.0)), + fconst("KF_1_24", np.float32(1.0) / np.float32(24.0)), fconst("KF_1_64", np.float32(1.0) / np.float32(64.0)), + fconst("KF2_INV_LN2_H", inv_ln2), fconst("KF2_INV_LN2_L", inv_ln2 - float(np.float32(inv_ln2))), + fconst("KF2_LN2_H", ln2), fconst("KF2_LN2_L", ln2 - float(np.float32(ln2)))] + os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) + with open(out_path, "w", newline="\n") as f: + f.write("".join(out)) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/clean/k/dx12/hlsl/k_fp.hlsli b/clean/k/dx12/hlsl/k_fp.hlsli new file mode 100644 index 00000000..6cbbcb97 --- /dev/null +++ b/clean/k/dx12/hlsl/k_fp.hlsli @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's exact scalar FP32 / FP16 / FP64 helpers (clean/k/kernels/k_fp.h; specs 42, 43). WrongGPU's own +// code. Same names and the same bits as the HIP helpers on gfx1201, in the exact arithmetic mode of build_dx12.py +// (clean/dx12/hlsl/wg_math.hlsli: one IEEE operation per written operation, mad() = one v_fma_f32, rcp() = v_rcp_f32, +// no float '/': wg_div is HIP's correctly rounded a / b). +// +// Include after wg_platform.hlsli (wg_math.hlsli) and, in a kernel, call KF_SETUP() right after WG_SETUP(). +// +// Representation: an FP16 value (HIP h16) is a float holding that value exactly (f16tof32 / wg_h2f of the bits); +// kh_round(x) is HIP's (h16) rounding of an FP32 value (round to nearest even, v_cvt_f16_f32: wg_f2h; the driver's +// (half) / f32tof16 round toward zero and are never used), kh_bits(x) the bits of such a value for a store. FP16 +// arithmetic is done in FP32 and rounded once: a sum, difference or product of two FP16 values rounded to FP32 and then +// to FP16 is the FP16 operation's result (products are exact in FP32; for + and - double rounding is innocuous since +// 24 >= 2 * 11 + 2, Figueroa 1995), kh_fma is wg_fma_f16 (one rounding), kh_div as k_fp.h's argument. This also avoids +// the driver's fusion of half multiply + add (wg_math.hlsli "driver exceptions"). +// +// Binary64 (kd_*): add / sub / mul / compare / convert and fma (the D3D11.1 extended doubles, which D3D12 includes in +// D3D12_FEATURE_DATA_D3D12_OPTIONS.DoublePrecisionFloatShaderOps): build_dx12.py marks ports with binary64 fma or +// division "fp64": "ext" and the devices refuse them on an adapter without doubles. HLSL has no binary64 sqrt / rint / +// constexpr: the constants and tables of k_fp.h come as bits +// from k_fp_tables.hlsli (clean/k/dx12/gen_k_fp_tables.py: the same binary64 series and roundings), kd_exp2's rint is +// done in FP32 (its argument is always a binary32 value, see there), and kf_rsqrt's binary64 fallback is replaced by an +// exact power-of-four scaling (see there). +// +// Opaque barriers (kf_opq): the gfx1201 D3D12 shader compiler may contract a product into a following addition or +// subtraction (one fma: one rounding instead of HIP's two; seen in the first port check of the input stage, about one +// FP16 value in 10^5 differing by one ulp), regroup chains of multiplications by constant / uniform / shared factors, +// and fold a product into a later fused operation (wg_math.hlsli). Wherever HIP rounds a product that then meets an +// addition (and the product is not exact, e.g. a power-of-two scaling), the product goes through kf_opq (an XOR with a +// zero the compiler cannot prove: wg_grid_ >> 31, the launch's workgroup count is below 2^31; one VALU operation). +// kf_exp2 applies it to its argument itself (its callers pass products). +// +// Differences from HIP (as for every D3D12 port, wg_math.hlsli): FP32 denormals are flushed to zero here (HIP keeps +// them); none of the input stage's values is an FP32 denormal for the frames the port checks run (clean/k/dx12/ +// k_port_check.py), and FP16 denormals are kept as in HIP. +#pragma once + +#include "k_fp_tables.hlsli" + +static uint kf_opq_z_; // 0, opaque to the compiler (KF_SETUP) +#define KF_SETUP() (kf_opq_z_ = wg_grid_ >> 31) +float kf_opq(float x) { return asfloat(asuint(x) ^ kf_opq_z_); } +float2 kf_opq2(float2 x) { return asfloat(asuint(x) ^ kf_opq_z_); } +float4 kf_opq4(float4 x) { return asfloat(asuint(x) ^ kf_opq_z_); } + +// ---- elementary steps ----------------------------------------------------------------------------------------------- + +float kf_fma(float a, float b, float c) { return mad(a, b, c); } // v_fma_f32 +float kf_bits(uint b) { return asfloat(b); } +float kf_floor(float x) { return floor(x); } +float kf_trunc(float x) { return trunc(x); } +float kf_rni(float x) { return round(x); } // v_rndne_f32 (ties to even) +// max / min of specs 42 / 43: IEEE maxNum / minNum (v_max_num_f32 / v_min_num_f32), as __builtin_fmaxf / fminf +float kf_max(float a, float b) { return max(a, b); } +float kf_min(float a, float b) { return min(a, b); } +int ki_min(int a, int b) { return a < b ? a : b; } +int ki_max(int a, int b) { return a > b ? a : b; } +// HIP's x - x == 0 test as bits: the D3D12 shader compiler folds x - x to 0 (seen for float and double: an infinite +// input passed as finite in the first probe, clean/k/dx12/probe/kfp_check.py) +bool kf_finite(float x) { return (asuint(x) & 0x7f800000u) != 0x7f800000u; } +bool kd_isnan(double x) { uint lo, hi; asuint(x, lo, hi); return (hi & 0x7ff00000u) == 0x7ff00000u && ((hi & 0xfffffu) | lo) != 0u; } +bool kd_finite(double x) { uint lo, hi; asuint(x, lo, hi); return (hi & 0x7ff00000u) != 0x7ff00000u; } + +// wg_div (HIP's correctly rounded a / b) with its common case first: both operands normal and finite and the quotient's +// exponent in range, where wg_div's result is the scaled quotient q with the exponent difference added to its exponent +// field (the same q: the same operations as wg_div); every other operand pair takes wg_div itself in a cold block. Equal +// to wg_div for every input (probe: build\k-d3d12\L2work\probe\kdiv_check.py); about half of wg_div's instructions and none of +// its selects on the common path. +float kf_div(float a, float b) +{ + const uint ua = asuint(a), ub = asuint(b); + const uint ea = (ua >> 23) & 255u, eb = (ub >> 23) & 255u; + const float as = asfloat((ua & 0x807fffffu) | 0x3f800000u), bs = asfloat((ub & 0x807fffffu) | 0x3f800000u); + float y = rcp(bs); + const float e = mad(-bs, y, 1.0f); + y = mad(e, y, y); + float q = as * y; + float r = mad(-bs, q, as); + q = mad(r, y, q); + r = mad(-bs, q, as); + q = mad(r, y, q); + const int d = (int)ea - (int)eb; + const uint en = ((asuint(q) >> 23) & 255u) + (uint)d; + float res; + [branch] if (ea - 1u < 254u && eb - 1u < 254u && en - 1u < 254u) + res = asfloat(asuint(q) + ((uint)d << 23)); + else + res = wg_div(a, b); + return res; +} + +// R(t): round half away from zero (specs 42 / 43), exact for every binary32 t +float kf_R(float t) +{ + const float a = abs(t); + const float fl = floor(a); + const float r = (a - fl) >= 0.5f ? fl + 1.0f : fl; // a - fl is exact + return t < 0.0f ? -r : r; +} +float kf_sat(float x) { return isnan(x) ? 0.0f : (x < 0.0f ? 0.0f : (x > 1.0f ? 1.0f : x)); } + +// ---- FP16 values (floats holding FP16 values exactly) --------------------------------------------------------------- + +uint kh_bits(float h) { return wg_f2h(h); } // exact for an FP16 value (NaN payload as v_cvt_f16_f32) +float kh_round(float x) { return wg_h2f(wg_f2h(x)); } // HIP kh_round: (h16) x, nearest even +uint kh_round_bits(float x) { return wg_f2h(x); } // the bits of kh_round(x) +float kh_add(float a, float b) { return kh_round(a + b); } +float kh_sub(float a, float b) { return kh_round(a - b); } +float kh_mul(float a, float b) { return kh_round(a * b); } +float kh_fma(float a, float b, float c) { return wg_h2f(wg_fma_f16(a, b, c)); } // v_fma_f16: one rounding +// a / b of FP16 values rounded once to FP16: the correctly rounded FP32 quotient rounded to FP16 (k_fp.h kh_div) +float kh_div(float a, float b) { return kh_round(wg_div(a, b)); } +float kh_floor(float x) { return floor(x); } // exact on FP16 values +float kh_rni(float x) { return round(x); } +float kh_sat(float x) { return isnan(x) ? 0.0f : (x < 0.0f ? 0.0f : (x > 1.0f ? 1.0f : x)); } +float kh_maxnum(float a, float b) { return max(a, b); } + +// packed FP16 (two independent values; each element exactly as the scalar helper, as HIP's v_pk_* do) +float2 kh2(float a, float b) { return float2(a, b); } +float2 kh2_splat(float a) { return float2(a, a); } +float2 kh2_add(float2 a, float2 b) { return float2(kh_add(a.x, b.x), kh_add(a.y, b.y)); } +float2 kh2_sub(float2 a, float2 b) { return float2(kh_sub(a.x, b.x), kh_sub(a.y, b.y)); } +float2 kh2_mul(float2 a, float2 b) { return float2(kh_mul(a.x, b.x), kh_mul(a.y, b.y)); } +float2 kh2_fma(float2 a, float2 b, float2 c) { return float2(kh_fma(a.x, b.x, c.x), kh_fma(a.y, b.y, c.y)); } +float2 kh2_abs(float2 a) { return abs(a); } +float2 kh2_rni(float2 a) { return round(a); } +float2 kh2_maxnum(float2 a, float2 b) { return max(a, b); } +// k_fp.h kh2_sat_nz: maxNum(x, 0) then minNum(., 1) per element (callers never pass -0) +float2 kh2_sat_nz(float2 a) { return min(max(a, 0.0f), 1.0f); } +// k_fp.h kh2_gtmask as a bool2: true where h(b - a) is negative and not NaN (IEEE a > b, except a = +0, b = -0: true) +bool2 kh2_gtmask(float2 a, float2 b) +{ + const float2 d = float2(kh_sub(b.x, a.x), kh_sub(b.y, a.y)); + return bool2((asuint(d.x) >> 31) != 0u && !isnan(d.x), (asuint(d.y) >> 31) != 0u && !isnan(d.y)); +} +float2 kh2_bsel(bool2 m, float2 a, float2 b) { return float2(m.x ? a.x : b.x, m.y ? a.y : b.y); } + +// ---- double-precision functions -------------------------------------------------------------------------------------- + +double kd_fma(double a, double b, double c) { return fma(a, b, c); } // extended doubles +double kd_from_bits(uint lo, uint hi) { return asdouble(lo, hi); } + +// 1 / d for d in [1.7, 2.5]: the FP32 reciprocal estimate (v_rcp_f32 of RN32(d), as HIP) and two Newton steps +double kd_rcp_near2(double d) +{ + double q = (double)rcp((float)d); + q = kd_fma(q, kd_fma(-d, q, 1.0L), q); + q = kd_fma(q, kd_fma(-d, q, 1.0L), q); + return q; +} + +// k_fp.h kd_log_core: x = 2^e m, m in (1/sqrt 2, sqrt 2], ln m = 2 s (1 + s^2/3 + ... + s^22/23), s = (m - 1)/(m + 1) +double kd_log_core(double x, out int e_out) +{ + uint lo, hi; + asuint(x, lo, hi); + int e = (int)((hi >> 20) & 0x7ffu) - 1023; + double m = asdouble(lo, (hi & 0x000fffffu) | 0x3ff00000u); // [1, 2) + if (m > KD_SQRT2) { + m = m * 0.5L; + e += 1; + } + const double s = (m - 1.0L) * kd_rcp_near2(m + 1.0L); // m - 1 and m + 1 are exact + const double s2 = s * s; + double p = KD_INV23; + [unroll] for (int k = 10; k >= 0; --k) p = kd_fma(p, s2, asdouble(KD_INV_ODD[k].x, KD_INV_ODD[k].y)); + e_out = e; + return 2.0L * s * p; +} + +double kd_log_special(double x, out bool special) +{ + special = true; + if (kd_isnan(x) || x < 0.0L) return asdouble(0u, 0x7ff80000u); + if (x == 0.0L) return -asdouble(0u, 0x7ff00000u); + if (!kd_finite(x)) return x; // +inf + special = false; + return 0.0L; +} + +// ln x +double kd_log(double x) +{ + bool sp; + const double v = kd_log_special(x, sp); + if (sp) return v; + int e; + const double l = kd_log_core(x, e); + return kd_fma((double)e, KD_LN2, l); +} + +// log2 x = e + ln(m) / ln 2 +double kd_log2(double x) +{ + bool sp; + const double v = kd_log_special(x, sp); + if (sp) return v; + int e; + const double l = kd_log_core(x, e); + return kd_fma(l, KD_INV_LN2, (double)e); +} + +double kd_ldexp(double v, int n) // v * 2^n, n in [-2044, 2046] +{ + [unroll] for (int i = 0; i < 2; ++i) { + const int k = n > 1023 ? 1023 : (n < -1022 ? -1022 : n); + v = v * asdouble(0u, (uint)(k + 1023) << 20); + n -= k; + } + return v; +} + +// 2^y. Precondition: y is a binary32 value (every caller passes (double) of a float: kf_exp2), so rint(y) is the FP32 +// v_rndne of (float) y, which is exact (HLSL has no binary64 rint). +double kd_exp2(double y) +{ + if (kd_isnan(y)) return y; + if (y > 1100.0L) return asdouble(0u, 0x7ff00000u); + if (y < -1100.0L) return 0.0L; + const double n = (double)round((float)y); + const double r = (y - n) * KD_LN2; // |r| <= 0.347 + double p = 1.0L; + [unroll] for (int k = 16; k >= 1; --k) p = kd_fma(p, r * asdouble(KD_INV_K[k].x, KD_INV_K[k].y), 1.0L); + return kd_ldexp(p, (int)n); +} + +// 1/sqrt(x) rounded once to FP32 (spec 43 6.8). HIP: for x >= 1 the hardware estimate v_rsq_f32 and exact rounding +// tests in binary64 (as here); otherwise kd_rsqrt32 = (float)(1.0 / sqrt((double)x)), the correctly rounded value. +// HLSL has no binary64 sqrt: a normal x < 1 is scaled by 4^k into [1, 4) (exact), the tested result scaled back by 2^k +// (exact: the result is normal), which is the correctly rounded value too. Zero (and FP32 denormals, flushed) -> +-inf +// as 1 / +-0, +inf -> 0, NaN -> quieted, negative -> the canonical NaN. Port check: the output stage's (G2). +float kf_rsqrt_ge1(float x) +{ + float y = rsqrt(x); // v_rsq_f32 + const double xd = (double)x; + [unroll] for (int it = 0; it < 3; ++it) { + const uint b = asuint(y); + const double yd = (double)y; + const double ulp = (double)asfloat((b & 0x7F800000u) - (23u << 23)); + const double up = yd + 0.5L * ulp; + const double lo = yd - ((b & 0x7FFFFFu) == 0u ? 0.25L : 0.5L) * ulp; // below a power of two the spacing halves + if (kd_fma(up * up, xd, -1.0L) < 0.0L) { + y = asfloat(b + 1u); + } else if (kd_fma(lo * lo, xd, -1.0L) > 0.0L) { + y = asfloat(b - 1u); + } else { + return y; + } + } + return y; +} +float kf_rsqrt(float x) +{ + if (x >= 1.0f && kf_finite(x)) return kf_rsqrt_ge1(x); + if (isnan(x)) return asfloat(asuint(x) | 0x400000u); + if (x == 0.0f) return asfloat((asuint(x) & 0x80000000u) | 0x7f800000u); + if (x < 0.0f) return asfloat(0x7fc00000u); + if (!kf_finite(x)) return 0.0f; // +inf + const int e = (int)((asuint(x) >> 23) & 255u) - 127; // x normal in [2^-126, 1): e in [-126, -1] + const int k = (1 - e) >> 1; // 4^k x in [1, 4) + const float xs = x * asfloat((uint)(2 * k + 127) << 23); + return kf_rsqrt_ge1(xs) * asfloat((uint)(k + 127) << 23); +} + +// ---- correctly rounded binary32 log2 / ln / 2^x in FP32 double-word arithmetic (k_fp.h: Ziv test, else binary64) ---- + +struct kf2 { float h, l; }; +kf2 kf2_make(float h, float l) { kf2 r; r.h = h; r.l = l; return r; } + +// Error-free transformations: the sums go through kf_opq, else the D3D12 compiler simplifies (a + b) - a to b and the +// like (the first probe found the double-word results off by one ulp in 16 % of 2^x and 0.1 % of log2 inputs) +kf2 kf2_two_sum(float a, float b) +{ + const float s = kf_opq(a + b); + const float bb = kf_opq(s - a); + return kf2_make(s, (a - (s - bb)) + (b - bb)); +} +kf2 kf2_fast_two_sum(float a, float b) // |a| >= |b| or a == 0 +{ + const float s = kf_opq(a + b); + return kf2_make(s, b - (s - a)); +} +kf2 kf2_two_prod(float a, float b) +{ + const float p = kf_opq(a * b); + return kf2_make(p, kf_fma(a, b, -p)); +} +kf2 kf2_add(kf2 a, kf2 b) +{ + kf2 s = kf2_two_sum(a.h, b.h); + s.l = s.l + (a.l + b.l); + return kf2_fast_two_sum(s.h, s.l); +} +kf2 kf2_mul(kf2 a, kf2 b) +{ + kf2 p = kf2_two_prod(a.h, b.h); + p.l = kf_fma(a.h, b.l, kf_fma(a.l, b.h, p.l)); + return kf2_fast_two_sum(p.h, p.l); +} + +// Ziv test: is RN32(v) = r.h for every v with |v - (r.h + r.l)| <= 2^-38 |r.h| ? +bool kf2_round_ok(kf2 r) +{ + const uint b = asuint(r.h); + const uint ex = (b >> 23) & 0xFFu; + if (ex - 40u > 214u) return false; // ex < 40 or 255: tiny, subnormal, inf / NaN: exact path + // half = 0.5 ulp, or 0.25 ulp for a power of two whose low part points toward zero (below it the spacing halves). + // Bit form: (sign(r.l) xor sign(r.h)) and (mantissa - 1 borrows), bit 31. It differs from (r.l < 0) != (r.h < 0) only + // for r.l = -0, where |r.l| + err = err < 2^-37 |r.h| is below either half: the same answer. + const uint s = ((asuint(r.l) ^ b) & ((b & 0x7FFFFFu) - 1u)) >> 31; + const float half = asfloat((ex - 24u - s) << 23); + const float err = abs(r.h) * asfloat(0x2C800000u); // 2^-38 + return abs(r.l) + err < half; +} + +// log2 x as hi + lo for finite normal x > 0 +kf2 kf2_log2(float x) +{ + const uint b = asuint(x); + int e = (int)((b >> 23) & 0xFFu) - 127; + float m = asfloat((b & 0x7FFFFFu) | 0x3F800000u); // [1, 2) + if (m >= 1.5f) { + m = m * 0.5f; + e += 1; + } + const int j = (int)round((m - 1.0f) * 128.0f); // -32 .. 64 + const float inv = asfloat(KF_LOG_INV[j + 32]); + const kf2 p = kf2_two_prod(m, inv); // m inv = 1 + t exactly + const kf2 t = kf2_two_sum(p.h - 1.0f, p.l); // p.h - 1 is exact + // ln(1 + t) = t (1 - t/2 + t^2/3 - t^3 (1/4 - t/5 + t^2/6)), |t| < 0.0053 + const float th = t.h; + const float q = -kf_fma(th, kf_fma(th, KF_SIXTH, -0.2f), 0.25f); + kf2 P = kf2_add(kf2_make(KF_THIRD_H, KF_THIRD_L), kf2_two_prod(th, q)); + P = kf2_add(kf2_make(-0.5f, 0.0f), kf2_mul(t, P)); + P = kf2_add(kf2_make(1.0f, 0.0f), kf2_mul(t, P)); + const kf2 ln1 = kf2_mul(t, P); + kf2 r = kf2_mul(ln1, kf2_make(KF2_INV_LN2_H, KF2_INV_LN2_L)); + r = kf2_add(r, kf2_make(asfloat(KF_LOG_H[j + 32]), asfloat(KF_LOG_L[j + 32]))); + return kf2_add(kf2_make((float)e, 0.0f), r); +} + +// 2^y as hi + lo for y in [-125, 127) +kf2 kf2_exp2(float y) +{ + const float n = round(y); + const float f = y - n; // exact, |f| <= 0.5 + const float jf = round(f * 64.0f); + const float r = f - jf * KF_1_64; // exact, |r| <= 1/128 + const kf2 x = kf2_mul(kf2_make(r, 0.0f), kf2_make(KF2_LN2_H, KF2_LN2_L)); // |x| < 0.0055 + // e^x = 1 + x + x^2/2 + x^3/6 + x^4/24 + x^5/120; HIP's order ((xh xh) xh) poly, each product rounded + const float xh = x.h; + const float x2 = kf_opq(xh * xh), x3 = kf_opq(x2 * xh); + const float tail = kf_opq(x3 * kf_fma(xh, kf_fma(xh, KF_1_120, KF_1_24), KF_SIXTH)); // not fused into q.h + tail + kf2 q = kf2_two_prod(xh, xh); + q.h = q.h * 0.5f; + q.l = kf_fma(xh, x.l, q.l * 0.5f); + kf2 s = kf2_add(kf2_make(1.0f, 0.0f), x); + s = kf2_add(s, kf2_add(q, kf2_make(tail, 0.0f))); + const int j = (int)jf + 32; + kf2 v = kf2_mul(kf2_make(asfloat(KF_EXP_H[j]), asfloat(KF_EXP_L[j])), s); + const float sc = asfloat((uint)((int)n + 127) << 23); // 2^n, exact (n in [-125, 127]) + v.h = v.h * sc; + v.l = v.l * sc; + return v; +} + +// The binary32 functions of specs 42 / 43: the double-word path, else the binary64 path above. +float kf_log2(float x) +{ + if (asuint(x) - 0x00800000u < 0x7f000000u) { // positive, normal, finite (bits 0x00800000 .. 0x7f7fffff) + const kf2 r = kf2_log2(x); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_log2((double)x); +} +float kf_ln(float x) +{ + if (asuint(x) - 0x00800000u < 0x7f000000u) { + const kf2 r = kf2_mul(kf2_log2(x), kf2_make(KF2_LN2_H, KF2_LN2_L)); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_log((double)x); +} +float kf_exp2(float y) +{ + // callers pass products (log2(z) c, e 9 c) that HIP rounds before kf2_exp2's y - rint(y); the gfx1201 ISA keeps + // them apart without a kf_opq here (G1's precaution link, dropped in speed round 1 after the ISA check) + if (y >= -125.0f && y < 127.0f) { + const kf2 r = kf2_exp2(y); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_exp2((double)y); +} diff --git a/clean/k/dx12/hlsl/k_frame.hlsli b/clean/k/dx12/hlsl/k_frame.hlsli new file mode 100644 index 00000000..fe37800a --- /dev/null +++ b/clean/k/dx12/hlsl/k_frame.hlsli @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's frame inputs and their fetch helpers (clean/k/kernels/k_io.hip: KFrameIn, KInParams, +// KOutParams, KiHost, kio_*). WrongGPU's own code; the same operations and bits as the HIP helpers (k_fp.hlsli rules). +// +// Arguments. The HIP kernels take the structs by value with 64-bit pointers inside; the D3D12 ports get them as root +// constants packed by the device from the manifest's struct tokens (clean/runtime/struct_args.h): every pointer one +// DWORD (the arena byte offset, 0 = null), byte runs copied, skipped runs dropped. The macros below are the matching +// argument-block declarations (between WG_ARGS_BEGIN and WG_ARGS_END, in this order) and the loaders that rebuild the +// kernel-side view: +// KInParams s200:p4.b128.p5 K_ARGS_FRAME_IN K_ARGS_IN_PTRS 41 DWORDs +// KInUniformParams s248:p4.b128.p5.b48 K_ARGS_FRAME_IN K_ARGS_IN_PTRS K_ARGS_HOST 53 +// KOutParams s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 K_ARGS_OUT 52 +// (skipped: cbx..zby, always 0 in the runtime (runtime_k.cpp sets no subrect base for the output stage); kappa / rho +// / lam, which the kernels replace by KFrameIn.xs (the runtime always passes xs); pad_; exp2tab, the emulator's +// table only: on the GPU ko_exp2h is v_exp_f16. K_LOAD_FRAME_OUT sets the skipped fields to 0 / 1.) +// Field names: F_ (KFrameIn), P_ (KInParams pointers), H_ (KiHost), O_ (KOutParams), U_ (KOutUni). +// +// Include after wg_platform.hlsli, the argument block and k_fp.hlsli. With WG_DIRECT_TEXTURES (rs=direct ports) the +// colour, depth and motion fetches read the game's textures t0 / t1 / t2 of clean/dx12/hlsl/native_direct.hlsli +// (include it first) instead of the arena (kio_texel / kio_motion / kio_depth; see there). +#pragma once + +#define KIO_HDR 1 +#define KIO_MVLO 2 +#define KIO_INV 4 +#define KIO_RESET 8 +#define KIO_C16 16 +#define KIO_M16 32 +#define KIO_OUT_RGBA32F 0 +#define KIO_OUT_RGBA16F 1 +#define KIO_OUT_RGB32F 2 + +// ---- argument blocks (root constants, DWORD order = the struct tokens) -------------------------------------------- +#define K_ARGS_FRAME_PTRS uint F_color, F_motion, F_depth, F_xs; +#define K_ARGS_FRAME_IN K_ARGS_FRAME_PTRS \ + int F_cpitch, F_mpitch, F_zpitch, F_cbx, F_cby, F_mbx, F_mby, F_zbx, F_zby, F_W, F_H, F_rw, F_rh, F_Bw, F_Bh, F_flags; \ + float F_jx, F_jy, F_sx, F_sy, F_mox, F_moy, F_kappa, F_rho, F_lam, F_s; \ + int F_mch; \ + float F_rmsx, F_rmsy, F_rmox, F_rmoy; \ + int F_pad_; +#define K_ARGS_IN_PTRS uint P_Hc, P_Hh, P_x0, P_cpre, P_cpre2; +#define K_ARGS_HOST float H_xi_x, H_xi_y, H_a_x, H_a_y, H_h_x, H_h_y, H_g_x, H_g_y, H_sp_x, H_sp_y, H_mp_x, H_mp_y; +#define K_ARGS_OUT K_ARGS_FRAME_PTRS \ + int F_cpitch, F_mpitch, F_zpitch; \ + int F_W, F_H, F_rw, F_rh, F_Bw, F_Bh, F_flags; \ + float F_jx, F_jy, F_sx, F_sy, F_mox, F_moy; \ + float F_s; \ + int F_mch; \ + float F_rmsx, F_rmsy, F_rmox, F_rmoy; \ + uint O_N, O_L, O_Hc, O_table; \ + uint O_out, O_Lout, O_Hcout, O_Hhout; \ + int O_opitch, O_omode; \ + uint O_ctap, O_ctapT; \ + float U_invW, U_invH, U_rhox, U_rhoy, U_rho2x, U_rho2y, U_jmx, U_jmy, U_spx, U_spy, U_invW16, U_invH16, U_kinv; \ + uint U_kux_kvy; + +// ---- kernel-side structs -------------------------------------------------------------------------------------------- +struct KFrameIn { + uint color, motion, depth, xs; // arena byte offsets (0 = null) + int cpitch, mpitch, zpitch; + int cbx, cby, mbx, mby, zbx, zby; // subrect bases + int W, H, rw, rh, Bw, Bh; // output, render, base sizes + int flags; // KIO_* + float jx, jy, sx, sy, mox, moy; // jitter, motion scale, motion offset + float kappa, rho, lam; // replaced by xs[0..2] when xs is set (kio_frame) + float s; // spec 44's base-size factor (draft 3: 1/2) + int mch; // KIO_M16: FP16 channels per motion texel + float rmsx, rmsy, rmox, rmoy; // KIO_M16: ms, mo of the raw motion +}; +struct KInParams { + KFrameIn f; + uint Hc, Hh, x0, cpre, cpre2; +}; +struct KiHost { // spec 42 section 8 + float xi_x, xi_y, a_x, a_y, h_x, h_y, g_x, g_y, sp_x, sp_y, mp_x, mp_y; +}; + +#define K_LOAD_FRAME_PTRS(F) F.color = F_color; F.motion = F_motion; F.depth = F_depth; F.xs = F_xs; +#define K_LOAD_FRAME_COMMON(F) F.cpitch = F_cpitch; F.mpitch = F_mpitch; F.zpitch = F_zpitch; \ + F.W = F_W; F.H = F_H; F.rw = F_rw; F.rh = F_rh; F.Bw = F_Bw; F.Bh = F_Bh; F.flags = F_flags; \ + F.jx = F_jx; F.jy = F_jy; F.sx = F_sx; F.sy = F_sy; F.mox = F_mox; F.moy = F_moy; F.s = F_s; F.mch = F_mch; \ + F.rmsx = F_rmsx; F.rmsy = F_rmsy; F.rmox = F_rmox; F.rmoy = F_rmoy; +// KFrameIn from K_ARGS_FRAME_IN +#define K_LOAD_FRAME_IN(F) K_LOAD_FRAME_PTRS(F) K_LOAD_FRAME_COMMON(F) \ + F.cbx = F_cbx; F.cby = F_cby; F.mbx = F_mbx; F.mby = F_mby; F.zbx = F_zbx; F.zby = F_zby; \ + F.kappa = F_kappa; F.rho = F_rho; F.lam = F_lam; +// KFrameIn from K_ARGS_OUT (the skipped fields as the runtime passes them: bases 0, scalars 1 before kio_frame) +#define K_LOAD_FRAME_OUT(F) K_LOAD_FRAME_PTRS(F) K_LOAD_FRAME_COMMON(F) \ + F.cbx = 0; F.cby = 0; F.mbx = 0; F.mby = 0; F.zbx = 0; F.zby = 0; F.kappa = 1.0f; F.rho = 1.0f; F.lam = 1.0f; +#define K_LOAD_IN_PARAMS(P) K_LOAD_FRAME_IN(P.f) P.Hc = P_Hc; P.Hh = P_Hh; P.x0 = P_x0; P.cpre = P_cpre; P.cpre2 = P_cpre2; +#define K_LOAD_HOST(k) k.xi_x = H_xi_x; k.xi_y = H_xi_y; k.a_x = H_a_x; k.a_y = H_a_y; k.h_x = H_h_x; k.h_y = H_h_y; \ + k.g_x = H_g_x; k.g_y = H_g_y; k.sp_x = H_sp_x; k.sp_y = H_sp_y; k.mp_x = H_mp_x; k.mp_y = H_mp_y; + +int kio_ceil16(int n) { return (n + 15) / 16 * 16; } + +// The kernels' view: with KFrameIn.xs the exposure scalars come from the GPU (k_expo_scalars) +KFrameIn kio_frame(KFrameIn f) +{ + KFrameIn F = f; + if (f.xs != 0u) { + F.kappa = wg_ldf(f.xs, 0u); + F.rho = wg_ldf(f.xs, 4u); + F.lam = wg_ldf(f.xs, 8u); + } + return F; +} +KInParams ki_params(KInParams p) +{ + KInParams P = p; + P.f = kio_frame(p.f); + return P; +} + +// spec 42 section 8, computed on the GPU (k_in_tiled; k_in_tiled_uniform takes the host's equal values) +KiHost ki_host(KFrameIn F) +{ + KiHost k; + const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); + k.xi_x = wg_div(1.0f, (float)F.W * F.s) * 0.5f; // RN32(RN32(1/RN32(W s)) 0.5) + k.xi_y = wg_div(1.0f, (float)F.H * F.s) * 0.5f; + k.a_x = (float)F.rw * k.xi_x; + k.a_y = (float)F.rh * k.xi_y; + k.h_x = wg_div((float)F.W, (float)W16) * k.xi_x; + k.h_y = wg_div((float)F.H, (float)H16) * k.xi_y; + k.g_x = wg_div(0.25f, (float)F.Bw); + k.g_y = wg_div(0.25f, (float)F.Bh); + const bool mvlo = (F.flags & KIO_MVLO) != 0; + k.sp_x = wg_div(F.sx, wg_div(1.0f, (float)F.W) * (float)(mvlo ? F.rw : F.W)); + k.sp_y = wg_div(F.sy, wg_div(1.0f, (float)F.H) * (float)(mvlo ? F.rh : F.H)); + k.mp_x = k.xi_x * (float)F.W; + k.mp_y = k.xi_y * (float)F.H; + return k; +} + +// the motion factor with a texture's scale folded in (direct forms: b1's ms; KIO_M16: rmsx ..): spec 42 section 8's +// sp = s / (RN(1 / n) r), s = sx ms (the host's KiHost sp_x with that scale), and the raw-unit offset +float ki_dsp(float sx, float ms, int n, int r) { return wg_div(sx * ms, wg_div(1.0f, (float)n) * (float)r); } +float ki_dmo(float o, float mo, float ms) { return mo == 0.0f ? o : o + wg_div(mo, ms); } + +// spec 42 section 3: index = b + min(i + d, n - 1) if i + d > 0, else b. For n >= 1 (every render / output size) that +// is b + max(min(i + d, n - 1), 0) (HIP's ko_clampi form: one v_maxmin_i32 instead of a compare, a minimum and a select) +int kio_tap(int i, int d, int b, int n) { return b + ki_max(ki_min(i + d, n - 1), 0); } +// [MVlo] motion fetch: i' = b + i + d; index = min(i' + b, b + n - 1) if i' > 0, else b; for n >= 1 that is +// b + max(min(i', n - 1), 0) +int kio_mtap(int i, int d, int b, int n) { return b + ki_max(ki_min(b + i + d, n - 1), 0); } + +// colour texel (x, y) of the texture (subrect base included by the caller): FP32 RGBA, or RGBA16F (KIO_C16, exact +// widening). Direct: the game's colour texture t0 at (nd_rx + x, nd_ry + y), alpha 0 (unused by K), not divided by the +// pre-exposure (K applies kappa): the values of the HIP executor's RGBA16F input copy for a 16-bit float colour texture. +float4 kio_texel(KFrameIn F, int x, int y) +{ +#if WG_DIRECT_TEXTURES + return nd_raw_color_xy(x, y); +#else + const uint i = ((uint)y * (uint)F.cpitch + (uint)x) * 4u; + if ((F.flags & KIO_C16) != 0) { + const uint2 h = wg_ld64(F.color, i * 2u); + return float4(wg_h2f_lo(h.x), wg_h2f_hi(h.x), wg_h2f_lo(h.y), wg_h2f_hi(h.y)); + } + return wg_ld4f(F.color, i * 4u); +#endif +} +float4 kio_color(KFrameIn F, int ix, int iy) +{ + return kio_texel(F, kio_tap(ix, 0, F.cbx, F.rw), kio_tap(iy, 0, F.cby, F.rh)); +} +// motion texel (x, y) (subrect base included by the caller): FP32 pairs, or KIO_M16's raw FP16 value r. Direct: the raw +// value r of the game's motion texture t2. ki_dsp / ki_dmo apply the texture's scale and offset as the buffer form +// applies WgRtFrame's (below). +// KIO_M16 (the interop's raw-buffer route: input copies or model K's compact input pass, ms / mo in KFrameIn rmsx .., +// WgRtFrame's scale 1): the raw FP16 value r, the scale folded into the motion factor and the offset RN(mo / ms) in raw +// units, as for the direct forms (issue #26; before: ms r + mo per texel, then the factor of scale 1). +// Direct forms (issue #26): the motion scale and offset of the game's motion texture t2 come in b1 (ms, mo; the +// executor passes 1 / 0 in WgRtFrame), and they are applied as the buffer forms and the reference (clean/k/reference +// instage.py / outstage.py, spec 42 / 43) apply WgRtFrame's mv_scale / mv_bias: the texel's raw value r plus the offset +// in raw units, times the motion factor with the scale folded in (sp = RN(s / RN(RN(1 / n) r)) of the host, s = the +// scale). Before, the direct forms scaled every texel (RN(ms r) + mo: input; one fused mad(ms, r, mo): output) and +// used the factor of scale 1: with a non-unit scale the products round differently, which moved history taps (0.02-0.04 % +// of the output and history values differed from the buffer form from the first frame after a reset, in-game packs). +// Offset: b1's mo is in render pixels after the scale (mo = ms b, one FP32 product); the raw-unit offset is RN(mo / ms) +// (exactly 0 for mo = 0; equal to the buffer drive's b whenever that quotient recovers it). +void kio_motion(KFrameIn F, int x, int y, out float mvx, out float mvy) +{ +#if WG_DIRECT_TEXTURES + const float2 r = native_direct_motion_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)).xy; + mvx = r.x; + mvy = r.y; +#else + if ((F.flags & KIO_M16) != 0) { + const uint o = ((uint)y * (uint)F.mpitch + (uint)x) * (uint)F.mch * 2u; + const uint r = wg_ld32(F.motion, o); // channels 0, 1 (4-byte aligned) + mvx = wg_h2f_lo(r); + mvy = wg_h2f_hi(r); + return; + } + const float2 m = asfloat(wg_ld64(F.motion, ((uint)y * (uint)F.mpitch + (uint)x) * 8u)); + mvx = m.x; + mvy = m.y; +#endif +} + +float kio_depth(KFrameIn F, int ix, int iy, int dx, int dy) +{ + const int x = kio_tap(ix, dx, F.zbx, F.rw), y = kio_tap(iy, dy, F.zby, F.rh); +#if WG_DIRECT_TEXTURES + return native_direct_depth_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)); +#else + return wg_ldf(F.depth, ((uint)y * (uint)F.zpitch + (uint)x) * 4u); +#endif +} + +// spec 42 5.2 [MVlo]: offset of the nearest-depth motion pixel +void kio_nearest(KFrameIn F, int ix, int iy, out int odx, out int ody) +{ + const float c = kio_depth(F, ix, iy, 0, 0), a = kio_depth(F, ix, iy, -1, -1), b = kio_depth(F, ix, iy, 1, -1); + const float d = kio_depth(F, ix, iy, -1, 1), e = kio_depth(F, ix, iy, 1, 1); + const float k = kf_bits(0x3A83126Fu); + const bool inv = (F.flags & KIO_INV) != 0; + bool edge; + if (inv) + edge = (c < 0.5f * (a + e) - kf_opq(k * c)) || (c < 0.5f * (b + d) - kf_opq(k * c)); + else + edge = (c > kf_fma(c, k, 0.5f * (a + e))) || (c > kf_fma(c, k, 0.5f * (b + d))); + int bx = 0, by = 0; + if (edge) { + float best = c; + const float v[4] = {a, b, d, e}; + const int ox[4] = {-1, 1, -1, 1}, oy[4] = {-1, -1, 1, 1}; + [unroll] for (int q = 0; q < 4; ++q) { + const bool nearer = inv ? v[q] > best : v[q] < best; + if (nearer) { + best = v[q]; + bx = ox[q]; + by = oy[q]; + } + } + } + odx = bx; + ody = by; +} + +// spec 42 decode factor f (t f is linear) and spec 43 encode e(Y) (stored = Y e(Y)), correctly rounded steps +#define K_KIO_DECODE_NEAR kf_bits(0x3F7FBE88u) // below it: 1 / (1 - t) +#define K_KIO_ENCODE_NEAR 1000.0f // below it: 1 / (Y + 1) +float kio_decode(float t) +{ + if (t < K_KIO_DECODE_NEAR) return wg_div(1.0f, 1.0f - t); + const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); + return wg_div(kf_exp2(e) - 1.0f, 9.0f * t); +} +float kio_encode(float Y) +{ + if (Y < K_KIO_ENCODE_NEAR) return wg_div(1.0f, Y + 1.0f); + return wg_div(wg_div(kf_ln(kf_fma(Y, 9.0f, 1.0f)), 9.0f) - kf_bits(0x3C4FAE29u), Y); +} + +// bilinear sample of a 4-channel FP16 image (arena offset img, [h][w][4]) at normalized (cx, cy), clamp to edge; +// spec 42 section 1 (draft 2): t0 = v00 + (v10 - v00) fx, t1 = v01 + (v11 - v01) fx, (t0 (1 - fy)) + (t1 fy) +float4 kio_texel_h4(uint img, uint idx) +{ + const uint2 q = wg_ld64(img, idx * 8u); + return float4(wg_h2f_lo(q.x), wg_h2f_hi(q.x), wg_h2f_lo(q.y), wg_h2f_hi(q.y)); +} +float4 kio_bilinear4(uint img, int w, int h, float cx, float cy) +{ + const float px = kf_opq(cx * (float)w) - 0.5f, py = kf_opq(cy * (float)h) - 0.5f; + const float fx0 = floor(px), fy0 = floor(py); + const float fx = px - fx0, fy = py - fy0; + // HIP: finite ? (int)max(min(f, w), -1) : (+inf: w, -inf: -1, NaN: 0). The clamp already gives w / -1 for +-inf + // (min / max are IEEE minNum / maxNum), so only NaN needs the select + const int x0 = isnan(fx0) ? 0 : (int)kf_max(kf_min(fx0, (float)w), -1.0f); + const int y0 = isnan(fy0) ? 0 : (int)kf_max(kf_min(fy0, (float)h), -1.0f); + const int xa = ki_max(ki_min(x0, w - 1), 0), xb = ki_max(ki_min(x0 + 1, w - 1), 0); + const int ya = ki_max(ki_min(y0, h - 1), 0), yb = ki_max(ki_min(y0 + 1, h - 1), 0); + const float omfy = 1.0f - fy; + const float4 q00 = kio_texel_h4(img, (uint)ya * (uint)w + (uint)xa), q10 = kio_texel_h4(img, (uint)ya * (uint)w + (uint)xb); + const float4 q01 = kio_texel_h4(img, (uint)yb * (uint)w + (uint)xa), q11 = kio_texel_h4(img, (uint)yb * (uint)w + (uint)xb); + const float4 t0 = q00 + kf_opq4(q10 - q00) * fx; + const float4 t1 = q01 + kf_opq4(q11 - q01) * fx; + return kf_opq4(t0 * omfy) + kf_opq4(t1 * fy); +} + +// (Y, Co, Cg) of max(RGB, 0) scaled by kappa first (spec 42 6.2) or not (spec 43 4.2: kappa after) +void kio_ycocg(float R, float G, float B, out float Y, out float Co, out float Cg) +{ + Y = (0.25f * R + 0.5f * G) + 0.25f * B; + Co = kf_fma(R, 0.5f, kf_opq(0.0f) * G) - 0.5f * B; // 0 G: NaN for an infinite G, as HIP + Cg = (0.5f * G - 0.25f * R) - 0.25f * B; +} +float kio_luma_fma(float R, float G, float B) { return kf_fma(B, 0.25f, kf_fma(R, 0.25f, 0.5f * G)); } diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py index b07dc5f5..82a1a2cb 100644 --- a/clean/k/dx12/k_plan_bridge.py +++ b/clean/k/dx12/k_plan_bridge.py @@ -124,10 +124,11 @@ def chain_step(kernel, grid, block, args, tokens, bufs, cdir, tag): return f"step {kernel} {grid} {block} " + " ".join(out) -def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, fills=None): +def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, d3d12_name=None): """The kernel_replay chain of a plan: buffers (inputs from files, the rest filled with the D3D12 sentinel), one step per launch, a save per output buffer (..d3d12). grid_div: launch with grid // grid_div (the - negative control uses 2). fills: {buffer name: bytes} initial contents that differ from the HIP run's (none).""" + negative control uses 2). d3d12_name: the manifest's name of a plan kernel (default: the part after the code object's + colon; the tagged exact input kernels are kx/). modules: module lines, "[ ]".""" bufs = {n: (p, pdev.sizes[p]) for n, p in plan["bufs"].items()} lines = [f"chain {prefix}{name}"] + [f"module {m}" for m in modules] for n, (p, size) in bufs.items(): @@ -136,7 +137,8 @@ def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", else: lines.append(f"buf b_{n} {size + SLACK} - {D3D12_SENTINEL}") for li, (kernel, grid, block, args) in enumerate(plan["launches"]): - lines.append(chain_step(kernel.split(":")[-1], max(1, grid // grid_div), block, args, tokens_of(kernel), bufs, cdir, + dname = d3d12_name(kernel) if d3d12_name else kernel.split(":")[-1] + lines.append(chain_step(dname, max(1, grid // grid_div), block, args, tokens_of(dname), bufs, cdir, f"{prefix}{name}.l{li}")) for n in outputs: lines.append(f"save b_{n} {prefix}{name}.{n}.d3d12") diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index 95575417..bee8e3f5 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -18,7 +18,9 @@ the half-grid control (the port again with half its workgroups) is used by the families whose grid has more than one workgroup; - Model M's scale_rgb (frame.co, its port already proven) runs through the same replay as a control. -The families: wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's +The families: input = the exact tiled input stage (k_io_exact.co, kx/k_in_tiled) on frames of every flag combination (HDR, +low-resolution motion, inverted depth, reset), odd and tiny base grids, exposure scalars from a device buffer, special values +in the colour, depth, motion and histories (the same three levels as the stage families); wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, +-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). @@ -50,6 +52,7 @@ "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "narrow": (["k_block.co"], NARROW_KERNELS), + "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), "wide": (["k_block_wide.co"], ["k_stagew2_e3", "k_stagew4_e4", "k_stagew2_e4", "k_stagew4_m", "k_stagew4_d4", "k_stagew2_d4", "k_stagew2_d3"])} CONTROL_CO = "frame.co" @@ -381,11 +384,11 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]} - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, [co_name], + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs) if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels neg = name - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs, prefix="negative_", grid_div=2) items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size), "offset": list(offset), "specials": special, "outputs": outputs, "hip": got, @@ -445,6 +448,91 @@ def check_narrow(state, read): "failures": [o for o in out if not o["identical"]][:4]}, ok +# ---- family input: the exact tiled input stage (k_io_exact.co) ----------------------------------------------------- + +INPUT_BASES = [(1, 1), (7, 9), (8, 8), (9, 7), (33, 17)] +INPUT_FLAG_SETS = [(h, m, i, r) for h in (True, False) for m in (True, False) for i in (True, False) + for r in (False, True)] + + +def input_specials(x, rng, level): + """level 1: infinity, huge, zero and -0 values in the colour, depth and motion, and non-finite history; level 2: NaN + as well (in the colour, depth, motion and histories).""" + pool = [np.inf, -np.inf, 3.0e38, -3.0e38, 0.0, -0.0, 65504.0, -65504.0, 1e-30] + if level == 2: + pool += [np.nan] + pool = np.array(pool, np.float32) + for k in ("color", "depth", "motion"): + a = x[k] + pick = rng.random(a.shape) < 0.004 + a[pick] = rng.choice(pool, int(pick.sum())) + for k in ("Hc", "Hh"): + a = x[k] + pick = rng.random(a.shape) < 0.01 + a[pick] = rng.choice(pool.astype(np.float16), int(pick.sum())) + + +def run_input(build, rocm, seeds, cdir, hip): + from clean.k.dx12 import k_plan_bridge as PB + from clean.k.runner import k_io, k_stages + from clean.k.tests.test_k_io_emu import frame + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed + 500) + for ci, (hdr, mvlo, inv, reset) in enumerate(INPUT_FLAG_SETS): + base = INPUT_BASES[(ci + si) % len(INPUT_BASES)] + special = (ci // 2) % 3 # 0 none, 1 specials without NaN, 2 with NaN + name = f"inp_s{si}_{ci}" + x = frame(rng, hdr, mvlo, inv, reset=reset, base=base) + x["base"] = base + x.update(E_cur=1.5, E_prev=1.0, E_aux=0.75, pre_exposure=2.0) + if special: + input_specials(x, rng, special) + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, "input@exact", b"", x, None, None) + try: + prm = plan["launches"][-1][3][0] + # the exposure scalars come from a device buffer, as in the runtime (the host values are wrong on purpose) + xs = np.array([prm.f.kappa, prm.f.rho, prm.f.lam, np.float32(1.0) / np.float32(prm.f.kappa)], np.float32) + ptr = pdev.malloc(xs.nbytes) + pdev.upload(ptr, xs.tobytes()) + plan["bufs"]["xs"], plan["sizes"]["xs"] = ptr, xs.nbytes + prm.f.xs = ptr + prm.f.kappa, prm.f.rho, prm.f.lam = -17.0, 7.0, 9.0 + bufs = {n: getattr(prm, n) for n in ("Hc", "Hh", "x0", "cpre", "cpre2")} + plan["launches"] = k_io.input_launches(prm.f, bufs, exact=True, tiled=True) + if len(plan["launches"]) != 1: + continue # unsupported geometry: the runtime falls back to HIP's untiled pair + if reset: # a reset frame reads no history, motion or depth + act = plan["launches"][0][3][0] + act.Hc = act.Hh = None + act.f.motion = act.f.depth = None + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + for k, grid, block, a in plan["launches"]: + co, kernel = k.split(":") + hip.h.launch(hip.h.get_function(hip.mods[co], kernel), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {n: min(int(np.prod(sh)) * 2, sizes[n]) for n, sh in plan["outputs"]} + kw = dict(modules=["k_io_exact.co kx"], outputs=outputs, d3d12_name=lambda k: "kx/" + k.split(":")[-1]) + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, **kw) + if neg is None and plan["launches"][0][1] >= 4 and not reset: + neg = name + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, prefix="negative_", grid_div=2, + **kw) + items.append({"case": name, "kernel": "kx/k_in_tiled", "stage": "input", "size": list(base), + "offset": [0, 0], "specials": special, "outputs": outputs, "hip": got, "defined": defined}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + return items, lines, neg + + # ---- run ----------------------------------------------------------------------------------------------------------- def run(build, rocm, families, seeds, json_out): @@ -465,6 +553,9 @@ def run(build, rocm, families, seeds, json_out): if "narrow" in families: state["narrow"] = run_stages(NARROW, "k_block.co", "nar", build, rocm, seeds, cdir, hip) lines += state["narrow"][1] + if "input" in families: + state["input"] = run_input(build, rocm, seeds, cdir, hip) + lines += state["input"][1] if "wide" in families: state["wide"] = run_stages(WIDE, "k_block_wide.co", "wid", build, rocm, seeds, cdir, hip, WIDE_SIZES) lines += state["wide"][1] @@ -520,7 +611,7 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 - for fam in ("narrow", "wide"): + for fam in ("narrow", "wide", "input"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) ok = ok and good diff --git a/clean/k/dx12/kernels/k_in.hlsl b/clean/k/dx12/kernels/k_in.hlsl new file mode 100644 index 00000000..ff103a3c --- /dev/null +++ b/clean/k/dx12/kernels/k_in.hlsl @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's exact tiled input-stage kernels (clean/k/kernels/k_io.hip ki_tiled_body, HIP build +// k_io_exact.co = WGK_IN_EXACT, wave64; spec 42 sections 3-8). WrongGPU's own code. +// +// One workgroup of 256 lanes per 8 x 8 base positions: lane 4 b + k is display position k of base position b. The +// workgroup first stages the 24 x 24 render-colour footprint of its tile in LDS (Y, Co, Cg, phi(Y) and the HDR weight +// omega per render pixel), then each lane does its display position's 4 fine positions (reprojected colour history, +// motion with the nearest-depth rule, hidden-history position), the 3 x 3 colour neighbourhood (mean, box, phi), the +// disagreement score, the own sample's luma channel, and the hidden-history sample; the 4 display positions of a base +// position exchange their hidden-history samples by two lane swaps (xor 1, xor 2: within groups of 4 lanes, so the +// wave size does not matter; wave64 as the HIP build) and add them in HIP's order. Writes x0 [Bh][Bw][16] FP16. +// Reset frames: only the own sample's channel 0 (channels 1 = 0 's, 2 and 3 zero), no LDS. +// +// Arithmetic: exactly the HIP kernel's operations in its order (k_io.hip has contraction off and every fma written +// out), with the exact path of WGK_IN_EXACT: log2 / 2^x correctly rounded (kf_log2 / kf_exp2, k_fp.hlsli, with their +// binary64 fallback: extended doubles), every division and reciprocal correctly rounded (wg_div; HIP's a / b and +// 1.0f / x), FP16 rounding to nearest even (wg_f2h; never the driver's (half), which rounds toward zero). Where the +// D3D12 compiler could merge two of HIP's roundings (a product by the uniform kappa or rho, a constant chain), the +// intermediate goes through kf_opq (k_fp.hlsli). FP32 denormals are flushed here (HIP keeps them): none occur in the +// port checks' frames (clean/k/dx12/k_port_check.py). +// +// Kernel declared here: kx/k_in_tiled (KInParams; computes KiHost on the GPU). The forms with KiHost in the arguments +// (kx/k_in_tiled_uniform, K_UNIFORM) and the forms reading the game's textures (_direct, WG_DIRECT_TEXTURES) are in this +// source for the runtime's use but are not declared yet: they are declared and checked with the runtime wiring. +// The KInParams pointers cpre / cpre2 are null in the tiled launches (runtime_k.cpp), as in HIP unused here. +// @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 +#include "wg_platform.hlsli" +#ifndef K_UNIFORM +#define K_UNIFORM 0 +#endif +#ifndef WG_DIRECT_TEXTURES +#define WG_DIRECT_TEXTURES 0 +#endif +#if WG_DIRECT_TEXTURES +#include "native_direct.hlsli" +#endif +#include "k_fp.hlsli" +#include "k_frame.hlsli" + +WG_ARGS_BEGIN + K_ARGS_FRAME_IN + K_ARGS_IN_PTRS +#if K_UNIFORM + K_ARGS_HOST +#endif +WG_ARGS_END + +float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } + +// ---- the exact input stage's arithmetic (k_io.hip with KI_FAST = 0) -------------------------------------------------- +float ki_log2(float x) { return kf_log2(x); } +float ki_exp2(float x) { return kf_exp2(x); } +float ki_rcp(float x) { return kf_div(1.0f, x); } +float ki_div(float a, float b) { return kf_div(a, b); } +// spec 42 decode factor D(t) +float ki_decode(float t) +{ + if (t < kf_bits(0x3F7FBE88u)) return ki_rcp(1.0f - t); + const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); + return ki_div(ki_exp2(e) - 1.0f, 9.0f * t); +} +float ki_phi(float Y, bool hdr) +{ + if (hdr) { + const float Yp = kf_max(Y, 0.0f); + return ki_div(ki_log2(kf_fma(Yp, 12.0f, 1.0f)) * kf_bits(0x3F317218u), Yp + kf_bits(0x358637BDu)); + } + return kf_fma(Y, 3.0f, 1.0f); +} +float ki_S(float z) +{ + const float zp = kf_min(kf_max(z, 0.0f), 1.0f); // HIP's NaN test is implied: maxNum(NaN, 0) = 0 + if (!(zp > kf_bits(0x3B4D2E1Cu))) return kf_bits(0x414EB852u) * zp; + const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once + return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu)); +} +// z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart +// (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it) +float ki_T(float z) +{ + return z * kf_max(ki_rcp(z + 1.0f), kf_bits(0x38D1B717u)); +} +float ki_L(float Y, bool hdr) { return hdr ? Y : ki_div(Y, kf_max(1.0f - kf_opq(Y), kf_bits(0x3E4CCCCDu))); } + +struct KiFine { // spec 42 section 5 per fine position + bool valid; + float3 h; // reprojected history (Y, Co, Cg) + float nx, ny; // hidden-history position (normalized) +}; + +KiFine ki_fine(KInParams P, KiHost k, int fx, int fy) +{ + const KFrameIn F = P.f; + const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); + const float px = (float)fx + 0.5f, py = (float)fy + 0.5f; + float mvx, mvy; + if ((F.flags & KIO_MVLO) != 0) { + const int ix = (int)kf_R(F.jx + kf_fma(px, k.a_x, -0.5f)), iy = (int)kf_R(F.jy + kf_fma(py, k.a_y, -0.5f)); + int dx, dy; + kio_nearest(F, ix, iy, dx, dy); + kio_motion(F, kio_mtap(ix, dx, F.mbx, F.rw), kio_mtap(iy, dy, F.mby, F.rh), mvx, mvy); + } else { + const int mx = (int)kf_R(kf_fma(px, k.mp_x, -0.5f)), my = (int)kf_R(kf_fma(py, k.mp_y, -0.5f)); + kio_motion(F, kio_tap(mx, 0, F.mbx, F.W), kio_tap(my, 0, F.mby, F.H), mvx, mvy); + } + // rounded before px + m (HIP); no kf_opq needed: the ISA keeps the products (precaution link dropped, ISA-checked) + // the texture's motion scale folded into the factor and its offset in raw units, as the buffer form gets them in + // WgRtFrame (hlsl/k_frame.hlsli kio_motion; issue #26): direct forms b1's ms / mo, KIO_M16 rmsx .. +#if WG_DIRECT_TEXTURES + const bool dfold = true; + const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy; +#else + const bool dfold = (F.flags & KIO_M16) != 0; + const float dmsx = F.rmsx, dmsy = F.rmsy, dmox = F.rmox, dmoy = F.rmoy; +#endif + float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy; + [branch] if (dfold) { + const bool dmvlo = (F.flags & KIO_MVLO) != 0; + spx = ki_dsp(F.sx, dmsx, F.W, dmvlo ? F.rw : F.W); + spy = ki_dsp(F.sy, dmsy, F.H, dmvlo ? F.rh : F.H); + ofx = ki_dmo(F.mox, dmox, dmsx); + ofy = ki_dmo(F.moy, dmoy, dmsy); + } + const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy); + const float wx = px + mx, wy = py + my; + KiFine r; + const float ux = wx * k.xi_x, uy = wy * k.xi_y; + r.valid = ux >= 0.0f && ux <= 1.0f && uy >= 0.0f && uy <= 1.0f; + const float4 s = kio_bilinear4(P.Hc, W16, H16, wx * k.h_x, wy * k.h_y); + if ((F.flags & KIO_HDR) != 0) { + const float f = ki_decode(s.x); + r.h = F.rho * kf_opq3(s.xyz * f); + } else { + r.h = s.xyz; + } + r.nx = ((float)(4 * (fx >> 2) + 2) + mx) * k.g_x; // fx, fy >= 0: fx / 4 = fx >> 2 + r.ny = ((float)(4 * (fy >> 2) + 2) + my) * k.g_y; + return r; +} + +// kappa max(RGB, 0), each product rounded (kf_opq: no merging with the 0.25 / 0.5 factors that follow) +float3 ki_rgb(KFrameIn F, float4 c) { return kf_opq3(F.kappa * max(c.xyz, 0.0f)); } + +struct KiTiledTap { + float4 cv; // Y, Co, Cg, phi(Y) + float om; // HDR weight max(1 / (Y + 1), 1e-4); 1 for LDR (A = fma(1, v, A) is HIP's A + v, one rounding of the sum) +}; +KiTiledTap ki_tiled_tap_from_raw(KFrameIn F, float4 c) +{ + const float3 rgb = ki_rgb(F, c); + float Y, Co, Cg; + kio_ycocg(rgb.x, rgb.y, rgb.z, Y, Co, Cg); + const bool hdr = (F.flags & KIO_HDR) != 0; + KiTiledTap r; + r.cv = float4(Y, Co, Cg, ki_phi(Y, hdr)); + r.om = hdr ? kf_max(ki_rcp(Y + 1.0f), kf_bits(0x38D1B717u)) : 1.0f; + return r; +} +KiTiledTap ki_tiled_tap(KFrameIn F, int rx, int ry) { return ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); } +float ki_tiled_ch0(KFrameIn F, int rx, int ry) +{ + const float3 rgb = ki_rgb(F, kio_color(F, rx, ry)); + const bool hdr = (F.flags & KIO_HDR) != 0; + const float Yc = kio_luma_fma(rgb.x, rgb.y, rgb.z); + return ki_S(ki_T(F.lam * ki_L(Yc, hdr))); +} +// HIP ki_tiled_transpose: lane bit 0 / bit 1 swaps through ds_bpermute (wg_xor = WaveReadLaneAt(lane ^ d)) +float4 ki_tiled_transpose(float4 v, int lane) +{ + const bool b0 = (lane & 1) != 0, b1 = (lane & 2) != 0; + const float x = wg_xor(b0 ? v.x : v.y, 1u), y = wg_xor(b0 ? v.z : v.w, 1u); + const float4 a = float4(b0 ? x : v.x, b0 ? v.y : x, b0 ? y : v.z, b0 ? v.w : y); + const float z = wg_xor(b1 ? a.x : a.z, 2u), w = wg_xor(b1 ? a.y : a.w, 2u); + return float4(b1 ? z : a.x, b1 ? w : a.y, b1 ? a.z : z, b1 ? a.w : w); +} +void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3) +{ + wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, + uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); +} + +groupshared float4 tile_cv[24 * 24]; +groupshared float tile_om[24 * 24]; + +#if K_UNIFORM +WG_KERNEL(k_in_tiled_uniform) +#else +WG_KERNEL(k_in_tiled) +#endif +{ + WG_SETUP(); + KF_SETUP(); + KInParams P0; + K_LOAD_IN_PARAMS(P0) + const KInParams P = ki_params(P0); + const KFrameIn F = P.f; +#if K_UNIFORM + KiHost hk; + K_LOAD_HOST(hk) +#else + const KiHost hk = ki_host(F); +#endif + const int lane = (int)wg_lane_id(); + const int tiles_x = (F.Bw + 7) / 8, tile = (int)wg_block_id(); + const int bx0 = (tile % tiles_x) * 8, by0 = (tile / tiles_x) * 8; + const int k = lane & 3, local = lane >> 2; + const int bx = bx0 + (local & 7), by = by0 + (local >> 3); + const bool active = bx < F.Bw && by < F.Bh; + const int id = by * F.Bw + bx; + const bool hdr = (F.flags & KIO_HDR) != 0; + const float c1e4 = kf_bits(0x38D1B717u); + float ch0 = 0.0f, ch1 = 0.0f, ch3 = 0.0f; + float4 hid = 0.0f; + if ((F.flags & KIO_RESET) != 0) { + if (active) { + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); + const float h = ki_tiled_ch0(F, rx, ry); + ki_store_x0(P.x0, id, k, h, h, 0.0f, 0.0f); + } + return; + } + const int tx0 = ki_max(ki_min((int)kf_R(F.jx + kf_fma((float)(4 * bx0 + 1), hk.a_x, -0.5f)) - 1, F.rw - 1), 0); + const int ty0 = ki_max(ki_min((int)kf_R(F.jy + kf_fma((float)(4 * by0 + 1), hk.a_y, -0.5f)) - 1, F.rh - 1), 0); + for (int q = lane; q < 24 * 24; q += 256) { + const int raw_x = tx0 + q % 24, raw_y = ty0 + q / 24; + const int rx = ki_min(raw_x, F.rw - 1), ry = ki_min(raw_y, F.rh - 1); + const KiTiledTap value = ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); + tile_cv[q] = value.cv; + tile_om[q] = value.om; + } + wg_barrier_lds(); + if (active) { + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + // colour neighbourhood first, then each fine position scored as it is produced (the same per-position + // operations and the same order of the D sum as HIP; nothing of a fine position stays live but the own + // sample's Yh, nx, ny) + const int cx = (int)kf_R(F.jx + kf_fma((float)(2 * u + 1), hk.a_x, -0.5f)); + const int cy = (int)kf_R(F.jy + kf_fma((float)(2 * v + 1), hk.a_y, -0.5f)); + float3 A = 0.0f, lo = 0.0f, hi = 0.0f; + [unroll] for (int ddy = -1; ddy <= 1; ++ddy) { + [unroll] for (int ddx = -1; ddx <= 1; ++ddx) { + const int rx = kio_tap(cx, ddx, 0, F.rw), ry = kio_tap(cy, ddy, 0, F.rh); + const int lx = rx - tx0, ly = ry - ty0; + const bool cached = (uint)lx < 24u && (uint)ly < 24u; + float4 cv; + float om; + [branch] if (cached) { + cv = tile_cv[ly * 24 + lx]; + om = tile_om[ly * 24 + lx]; + } else { // defensive, as HIP: a footprint wider than the tile (the runtime's guard prevents it) + const KiTiledTap fb = ki_tiled_tap(F, rx, ry); + cv = fb.cv; + om = fb.om; + } + A = float3(kf_fma(om, cv.x, A.x), kf_fma(om, cv.y, A.y), kf_fma(om, cv.z, A.z)); // LDR: om = 1 + const float3 val = cv.xyz * cv.w; + const bool first = ddx == -1 && ddy == -1; + lo = first ? val : min(lo, val); + hi = first ? val : max(hi, val); + } + } + float3 mu = A * kf_bits(0x3DE38E39u); // rounded before 1 - mu (the ISA keeps the products: no kf_opq) + if (hdr) { + const float den = kf_max(1.0f - mu.x, c1e4); + mu = float3(ki_div(mu.x, den), ki_div(mu.y, den), ki_div(mu.z, den)); + } + const float pm = ki_phi(mu.x, hdr); + const float3 mp = kf_opq3(mu * pm); + const float3 eta = 0.5f * (hi - lo); + const float3 nu = float3(kf_fma(hi.x - lo.x, 8.0f, 1.0f), kf_fma(hi.y - lo.y, 8.0f, 1.0f), kf_fma(hi.z - lo.z, 8.0f, 1.0f)); + const int oq = (1 - (v & 1)) + 2 * (u & 1); // (2u + 1 - (v mod 2), 2v + (u mod 2)) as a fine position index + // fine positions of the display position, in the order (2u, 2v), (2u+1, 2v), (2u, 2v+1), (2u+1, 2v+1) + bool valid = true; + float D = 0.0f, own_hx = 0.0f, own_nx = 0.0f, own_ny = 0.0f; + [unroll] for (int q = 0; q < 4; ++q) { + const KiFine fq = ki_fine(P, hk, 2 * u + (q & 1), 2 * v + (q >> 1)); + valid = valid && fq.valid; + const float ph = ki_phi(fq.h.x, hdr); + const float3 g = max(abs(mp - kf_opq3(fq.h * ph)) - eta, 0.0f); + const float3 dl = float3(ki_div(g.x, nu.x), ki_div(g.y, nu.y), ki_div(g.z, nu.z)); + const float score = kf_min(kf_fma(dl.z, 0.5f, kf_fma(dl.x, 2.0f, 0.5f * dl.y)), 1.0f); + D = D + score; + if (q == oq) { + own_hx = fq.h.x; + own_nx = fq.nx; + own_ny = fq.ny; + } + } + D = D * 0.25f; + // own sample + const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); + ch0 = ki_tiled_ch0(F, rx, ry); + const float4 hs = kio_bilinear4(P.Hh, F.Bw, F.Bh, own_nx, own_ny); + hid = valid ? hs : 0.0f; + // channel 1 is channel 0's function of Yh, and Yh = Yc without a valid history: then it equals channel 0 + ch1 = valid ? ki_S(ki_T(F.lam * ki_L(own_hx, hdr))) : ch0; + const float d3 = valid ? D : 0.0f; + ch3 = kf_finite(d3) ? d3 : 0.0f; + } + const float4 hs = ki_tiled_transpose(hid, lane); + if (!active) return; + float hsum = (hs.x + hs.w) + (hs.y + hs.z); + hsum = hsum * 0.25f; + ki_store_x0(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3); +}