diff --git a/clean/FORMATS.md b/clean/FORMATS.md index 1f37fb82..75d6af0d 100644 --- a/clean/FORMATS.md +++ b/clean/FORMATS.md @@ -492,12 +492,12 @@ the runtime is a shared D3D12 object except the three fences. | Buffer | Size | Content | |---|---|---| -| color | max R × 16 | `[rh][rw]` 4 × f32 (alpha 0), the game's color in the active region | +| color | max R × 16 | `[rh][rw]` 4 × f32, the game's color in the active region; the fourth value +0 (model M), or for model K the texel's alpha, unmodified (1 for a format without an alpha channel): model K's output alpha follows it | | depth | max R × 4 | `[rh][rw]` f32, the game's depth as given | | motion | max R × 8 | `[rh][rw]` 2 × f32, m = motion_scale · raw + motion_offset (render pixels); with `WG_CREATE_MOTION_OUTPUT_RES` the texel ⌊(x + ½)·mw/rw⌋, ⌊(y + ½)·mh/rh⌋ (our point resampling: what the network reads with `motion_outres=point`; with `original` it reads `motion_hi`) | | motion_hi | O × 4 + 8, or none | only with `WG_CREATE_MOTION_OUTPUT_RES` (model M; with `motion_outres=point` only while a frame capture is armed): `[oh][ow]` 2 × FP16, the raw motion texels of the output-size region ("Motion vectors at output resolution"). It follows `out` in each slot; without it every offset is what it was | | expo | 256 bytes | f32 at offset 0: texel (0, 0) channel R of the exposure texture (exposure mode texture) | -| out | O × 12 | `[oh][ow]` 3 × f32, the network's result (the runtime's `filter.out` format) | +| out | O × 12 (model K: O × 16) | `[oh][ow]` 3 × f32, the network's result (the runtime's `filter.out` format); model K: `[oh][ow]` 4 × f32, the fourth value its output alpha (`FrameIo.out_rgba32f`), which the output pass stores | Fences (shared, `hipImportExternalSemaphore`, D3D12 fence): `ready` (the split request's done fence: the front end's queue signals v after the first part of the game's list, which ends with the input conversion), `done` (HIP signals v @@ -524,7 +524,7 @@ whole texture; otherwise the frame uses the input pass (logged). The network's f 0.309 → 0.251 ms, output 0.856 → 0.838 ms against the FP32 slot). - Model K's compact input pass (default for `model=K` when no copy is made; `k_input=f32`: the FP32 input pass): where the game's colour is RGBA16F or R11G11B10 and its motion RG16F or RGBA16F (FP16-representable values), the - pass `inputs16` writes dense RGBA16F colour (rw·8-byte rows, alpha 0), R32F depth and the raw RG16F motion texel + pass `inputs16` writes dense RGBA16F colour (rw·8-byte rows, the texel's alpha kept; 1 for R11G11B10), R32F depth and the raw RG16F motion texel (rw·4-byte rows; at output size the texel under the render pixel's centre, as the FP32 pass samples it) into the slot's buffers, and the network reads them as it reads the copies (motion_scale / motion_offset applied on load). Same output bits as the FP32 pass (`test_interop_d3d12 hipsame`, render- and output-size motion); logged as @@ -568,7 +568,8 @@ pre-exposure and exposure, then signals `done = v`. Values only grow. 3. Network enqueued (above). If it cannot be enqueued, the runtime's own compute queue waits for the previous `done` value and signals `done = v` (and `pipe = v`), so the game's queue cannot wait forever; the dispatch returns an error. 4. Output pass into the re-opened list: `out` of slot v (inline) or of the previous network frame (pipelined; the - first frame: bilinear upscale of the color) → the output texture (alpha 1; sRGB formats encoded), output + first frame: bilinear upscale of the color) → the output texture (alpha 1; model K: the result's fourth value, in + the upscale the bilinear of the color's alpha; sRGB formats encoded), output transitioned to UNORDERED_ACCESS if needed and back (else a UAV barrier). Shaders: HLSL compiled at create with `d3dcompiler_47.dll` (cs_5_0); the result buffer is read through an R32_FLOAT typed UAV. diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 05dd92fa..0ac9e2aa 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -383,6 +383,12 @@ def main(argv=None): for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: if fn.startswith("fused_") and fn.endswith("_gen.py"): subprocess.run([sys.executable, os.path.join(KERNEL_DIR, fn)], check=True) + # Model K's FP constants and tables (clean/k/dx12/gen_k_fp_tables.py: computed from their series, not stored in the + # repository) into /dx12/gen, the K ports' second include folder; only when a K kernel can be built + k_gen = os.path.join(dst, "gen") + if (a.model_k or only) and os.path.exists(os.path.join(K_DX12_DIR, "gen_k_fp_tables.py")): + subprocess.run([sys.executable, os.path.join(K_DX12_DIR, "gen_k_fp_tables.py"), + os.path.join(k_gen, "k_fp_tables.hlsli")], check=True) jobs = [] # (source, declaration, output path) in declaration order (sorted file name, then line) for src, k_source in source_files(a.model_k or bool(only)): for d in kernel_decls(src): @@ -410,7 +416,7 @@ def build_one(job): if d["group"] != d["block"]: defs["WG_LAUNCH_BLOCK"] = d["block"] info = {} - compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR,) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm, + compile_hlsl(src, d["entry"], out, defs, includes=(K_HLSL_DIR, k_gen) if k_source and os.path.isdir(K_HLSL_DIR) else (), asm_tool=asm, keep_listing=out[:-5] + ".ll" if a.listing else None, info=info) with open(out, "rb") as f: sha = hashlib.sha256(f.read()).hexdigest() diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 5af75100..a8bdcb5e 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -367,6 +367,10 @@ [ "k_warm", "k_block_wide.co" + ], + [ + "kx/k_in_tiled", + "k_io_exact.co" ] ] } diff --git a/clean/k/FORMATS.md b/clean/k/FORMATS.md index c2916d3c..d11fef25 100644 --- a/clean/k/FORMATS.md +++ b/clean/k/FORMATS.md @@ -91,7 +91,7 @@ Sizes: `k_pack.stage_sections(st)["_size"]` = `model_k.cpp k_stage_blob_bytes`. | luma history L | `[2⌈H⌉₁₆][2⌈W⌉₁₆]` FP16 | `k_out_pixel` | next frame's `k_out_pixel` | | hidden history Hh | `[Bh][Bw][4]` FP16 | complete G2 frames: `k_stage2_d0_hidden` (FP32) or `k_stage2h_d0_hidden` ("fused16"); otherwise `k_out_hidden` | next frame's `k_in_base` or `k_in_tiled` | | game colour / motion / depth | RGBA float32 / 2 × float32 / float32, row pitch in the launch arguments, subrect bases | the front end | both stages | -| game output | `KOutParams.omode`: 0 RGBA float32 `[H][opitch][4]` (alpha Ā); 1 RGBA16F rows of `opitch` texels (RGB rounded to nearest even, alpha 1: the interop's output copy); 2 dense RGB float32 `[H][opitch][3]` (the interop's output pass) | `k_out_pixel` | the front end | +| game output | `KOutParams.omode`: 0 RGBA float32 `[H][opitch][4]` (alpha Ā; the interop's output pass for model K, `FrameIo.out_rgba32f`); 1 RGBA16F rows of `opitch` texels (RGB rounded to nearest even, alpha Ā as it is: the interop's output copy); 2 dense RGB float32 `[H][opitch][3]` (no alpha: the runtime's own test buffer). Ā: the colour input's alpha of the frame through the 4.3 bilinear (the stored tap's fourth value RN16(alpha), the four taps and FP16 weights of Y / Co / Cg, RN16(w₀a₀) then three FP16 fused multiply-adds); spatial only, no exposure, no clamp | `k_out_pixel` | the front end | | exposure scalars `xs` | float32 [4]: κ, ρ, λ, RN(1/κ) of spec 42 section 4 / spec 43 3.7 (`KFrameIn.xs`; null = the struct's κ, ρ, λ and `KOutUni.kinv`) | `k_expo_scalars` (runtime) | all I/O kernels | | filter table | `[2048][8]` FP16, the frame's set (spec 43 section 7), generated on the host per level | host | `k_out_pixel` | | 2^E table | `[65536]` FP16: h(2^E) for every binary16 bit pattern E | host (once) | `k_out_pixel` | @@ -116,7 +116,7 @@ The per-frame runtime runs `KFrame`'s frame on its HIP stream (the interop's Fra | exposure | `exposure_auto_diet` (`exposure.co`, spec 07, HDR only, when the game asked for auto-exposure at creation or the front end asks this frame; `exposure_auto_diet_c16` on the RGBA16F copy), `k_expo_scalars` (`k_io.co`) | E_cur, E_prev, E_aux of spec 42 draft 3 section 4 (the game's value or texel as E_cur, else 1; the measured value as E_aux), then `xs` | | preparation | `k_out_color`, then `k_in_tiled` when supported; otherwise `k_in_color` plus `k_in_base`/`k_in_reset` | `KFrameIn.flags` = [HDR] from `WgRtConfig.hdr` (front end `WG_CREATE_HDR`), always [MVlo], [inv], reset | | network | the eleven `k_stage_*` / `k_stagew_*` launches of `k_net.py` variant "fused" | window offsets `k_stage_offsets(f, net/window_offsets)`; the G2 d0 also emits hidden history and, by default, the compact pixel head instead of the head map (`k_stage2_d0_hp`, `k_stage2h_d0_hp`) | -| output stage | `k_out_pixel_c` (compact pixel head, default) or `k_out_pixel` (runtime option `k_head=full`) | game output omode 1 (FrameIo.out_half_pitch > 0) or 2 (interop dense RGB) | +| output stage | `k_out_pixel_c` (compact pixel head, default) or `k_out_pixel` (runtime option `k_head=full`) | game output omode 1 (FrameIo.out_half_pitch > 0), 0 (FrameIo.out_rgba32f: dense RGBA f32, the interop's output pass) or 2 (dense RGB f32, tests); the direct forms write the output texture (alpha Ā in every form that has one) | `k_expo_scalars(st, src, measured, xs, KExpoArgs)` (one lane), spec 42 draft 3 section 4 (the host rule of `clean/k/reference/exposure.py`): [HDR] E_cur = `value` (`src_mode` 0: the front end's exposure value, 1 when the @@ -143,8 +143,9 @@ frame, raw), `exposure.measured` (the last auto-exposure value), `k.scalars` (xs (network output, the 40-channel head map: only with the runtime option `k_head=full`; the default frame writes the compact pixel head instead, buffer `k.Np`, and asking for `k.N` fails with a message naming the option; `k.Np` in turn fails with `k_head=full`), -`k.Hc`, `k.L`, `k.Hh` (the copies the next frame reads), `filter.out` (own output: dense RGB f32, -not with external I/O), `input.color`, `input.depth`, `input.motion`. Timing marks (FrameIo.timing_base, +`k.Hc`, `k.L`, `k.Hh` (the copies the next frame reads), `filter.out` (own output: dense RGB f32, or dense RGBA f32 +after a frame with `FrameIo.out_rgba32f`; not with external I/O), `input.color` (the fourth value is the colour's +alpha: `wg_rt_upload_inputs_rgba`; 1 after the RGB upload `wg_rt_upload_inputs`), `input.depth`, `input.motion`. Timing marks (FrameIo.timing_base, `wg_rt_frame_io_timed`): exposure = first kernel → `k_out_color`; input = preparation → first neural stage; network = eleven stages (including Hh in FP32 d0) → `k_out_pixel`; output = reconstruction and any standalone hidden pass → the final empty timing marker. Full end-to-end comparisons retain all these intervals. diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 63978019..480d2c73 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -30,7 +30,11 @@ and starts filled with its run's own sentinel (HIP `0xA5`, D3D12 `0x5A`); the de slack must keep its sentinel; special values are among the inputs; a negative control (the port given another input than HIP) must be caught; Model M's `scale_rgb` runs through the same replay as a control. Output: counts and verdicts. -Families: `narrow` (the narrow network stages of `k_block.co`: `k_stage_e2`, `k_stage_e0`, `k_stage_e1`, `k_stage_d2`, +Families: `input` (the exact tiled input stage `kx/k_in_tiled` of `k_io_exact.co`: 16 flag sets of HDR, low-resolution +motion, inverted depth and reset, odd and tiny base grids down to 1 x 1, the exposure scalars from a device buffer as the +runtime passes them, infinity, huge values, zero and -0 in the colour, depth, motion and histories, NaN in a third of +the cases; the raw bytes are identical in every case; the port needs the adapter's binary64 fma and division (manifest +`fp64: ext`, dropped on an adapter without them), and flushes FP32 denormals); `narrow` (the narrow network stages of `k_block.co`: `k_stage_e2`, `k_stage_e0`, `k_stage_e1`, `k_stage_d2`, `k_stage_d1`, `k_stage2_e0`, `k_stage2_e1`, `k_stage2_d1`, `k_stage2_d0_hp`, `k_stage2_d0_hidden`) on random stage weights and inputs, the stage runner's launch plans run on HIP and replayed on D3D12 (`k_plan_bridge.py`); the inputs of every third case carry special values (infinity, +-65504, zero, -0, subnormals) and of every third case NaN as well; the diff --git a/clean/k/dx12/gen_k_fp_tables.py b/clean/k/dx12/gen_k_fp_tables.py new file mode 100644 index 00000000..c02b76d3 --- /dev/null +++ b/clean/k/dx12/gen_k_fp_tables.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: MIT +"""Generates k_fp_tables.hlsli: the constants of model K's exact FP32 / FP64 helpers (k_fp.h) as bit +patterns for the D3D12 port (k_fp.hlsli). WrongGPU's own code. + +k_fp.h builds its tables at compile time (constexpr binary64 series KfExpTab / KfLogTab) and folds constants such as +1.0 / 23.0 or (float)(1.0 / 3.0). HLSL has no constexpr evaluation in binary64 and its float literals are parsed by +DXC, so the port takes every such value as bits computed here with the same IEEE operations in the same order (Python +floats are binary64 with round to nearest even; numpy.float32 rounds binary64 to binary32 to nearest even, as the C +casts do). clean/dx12/build_dx12.py runs it into /dx12/gen (an include folder of the K ports), so the tables are +computed at every build and never stored in the repository: python clean/k/dx12/gen_k_fp_tables.py +""" +import os +import struct +import sys + +import numpy as np + + + +def f32(v): + return struct.unpack("> 32 + + +def kc_exp(x): # k_fp.h kc_exp, |x| <= 0.4 + t, s = 1.0, 1.0 + for k in range(1, 30): + t = t * x / k + s = s + t + return s + + +def kc_log(x): # k_fp.h kc_log, x in [0.5, 2] + s = (x - 1.0) / (x + 1.0) + s2 = s * s + t, r = s, 0.0 + for k in range(40): + r = r + t / (2 * k + 1) + t = t * s2 + return 2.0 * r + + +KC_LN2 = 0.69314718055994530942 + + +def main(out_path): + exp_h, exp_l = [], [] + for i in range(65): # KfExpTab: 2^(j/64), j = -32 .. 32 + v = kc_exp((i - 32) / 64.0 * KC_LN2) + h = float(np.float32(v)) + exp_h.append(f32(h)) + exp_l.append(f32(v - h)) + log_inv, log_h, log_l = [], [], [] + for i in range(97): # KfLogTab: c_j = 1 + j/128, inv_j = RN32(1/c_j), -log2(inv_j) as hi + lo + c = 1.0 + (i - 32) / 128.0 + inv = float(np.float32(1.0 / c)) + log_inv.append(f32(inv)) + v = -kc_log(inv) / KC_LN2 + h = float(np.float32(v)) + log_h.append(f32(h)) + log_l.append(f32(v - h)) + + def arr(name, vals): + rows = [", ".join(f"0x{v:08X}u" for v in vals[i:i + 8]) for i in range(0, len(vals), 8)] + return f"static const uint {name}[{len(vals)}] = {{\n " + ",\n ".join(rows) + "\n};\n" + + def dconst(name, v): + lo, hi = f64(v) + return f"#define {name} asdouble(0x{lo:08X}u, 0x{hi:08X}u) // {v!r}\n" + + def fconst(name, v): + return f"#define {name} asfloat(0x{f32(v):08X}u) // {float(np.float32(v))!r}\n" + + third = 1.0 / 3.0 + third_h = float(np.float32(third)) + inv_ln2 = 1.4426950408889634074 + ln2 = 0.69314718055994530942 + out = ["// Generated by clean/k/dx12/gen_k_fp_tables.py at build time: the constants of k_fp.h as bits. WrongGPU's own code.\n", + "#pragma once\n\n", + "// KfExpTab: 2^(j/64), j = -32 .. 32, hi + lo\n", arr("KF_EXP_H", exp_h), arr("KF_EXP_L", exp_l), + "// KfLogTab: inv_j = RN32(1 / (1 + j/128)), -log2(inv_j) as hi + lo, j = -32 .. 64\n", + arr("KF_LOG_INV", log_inv), arr("KF_LOG_H", log_h), arr("KF_LOG_L", log_l), "\n", + "// binary64 constants of kd_log_core / kd_log / kd_log2 / kd_exp2 (k_fp.h; folded there at compile time)\n", + dconst("KD_LN2", 0.6931471805599453094), dconst("KD_INV_LN2", 1.4426950408889634074), + dconst("KD_SQRT2", 1.4142135623730951), dconst("KD_INV23", 1.0 / 23.0)] + out.append("// 1 / (2k + 1), k = 0 .. 10 (kd_log_core's series)\n") + out.append("static const uint2 KD_INV_ODD[11] = {\n " + ",\n ".join( + "uint2(0x%08Xu, 0x%08Xu)" % f64(1.0 / (2 * k + 1)) for k in range(11)) + "\n};\n") + out.append("// 1 / k, k = 0 .. 16 (kd_exp2's series; entry 0 unused)\n") + out.append("static const uint2 KD_INV_K[17] = {\n uint2(0u, 0u),\n " + ",\n ".join( + "uint2(0x%08Xu, 0x%08Xu)" % f64(1.0 / k) for k in range(1, 17)) + "\n};\n\n") + out.append("// binary32 constants of kf2_log2 / kf2_exp2 / kf_ln (k_fp.h)\n") + out += [fconst("KF_THIRD_H", third), fconst("KF_THIRD_L", third - third_h), + fconst("KF_SIXTH", np.float32(1.0) / np.float32(6.0)), fconst("KF_1_120", np.float32(1.0) / np.float32(120.0)), + fconst("KF_1_24", np.float32(1.0) / np.float32(24.0)), fconst("KF_1_64", np.float32(1.0) / np.float32(64.0)), + fconst("KF2_INV_LN2_H", inv_ln2), fconst("KF2_INV_LN2_L", inv_ln2 - float(np.float32(inv_ln2))), + fconst("KF2_LN2_H", ln2), fconst("KF2_LN2_L", ln2 - float(np.float32(ln2)))] + os.makedirs(os.path.dirname(os.path.abspath(out_path)), exist_ok=True) + with open(out_path, "w", newline="\n") as f: + f.write("".join(out)) + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/clean/k/dx12/hlsl/k_fp.hlsli b/clean/k/dx12/hlsl/k_fp.hlsli new file mode 100644 index 00000000..6cbbcb97 --- /dev/null +++ b/clean/k/dx12/hlsl/k_fp.hlsli @@ -0,0 +1,393 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's exact scalar FP32 / FP16 / FP64 helpers (clean/k/kernels/k_fp.h; specs 42, 43). WrongGPU's own +// code. Same names and the same bits as the HIP helpers on gfx1201, in the exact arithmetic mode of build_dx12.py +// (clean/dx12/hlsl/wg_math.hlsli: one IEEE operation per written operation, mad() = one v_fma_f32, rcp() = v_rcp_f32, +// no float '/': wg_div is HIP's correctly rounded a / b). +// +// Include after wg_platform.hlsli (wg_math.hlsli) and, in a kernel, call KF_SETUP() right after WG_SETUP(). +// +// Representation: an FP16 value (HIP h16) is a float holding that value exactly (f16tof32 / wg_h2f of the bits); +// kh_round(x) is HIP's (h16) rounding of an FP32 value (round to nearest even, v_cvt_f16_f32: wg_f2h; the driver's +// (half) / f32tof16 round toward zero and are never used), kh_bits(x) the bits of such a value for a store. FP16 +// arithmetic is done in FP32 and rounded once: a sum, difference or product of two FP16 values rounded to FP32 and then +// to FP16 is the FP16 operation's result (products are exact in FP32; for + and - double rounding is innocuous since +// 24 >= 2 * 11 + 2, Figueroa 1995), kh_fma is wg_fma_f16 (one rounding), kh_div as k_fp.h's argument. This also avoids +// the driver's fusion of half multiply + add (wg_math.hlsli "driver exceptions"). +// +// Binary64 (kd_*): add / sub / mul / compare / convert and fma (the D3D11.1 extended doubles, which D3D12 includes in +// D3D12_FEATURE_DATA_D3D12_OPTIONS.DoublePrecisionFloatShaderOps): build_dx12.py marks ports with binary64 fma or +// division "fp64": "ext" and the devices refuse them on an adapter without doubles. HLSL has no binary64 sqrt / rint / +// constexpr: the constants and tables of k_fp.h come as bits +// from k_fp_tables.hlsli (clean/k/dx12/gen_k_fp_tables.py: the same binary64 series and roundings), kd_exp2's rint is +// done in FP32 (its argument is always a binary32 value, see there), and kf_rsqrt's binary64 fallback is replaced by an +// exact power-of-four scaling (see there). +// +// Opaque barriers (kf_opq): the gfx1201 D3D12 shader compiler may contract a product into a following addition or +// subtraction (one fma: one rounding instead of HIP's two; seen in the first port check of the input stage, about one +// FP16 value in 10^5 differing by one ulp), regroup chains of multiplications by constant / uniform / shared factors, +// and fold a product into a later fused operation (wg_math.hlsli). Wherever HIP rounds a product that then meets an +// addition (and the product is not exact, e.g. a power-of-two scaling), the product goes through kf_opq (an XOR with a +// zero the compiler cannot prove: wg_grid_ >> 31, the launch's workgroup count is below 2^31; one VALU operation). +// kf_exp2 applies it to its argument itself (its callers pass products). +// +// Differences from HIP (as for every D3D12 port, wg_math.hlsli): FP32 denormals are flushed to zero here (HIP keeps +// them); none of the input stage's values is an FP32 denormal for the frames the port checks run (clean/k/dx12/ +// k_port_check.py), and FP16 denormals are kept as in HIP. +#pragma once + +#include "k_fp_tables.hlsli" + +static uint kf_opq_z_; // 0, opaque to the compiler (KF_SETUP) +#define KF_SETUP() (kf_opq_z_ = wg_grid_ >> 31) +float kf_opq(float x) { return asfloat(asuint(x) ^ kf_opq_z_); } +float2 kf_opq2(float2 x) { return asfloat(asuint(x) ^ kf_opq_z_); } +float4 kf_opq4(float4 x) { return asfloat(asuint(x) ^ kf_opq_z_); } + +// ---- elementary steps ----------------------------------------------------------------------------------------------- + +float kf_fma(float a, float b, float c) { return mad(a, b, c); } // v_fma_f32 +float kf_bits(uint b) { return asfloat(b); } +float kf_floor(float x) { return floor(x); } +float kf_trunc(float x) { return trunc(x); } +float kf_rni(float x) { return round(x); } // v_rndne_f32 (ties to even) +// max / min of specs 42 / 43: IEEE maxNum / minNum (v_max_num_f32 / v_min_num_f32), as __builtin_fmaxf / fminf +float kf_max(float a, float b) { return max(a, b); } +float kf_min(float a, float b) { return min(a, b); } +int ki_min(int a, int b) { return a < b ? a : b; } +int ki_max(int a, int b) { return a > b ? a : b; } +// HIP's x - x == 0 test as bits: the D3D12 shader compiler folds x - x to 0 (seen for float and double: an infinite +// input passed as finite in the first probe, clean/k/dx12/probe/kfp_check.py) +bool kf_finite(float x) { return (asuint(x) & 0x7f800000u) != 0x7f800000u; } +bool kd_isnan(double x) { uint lo, hi; asuint(x, lo, hi); return (hi & 0x7ff00000u) == 0x7ff00000u && ((hi & 0xfffffu) | lo) != 0u; } +bool kd_finite(double x) { uint lo, hi; asuint(x, lo, hi); return (hi & 0x7ff00000u) != 0x7ff00000u; } + +// wg_div (HIP's correctly rounded a / b) with its common case first: both operands normal and finite and the quotient's +// exponent in range, where wg_div's result is the scaled quotient q with the exponent difference added to its exponent +// field (the same q: the same operations as wg_div); every other operand pair takes wg_div itself in a cold block. Equal +// to wg_div for every input (probe: build\k-d3d12\L2work\probe\kdiv_check.py); about half of wg_div's instructions and none of +// its selects on the common path. +float kf_div(float a, float b) +{ + const uint ua = asuint(a), ub = asuint(b); + const uint ea = (ua >> 23) & 255u, eb = (ub >> 23) & 255u; + const float as = asfloat((ua & 0x807fffffu) | 0x3f800000u), bs = asfloat((ub & 0x807fffffu) | 0x3f800000u); + float y = rcp(bs); + const float e = mad(-bs, y, 1.0f); + y = mad(e, y, y); + float q = as * y; + float r = mad(-bs, q, as); + q = mad(r, y, q); + r = mad(-bs, q, as); + q = mad(r, y, q); + const int d = (int)ea - (int)eb; + const uint en = ((asuint(q) >> 23) & 255u) + (uint)d; + float res; + [branch] if (ea - 1u < 254u && eb - 1u < 254u && en - 1u < 254u) + res = asfloat(asuint(q) + ((uint)d << 23)); + else + res = wg_div(a, b); + return res; +} + +// R(t): round half away from zero (specs 42 / 43), exact for every binary32 t +float kf_R(float t) +{ + const float a = abs(t); + const float fl = floor(a); + const float r = (a - fl) >= 0.5f ? fl + 1.0f : fl; // a - fl is exact + return t < 0.0f ? -r : r; +} +float kf_sat(float x) { return isnan(x) ? 0.0f : (x < 0.0f ? 0.0f : (x > 1.0f ? 1.0f : x)); } + +// ---- FP16 values (floats holding FP16 values exactly) --------------------------------------------------------------- + +uint kh_bits(float h) { return wg_f2h(h); } // exact for an FP16 value (NaN payload as v_cvt_f16_f32) +float kh_round(float x) { return wg_h2f(wg_f2h(x)); } // HIP kh_round: (h16) x, nearest even +uint kh_round_bits(float x) { return wg_f2h(x); } // the bits of kh_round(x) +float kh_add(float a, float b) { return kh_round(a + b); } +float kh_sub(float a, float b) { return kh_round(a - b); } +float kh_mul(float a, float b) { return kh_round(a * b); } +float kh_fma(float a, float b, float c) { return wg_h2f(wg_fma_f16(a, b, c)); } // v_fma_f16: one rounding +// a / b of FP16 values rounded once to FP16: the correctly rounded FP32 quotient rounded to FP16 (k_fp.h kh_div) +float kh_div(float a, float b) { return kh_round(wg_div(a, b)); } +float kh_floor(float x) { return floor(x); } // exact on FP16 values +float kh_rni(float x) { return round(x); } +float kh_sat(float x) { return isnan(x) ? 0.0f : (x < 0.0f ? 0.0f : (x > 1.0f ? 1.0f : x)); } +float kh_maxnum(float a, float b) { return max(a, b); } + +// packed FP16 (two independent values; each element exactly as the scalar helper, as HIP's v_pk_* do) +float2 kh2(float a, float b) { return float2(a, b); } +float2 kh2_splat(float a) { return float2(a, a); } +float2 kh2_add(float2 a, float2 b) { return float2(kh_add(a.x, b.x), kh_add(a.y, b.y)); } +float2 kh2_sub(float2 a, float2 b) { return float2(kh_sub(a.x, b.x), kh_sub(a.y, b.y)); } +float2 kh2_mul(float2 a, float2 b) { return float2(kh_mul(a.x, b.x), kh_mul(a.y, b.y)); } +float2 kh2_fma(float2 a, float2 b, float2 c) { return float2(kh_fma(a.x, b.x, c.x), kh_fma(a.y, b.y, c.y)); } +float2 kh2_abs(float2 a) { return abs(a); } +float2 kh2_rni(float2 a) { return round(a); } +float2 kh2_maxnum(float2 a, float2 b) { return max(a, b); } +// k_fp.h kh2_sat_nz: maxNum(x, 0) then minNum(., 1) per element (callers never pass -0) +float2 kh2_sat_nz(float2 a) { return min(max(a, 0.0f), 1.0f); } +// k_fp.h kh2_gtmask as a bool2: true where h(b - a) is negative and not NaN (IEEE a > b, except a = +0, b = -0: true) +bool2 kh2_gtmask(float2 a, float2 b) +{ + const float2 d = float2(kh_sub(b.x, a.x), kh_sub(b.y, a.y)); + return bool2((asuint(d.x) >> 31) != 0u && !isnan(d.x), (asuint(d.y) >> 31) != 0u && !isnan(d.y)); +} +float2 kh2_bsel(bool2 m, float2 a, float2 b) { return float2(m.x ? a.x : b.x, m.y ? a.y : b.y); } + +// ---- double-precision functions -------------------------------------------------------------------------------------- + +double kd_fma(double a, double b, double c) { return fma(a, b, c); } // extended doubles +double kd_from_bits(uint lo, uint hi) { return asdouble(lo, hi); } + +// 1 / d for d in [1.7, 2.5]: the FP32 reciprocal estimate (v_rcp_f32 of RN32(d), as HIP) and two Newton steps +double kd_rcp_near2(double d) +{ + double q = (double)rcp((float)d); + q = kd_fma(q, kd_fma(-d, q, 1.0L), q); + q = kd_fma(q, kd_fma(-d, q, 1.0L), q); + return q; +} + +// k_fp.h kd_log_core: x = 2^e m, m in (1/sqrt 2, sqrt 2], ln m = 2 s (1 + s^2/3 + ... + s^22/23), s = (m - 1)/(m + 1) +double kd_log_core(double x, out int e_out) +{ + uint lo, hi; + asuint(x, lo, hi); + int e = (int)((hi >> 20) & 0x7ffu) - 1023; + double m = asdouble(lo, (hi & 0x000fffffu) | 0x3ff00000u); // [1, 2) + if (m > KD_SQRT2) { + m = m * 0.5L; + e += 1; + } + const double s = (m - 1.0L) * kd_rcp_near2(m + 1.0L); // m - 1 and m + 1 are exact + const double s2 = s * s; + double p = KD_INV23; + [unroll] for (int k = 10; k >= 0; --k) p = kd_fma(p, s2, asdouble(KD_INV_ODD[k].x, KD_INV_ODD[k].y)); + e_out = e; + return 2.0L * s * p; +} + +double kd_log_special(double x, out bool special) +{ + special = true; + if (kd_isnan(x) || x < 0.0L) return asdouble(0u, 0x7ff80000u); + if (x == 0.0L) return -asdouble(0u, 0x7ff00000u); + if (!kd_finite(x)) return x; // +inf + special = false; + return 0.0L; +} + +// ln x +double kd_log(double x) +{ + bool sp; + const double v = kd_log_special(x, sp); + if (sp) return v; + int e; + const double l = kd_log_core(x, e); + return kd_fma((double)e, KD_LN2, l); +} + +// log2 x = e + ln(m) / ln 2 +double kd_log2(double x) +{ + bool sp; + const double v = kd_log_special(x, sp); + if (sp) return v; + int e; + const double l = kd_log_core(x, e); + return kd_fma(l, KD_INV_LN2, (double)e); +} + +double kd_ldexp(double v, int n) // v * 2^n, n in [-2044, 2046] +{ + [unroll] for (int i = 0; i < 2; ++i) { + const int k = n > 1023 ? 1023 : (n < -1022 ? -1022 : n); + v = v * asdouble(0u, (uint)(k + 1023) << 20); + n -= k; + } + return v; +} + +// 2^y. Precondition: y is a binary32 value (every caller passes (double) of a float: kf_exp2), so rint(y) is the FP32 +// v_rndne of (float) y, which is exact (HLSL has no binary64 rint). +double kd_exp2(double y) +{ + if (kd_isnan(y)) return y; + if (y > 1100.0L) return asdouble(0u, 0x7ff00000u); + if (y < -1100.0L) return 0.0L; + const double n = (double)round((float)y); + const double r = (y - n) * KD_LN2; // |r| <= 0.347 + double p = 1.0L; + [unroll] for (int k = 16; k >= 1; --k) p = kd_fma(p, r * asdouble(KD_INV_K[k].x, KD_INV_K[k].y), 1.0L); + return kd_ldexp(p, (int)n); +} + +// 1/sqrt(x) rounded once to FP32 (spec 43 6.8). HIP: for x >= 1 the hardware estimate v_rsq_f32 and exact rounding +// tests in binary64 (as here); otherwise kd_rsqrt32 = (float)(1.0 / sqrt((double)x)), the correctly rounded value. +// HLSL has no binary64 sqrt: a normal x < 1 is scaled by 4^k into [1, 4) (exact), the tested result scaled back by 2^k +// (exact: the result is normal), which is the correctly rounded value too. Zero (and FP32 denormals, flushed) -> +-inf +// as 1 / +-0, +inf -> 0, NaN -> quieted, negative -> the canonical NaN. Port check: the output stage's (G2). +float kf_rsqrt_ge1(float x) +{ + float y = rsqrt(x); // v_rsq_f32 + const double xd = (double)x; + [unroll] for (int it = 0; it < 3; ++it) { + const uint b = asuint(y); + const double yd = (double)y; + const double ulp = (double)asfloat((b & 0x7F800000u) - (23u << 23)); + const double up = yd + 0.5L * ulp; + const double lo = yd - ((b & 0x7FFFFFu) == 0u ? 0.25L : 0.5L) * ulp; // below a power of two the spacing halves + if (kd_fma(up * up, xd, -1.0L) < 0.0L) { + y = asfloat(b + 1u); + } else if (kd_fma(lo * lo, xd, -1.0L) > 0.0L) { + y = asfloat(b - 1u); + } else { + return y; + } + } + return y; +} +float kf_rsqrt(float x) +{ + if (x >= 1.0f && kf_finite(x)) return kf_rsqrt_ge1(x); + if (isnan(x)) return asfloat(asuint(x) | 0x400000u); + if (x == 0.0f) return asfloat((asuint(x) & 0x80000000u) | 0x7f800000u); + if (x < 0.0f) return asfloat(0x7fc00000u); + if (!kf_finite(x)) return 0.0f; // +inf + const int e = (int)((asuint(x) >> 23) & 255u) - 127; // x normal in [2^-126, 1): e in [-126, -1] + const int k = (1 - e) >> 1; // 4^k x in [1, 4) + const float xs = x * asfloat((uint)(2 * k + 127) << 23); + return kf_rsqrt_ge1(xs) * asfloat((uint)(k + 127) << 23); +} + +// ---- correctly rounded binary32 log2 / ln / 2^x in FP32 double-word arithmetic (k_fp.h: Ziv test, else binary64) ---- + +struct kf2 { float h, l; }; +kf2 kf2_make(float h, float l) { kf2 r; r.h = h; r.l = l; return r; } + +// Error-free transformations: the sums go through kf_opq, else the D3D12 compiler simplifies (a + b) - a to b and the +// like (the first probe found the double-word results off by one ulp in 16 % of 2^x and 0.1 % of log2 inputs) +kf2 kf2_two_sum(float a, float b) +{ + const float s = kf_opq(a + b); + const float bb = kf_opq(s - a); + return kf2_make(s, (a - (s - bb)) + (b - bb)); +} +kf2 kf2_fast_two_sum(float a, float b) // |a| >= |b| or a == 0 +{ + const float s = kf_opq(a + b); + return kf2_make(s, b - (s - a)); +} +kf2 kf2_two_prod(float a, float b) +{ + const float p = kf_opq(a * b); + return kf2_make(p, kf_fma(a, b, -p)); +} +kf2 kf2_add(kf2 a, kf2 b) +{ + kf2 s = kf2_two_sum(a.h, b.h); + s.l = s.l + (a.l + b.l); + return kf2_fast_two_sum(s.h, s.l); +} +kf2 kf2_mul(kf2 a, kf2 b) +{ + kf2 p = kf2_two_prod(a.h, b.h); + p.l = kf_fma(a.h, b.l, kf_fma(a.l, b.h, p.l)); + return kf2_fast_two_sum(p.h, p.l); +} + +// Ziv test: is RN32(v) = r.h for every v with |v - (r.h + r.l)| <= 2^-38 |r.h| ? +bool kf2_round_ok(kf2 r) +{ + const uint b = asuint(r.h); + const uint ex = (b >> 23) & 0xFFu; + if (ex - 40u > 214u) return false; // ex < 40 or 255: tiny, subnormal, inf / NaN: exact path + // half = 0.5 ulp, or 0.25 ulp for a power of two whose low part points toward zero (below it the spacing halves). + // Bit form: (sign(r.l) xor sign(r.h)) and (mantissa - 1 borrows), bit 31. It differs from (r.l < 0) != (r.h < 0) only + // for r.l = -0, where |r.l| + err = err < 2^-37 |r.h| is below either half: the same answer. + const uint s = ((asuint(r.l) ^ b) & ((b & 0x7FFFFFu) - 1u)) >> 31; + const float half = asfloat((ex - 24u - s) << 23); + const float err = abs(r.h) * asfloat(0x2C800000u); // 2^-38 + return abs(r.l) + err < half; +} + +// log2 x as hi + lo for finite normal x > 0 +kf2 kf2_log2(float x) +{ + const uint b = asuint(x); + int e = (int)((b >> 23) & 0xFFu) - 127; + float m = asfloat((b & 0x7FFFFFu) | 0x3F800000u); // [1, 2) + if (m >= 1.5f) { + m = m * 0.5f; + e += 1; + } + const int j = (int)round((m - 1.0f) * 128.0f); // -32 .. 64 + const float inv = asfloat(KF_LOG_INV[j + 32]); + const kf2 p = kf2_two_prod(m, inv); // m inv = 1 + t exactly + const kf2 t = kf2_two_sum(p.h - 1.0f, p.l); // p.h - 1 is exact + // ln(1 + t) = t (1 - t/2 + t^2/3 - t^3 (1/4 - t/5 + t^2/6)), |t| < 0.0053 + const float th = t.h; + const float q = -kf_fma(th, kf_fma(th, KF_SIXTH, -0.2f), 0.25f); + kf2 P = kf2_add(kf2_make(KF_THIRD_H, KF_THIRD_L), kf2_two_prod(th, q)); + P = kf2_add(kf2_make(-0.5f, 0.0f), kf2_mul(t, P)); + P = kf2_add(kf2_make(1.0f, 0.0f), kf2_mul(t, P)); + const kf2 ln1 = kf2_mul(t, P); + kf2 r = kf2_mul(ln1, kf2_make(KF2_INV_LN2_H, KF2_INV_LN2_L)); + r = kf2_add(r, kf2_make(asfloat(KF_LOG_H[j + 32]), asfloat(KF_LOG_L[j + 32]))); + return kf2_add(kf2_make((float)e, 0.0f), r); +} + +// 2^y as hi + lo for y in [-125, 127) +kf2 kf2_exp2(float y) +{ + const float n = round(y); + const float f = y - n; // exact, |f| <= 0.5 + const float jf = round(f * 64.0f); + const float r = f - jf * KF_1_64; // exact, |r| <= 1/128 + const kf2 x = kf2_mul(kf2_make(r, 0.0f), kf2_make(KF2_LN2_H, KF2_LN2_L)); // |x| < 0.0055 + // e^x = 1 + x + x^2/2 + x^3/6 + x^4/24 + x^5/120; HIP's order ((xh xh) xh) poly, each product rounded + const float xh = x.h; + const float x2 = kf_opq(xh * xh), x3 = kf_opq(x2 * xh); + const float tail = kf_opq(x3 * kf_fma(xh, kf_fma(xh, KF_1_120, KF_1_24), KF_SIXTH)); // not fused into q.h + tail + kf2 q = kf2_two_prod(xh, xh); + q.h = q.h * 0.5f; + q.l = kf_fma(xh, x.l, q.l * 0.5f); + kf2 s = kf2_add(kf2_make(1.0f, 0.0f), x); + s = kf2_add(s, kf2_add(q, kf2_make(tail, 0.0f))); + const int j = (int)jf + 32; + kf2 v = kf2_mul(kf2_make(asfloat(KF_EXP_H[j]), asfloat(KF_EXP_L[j])), s); + const float sc = asfloat((uint)((int)n + 127) << 23); // 2^n, exact (n in [-125, 127]) + v.h = v.h * sc; + v.l = v.l * sc; + return v; +} + +// The binary32 functions of specs 42 / 43: the double-word path, else the binary64 path above. +float kf_log2(float x) +{ + if (asuint(x) - 0x00800000u < 0x7f000000u) { // positive, normal, finite (bits 0x00800000 .. 0x7f7fffff) + const kf2 r = kf2_log2(x); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_log2((double)x); +} +float kf_ln(float x) +{ + if (asuint(x) - 0x00800000u < 0x7f000000u) { + const kf2 r = kf2_mul(kf2_log2(x), kf2_make(KF2_LN2_H, KF2_LN2_L)); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_log((double)x); +} +float kf_exp2(float y) +{ + // callers pass products (log2(z) c, e 9 c) that HIP rounds before kf2_exp2's y - rint(y); the gfx1201 ISA keeps + // them apart without a kf_opq here (G1's precaution link, dropped in speed round 1 after the ISA check) + if (y >= -125.0f && y < 127.0f) { + const kf2 r = kf2_exp2(y); + if (kf2_round_ok(r)) return r.h; + } + return (float)kd_exp2((double)y); +} diff --git a/clean/k/dx12/hlsl/k_frame.hlsli b/clean/k/dx12/hlsl/k_frame.hlsli new file mode 100644 index 00000000..fe37800a --- /dev/null +++ b/clean/k/dx12/hlsl/k_frame.hlsli @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's frame inputs and their fetch helpers (clean/k/kernels/k_io.hip: KFrameIn, KInParams, +// KOutParams, KiHost, kio_*). WrongGPU's own code; the same operations and bits as the HIP helpers (k_fp.hlsli rules). +// +// Arguments. The HIP kernels take the structs by value with 64-bit pointers inside; the D3D12 ports get them as root +// constants packed by the device from the manifest's struct tokens (clean/runtime/struct_args.h): every pointer one +// DWORD (the arena byte offset, 0 = null), byte runs copied, skipped runs dropped. The macros below are the matching +// argument-block declarations (between WG_ARGS_BEGIN and WG_ARGS_END, in this order) and the loaders that rebuild the +// kernel-side view: +// KInParams s200:p4.b128.p5 K_ARGS_FRAME_IN K_ARGS_IN_PTRS 41 DWORDs +// KInUniformParams s248:p4.b128.p5.b48 K_ARGS_FRAME_IN K_ARGS_IN_PTRS K_ARGS_HOST 53 +// KOutParams s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 K_ARGS_OUT 52 +// (skipped: cbx..zby, always 0 in the runtime (runtime_k.cpp sets no subrect base for the output stage); kappa / rho +// / lam, which the kernels replace by KFrameIn.xs (the runtime always passes xs); pad_; exp2tab, the emulator's +// table only: on the GPU ko_exp2h is v_exp_f16. K_LOAD_FRAME_OUT sets the skipped fields to 0 / 1.) +// Field names: F_ (KFrameIn), P_ (KInParams pointers), H_ (KiHost), O_ (KOutParams), U_ (KOutUni). +// +// Include after wg_platform.hlsli, the argument block and k_fp.hlsli. With WG_DIRECT_TEXTURES (rs=direct ports) the +// colour, depth and motion fetches read the game's textures t0 / t1 / t2 of clean/dx12/hlsl/native_direct.hlsli +// (include it first) instead of the arena (kio_texel / kio_motion / kio_depth; see there). +#pragma once + +#define KIO_HDR 1 +#define KIO_MVLO 2 +#define KIO_INV 4 +#define KIO_RESET 8 +#define KIO_C16 16 +#define KIO_M16 32 +#define KIO_OUT_RGBA32F 0 +#define KIO_OUT_RGBA16F 1 +#define KIO_OUT_RGB32F 2 + +// ---- argument blocks (root constants, DWORD order = the struct tokens) -------------------------------------------- +#define K_ARGS_FRAME_PTRS uint F_color, F_motion, F_depth, F_xs; +#define K_ARGS_FRAME_IN K_ARGS_FRAME_PTRS \ + int F_cpitch, F_mpitch, F_zpitch, F_cbx, F_cby, F_mbx, F_mby, F_zbx, F_zby, F_W, F_H, F_rw, F_rh, F_Bw, F_Bh, F_flags; \ + float F_jx, F_jy, F_sx, F_sy, F_mox, F_moy, F_kappa, F_rho, F_lam, F_s; \ + int F_mch; \ + float F_rmsx, F_rmsy, F_rmox, F_rmoy; \ + int F_pad_; +#define K_ARGS_IN_PTRS uint P_Hc, P_Hh, P_x0, P_cpre, P_cpre2; +#define K_ARGS_HOST float H_xi_x, H_xi_y, H_a_x, H_a_y, H_h_x, H_h_y, H_g_x, H_g_y, H_sp_x, H_sp_y, H_mp_x, H_mp_y; +#define K_ARGS_OUT K_ARGS_FRAME_PTRS \ + int F_cpitch, F_mpitch, F_zpitch; \ + int F_W, F_H, F_rw, F_rh, F_Bw, F_Bh, F_flags; \ + float F_jx, F_jy, F_sx, F_sy, F_mox, F_moy; \ + float F_s; \ + int F_mch; \ + float F_rmsx, F_rmsy, F_rmox, F_rmoy; \ + uint O_N, O_L, O_Hc, O_table; \ + uint O_out, O_Lout, O_Hcout, O_Hhout; \ + int O_opitch, O_omode; \ + uint O_ctap, O_ctapT; \ + float U_invW, U_invH, U_rhox, U_rhoy, U_rho2x, U_rho2y, U_jmx, U_jmy, U_spx, U_spy, U_invW16, U_invH16, U_kinv; \ + uint U_kux_kvy; + +// ---- kernel-side structs -------------------------------------------------------------------------------------------- +struct KFrameIn { + uint color, motion, depth, xs; // arena byte offsets (0 = null) + int cpitch, mpitch, zpitch; + int cbx, cby, mbx, mby, zbx, zby; // subrect bases + int W, H, rw, rh, Bw, Bh; // output, render, base sizes + int flags; // KIO_* + float jx, jy, sx, sy, mox, moy; // jitter, motion scale, motion offset + float kappa, rho, lam; // replaced by xs[0..2] when xs is set (kio_frame) + float s; // spec 44's base-size factor (draft 3: 1/2) + int mch; // KIO_M16: FP16 channels per motion texel + float rmsx, rmsy, rmox, rmoy; // KIO_M16: ms, mo of the raw motion +}; +struct KInParams { + KFrameIn f; + uint Hc, Hh, x0, cpre, cpre2; +}; +struct KiHost { // spec 42 section 8 + float xi_x, xi_y, a_x, a_y, h_x, h_y, g_x, g_y, sp_x, sp_y, mp_x, mp_y; +}; + +#define K_LOAD_FRAME_PTRS(F) F.color = F_color; F.motion = F_motion; F.depth = F_depth; F.xs = F_xs; +#define K_LOAD_FRAME_COMMON(F) F.cpitch = F_cpitch; F.mpitch = F_mpitch; F.zpitch = F_zpitch; \ + F.W = F_W; F.H = F_H; F.rw = F_rw; F.rh = F_rh; F.Bw = F_Bw; F.Bh = F_Bh; F.flags = F_flags; \ + F.jx = F_jx; F.jy = F_jy; F.sx = F_sx; F.sy = F_sy; F.mox = F_mox; F.moy = F_moy; F.s = F_s; F.mch = F_mch; \ + F.rmsx = F_rmsx; F.rmsy = F_rmsy; F.rmox = F_rmox; F.rmoy = F_rmoy; +// KFrameIn from K_ARGS_FRAME_IN +#define K_LOAD_FRAME_IN(F) K_LOAD_FRAME_PTRS(F) K_LOAD_FRAME_COMMON(F) \ + F.cbx = F_cbx; F.cby = F_cby; F.mbx = F_mbx; F.mby = F_mby; F.zbx = F_zbx; F.zby = F_zby; \ + F.kappa = F_kappa; F.rho = F_rho; F.lam = F_lam; +// KFrameIn from K_ARGS_OUT (the skipped fields as the runtime passes them: bases 0, scalars 1 before kio_frame) +#define K_LOAD_FRAME_OUT(F) K_LOAD_FRAME_PTRS(F) K_LOAD_FRAME_COMMON(F) \ + F.cbx = 0; F.cby = 0; F.mbx = 0; F.mby = 0; F.zbx = 0; F.zby = 0; F.kappa = 1.0f; F.rho = 1.0f; F.lam = 1.0f; +#define K_LOAD_IN_PARAMS(P) K_LOAD_FRAME_IN(P.f) P.Hc = P_Hc; P.Hh = P_Hh; P.x0 = P_x0; P.cpre = P_cpre; P.cpre2 = P_cpre2; +#define K_LOAD_HOST(k) k.xi_x = H_xi_x; k.xi_y = H_xi_y; k.a_x = H_a_x; k.a_y = H_a_y; k.h_x = H_h_x; k.h_y = H_h_y; \ + k.g_x = H_g_x; k.g_y = H_g_y; k.sp_x = H_sp_x; k.sp_y = H_sp_y; k.mp_x = H_mp_x; k.mp_y = H_mp_y; + +int kio_ceil16(int n) { return (n + 15) / 16 * 16; } + +// The kernels' view: with KFrameIn.xs the exposure scalars come from the GPU (k_expo_scalars) +KFrameIn kio_frame(KFrameIn f) +{ + KFrameIn F = f; + if (f.xs != 0u) { + F.kappa = wg_ldf(f.xs, 0u); + F.rho = wg_ldf(f.xs, 4u); + F.lam = wg_ldf(f.xs, 8u); + } + return F; +} +KInParams ki_params(KInParams p) +{ + KInParams P = p; + P.f = kio_frame(p.f); + return P; +} + +// spec 42 section 8, computed on the GPU (k_in_tiled; k_in_tiled_uniform takes the host's equal values) +KiHost ki_host(KFrameIn F) +{ + KiHost k; + const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); + k.xi_x = wg_div(1.0f, (float)F.W * F.s) * 0.5f; // RN32(RN32(1/RN32(W s)) 0.5) + k.xi_y = wg_div(1.0f, (float)F.H * F.s) * 0.5f; + k.a_x = (float)F.rw * k.xi_x; + k.a_y = (float)F.rh * k.xi_y; + k.h_x = wg_div((float)F.W, (float)W16) * k.xi_x; + k.h_y = wg_div((float)F.H, (float)H16) * k.xi_y; + k.g_x = wg_div(0.25f, (float)F.Bw); + k.g_y = wg_div(0.25f, (float)F.Bh); + const bool mvlo = (F.flags & KIO_MVLO) != 0; + k.sp_x = wg_div(F.sx, wg_div(1.0f, (float)F.W) * (float)(mvlo ? F.rw : F.W)); + k.sp_y = wg_div(F.sy, wg_div(1.0f, (float)F.H) * (float)(mvlo ? F.rh : F.H)); + k.mp_x = k.xi_x * (float)F.W; + k.mp_y = k.xi_y * (float)F.H; + return k; +} + +// the motion factor with a texture's scale folded in (direct forms: b1's ms; KIO_M16: rmsx ..): spec 42 section 8's +// sp = s / (RN(1 / n) r), s = sx ms (the host's KiHost sp_x with that scale), and the raw-unit offset +float ki_dsp(float sx, float ms, int n, int r) { return wg_div(sx * ms, wg_div(1.0f, (float)n) * (float)r); } +float ki_dmo(float o, float mo, float ms) { return mo == 0.0f ? o : o + wg_div(mo, ms); } + +// spec 42 section 3: index = b + min(i + d, n - 1) if i + d > 0, else b. For n >= 1 (every render / output size) that +// is b + max(min(i + d, n - 1), 0) (HIP's ko_clampi form: one v_maxmin_i32 instead of a compare, a minimum and a select) +int kio_tap(int i, int d, int b, int n) { return b + ki_max(ki_min(i + d, n - 1), 0); } +// [MVlo] motion fetch: i' = b + i + d; index = min(i' + b, b + n - 1) if i' > 0, else b; for n >= 1 that is +// b + max(min(i', n - 1), 0) +int kio_mtap(int i, int d, int b, int n) { return b + ki_max(ki_min(b + i + d, n - 1), 0); } + +// colour texel (x, y) of the texture (subrect base included by the caller): FP32 RGBA, or RGBA16F (KIO_C16, exact +// widening). Direct: the game's colour texture t0 at (nd_rx + x, nd_ry + y), alpha 0 (unused by K), not divided by the +// pre-exposure (K applies kappa): the values of the HIP executor's RGBA16F input copy for a 16-bit float colour texture. +float4 kio_texel(KFrameIn F, int x, int y) +{ +#if WG_DIRECT_TEXTURES + return nd_raw_color_xy(x, y); +#else + const uint i = ((uint)y * (uint)F.cpitch + (uint)x) * 4u; + if ((F.flags & KIO_C16) != 0) { + const uint2 h = wg_ld64(F.color, i * 2u); + return float4(wg_h2f_lo(h.x), wg_h2f_hi(h.x), wg_h2f_lo(h.y), wg_h2f_hi(h.y)); + } + return wg_ld4f(F.color, i * 4u); +#endif +} +float4 kio_color(KFrameIn F, int ix, int iy) +{ + return kio_texel(F, kio_tap(ix, 0, F.cbx, F.rw), kio_tap(iy, 0, F.cby, F.rh)); +} +// motion texel (x, y) (subrect base included by the caller): FP32 pairs, or KIO_M16's raw FP16 value r. Direct: the raw +// value r of the game's motion texture t2. ki_dsp / ki_dmo apply the texture's scale and offset as the buffer form +// applies WgRtFrame's (below). +// KIO_M16 (the interop's raw-buffer route: input copies or model K's compact input pass, ms / mo in KFrameIn rmsx .., +// WgRtFrame's scale 1): the raw FP16 value r, the scale folded into the motion factor and the offset RN(mo / ms) in raw +// units, as for the direct forms (issue #26; before: ms r + mo per texel, then the factor of scale 1). +// Direct forms (issue #26): the motion scale and offset of the game's motion texture t2 come in b1 (ms, mo; the +// executor passes 1 / 0 in WgRtFrame), and they are applied as the buffer forms and the reference (clean/k/reference +// instage.py / outstage.py, spec 42 / 43) apply WgRtFrame's mv_scale / mv_bias: the texel's raw value r plus the offset +// in raw units, times the motion factor with the scale folded in (sp = RN(s / RN(RN(1 / n) r)) of the host, s = the +// scale). Before, the direct forms scaled every texel (RN(ms r) + mo: input; one fused mad(ms, r, mo): output) and +// used the factor of scale 1: with a non-unit scale the products round differently, which moved history taps (0.02-0.04 % +// of the output and history values differed from the buffer form from the first frame after a reset, in-game packs). +// Offset: b1's mo is in render pixels after the scale (mo = ms b, one FP32 product); the raw-unit offset is RN(mo / ms) +// (exactly 0 for mo = 0; equal to the buffer drive's b whenever that quotient recovers it). +void kio_motion(KFrameIn F, int x, int y, out float mvx, out float mvy) +{ +#if WG_DIRECT_TEXTURES + const float2 r = native_direct_motion_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)).xy; + mvx = r.x; + mvy = r.y; +#else + if ((F.flags & KIO_M16) != 0) { + const uint o = ((uint)y * (uint)F.mpitch + (uint)x) * (uint)F.mch * 2u; + const uint r = wg_ld32(F.motion, o); // channels 0, 1 (4-byte aligned) + mvx = wg_h2f_lo(r); + mvy = wg_h2f_hi(r); + return; + } + const float2 m = asfloat(wg_ld64(F.motion, ((uint)y * (uint)F.mpitch + (uint)x) * 8u)); + mvx = m.x; + mvy = m.y; +#endif +} + +float kio_depth(KFrameIn F, int ix, int iy, int dx, int dy) +{ + const int x = kio_tap(ix, dx, F.zbx, F.rw), y = kio_tap(iy, dy, F.zby, F.rh); +#if WG_DIRECT_TEXTURES + return native_direct_depth_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)); +#else + return wg_ldf(F.depth, ((uint)y * (uint)F.zpitch + (uint)x) * 4u); +#endif +} + +// spec 42 5.2 [MVlo]: offset of the nearest-depth motion pixel +void kio_nearest(KFrameIn F, int ix, int iy, out int odx, out int ody) +{ + const float c = kio_depth(F, ix, iy, 0, 0), a = kio_depth(F, ix, iy, -1, -1), b = kio_depth(F, ix, iy, 1, -1); + const float d = kio_depth(F, ix, iy, -1, 1), e = kio_depth(F, ix, iy, 1, 1); + const float k = kf_bits(0x3A83126Fu); + const bool inv = (F.flags & KIO_INV) != 0; + bool edge; + if (inv) + edge = (c < 0.5f * (a + e) - kf_opq(k * c)) || (c < 0.5f * (b + d) - kf_opq(k * c)); + else + edge = (c > kf_fma(c, k, 0.5f * (a + e))) || (c > kf_fma(c, k, 0.5f * (b + d))); + int bx = 0, by = 0; + if (edge) { + float best = c; + const float v[4] = {a, b, d, e}; + const int ox[4] = {-1, 1, -1, 1}, oy[4] = {-1, -1, 1, 1}; + [unroll] for (int q = 0; q < 4; ++q) { + const bool nearer = inv ? v[q] > best : v[q] < best; + if (nearer) { + best = v[q]; + bx = ox[q]; + by = oy[q]; + } + } + } + odx = bx; + ody = by; +} + +// spec 42 decode factor f (t f is linear) and spec 43 encode e(Y) (stored = Y e(Y)), correctly rounded steps +#define K_KIO_DECODE_NEAR kf_bits(0x3F7FBE88u) // below it: 1 / (1 - t) +#define K_KIO_ENCODE_NEAR 1000.0f // below it: 1 / (Y + 1) +float kio_decode(float t) +{ + if (t < K_KIO_DECODE_NEAR) return wg_div(1.0f, 1.0f - t); + const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); + return wg_div(kf_exp2(e) - 1.0f, 9.0f * t); +} +float kio_encode(float Y) +{ + if (Y < K_KIO_ENCODE_NEAR) return wg_div(1.0f, Y + 1.0f); + return wg_div(wg_div(kf_ln(kf_fma(Y, 9.0f, 1.0f)), 9.0f) - kf_bits(0x3C4FAE29u), Y); +} + +// bilinear sample of a 4-channel FP16 image (arena offset img, [h][w][4]) at normalized (cx, cy), clamp to edge; +// spec 42 section 1 (draft 2): t0 = v00 + (v10 - v00) fx, t1 = v01 + (v11 - v01) fx, (t0 (1 - fy)) + (t1 fy) +float4 kio_texel_h4(uint img, uint idx) +{ + const uint2 q = wg_ld64(img, idx * 8u); + return float4(wg_h2f_lo(q.x), wg_h2f_hi(q.x), wg_h2f_lo(q.y), wg_h2f_hi(q.y)); +} +float4 kio_bilinear4(uint img, int w, int h, float cx, float cy) +{ + const float px = kf_opq(cx * (float)w) - 0.5f, py = kf_opq(cy * (float)h) - 0.5f; + const float fx0 = floor(px), fy0 = floor(py); + const float fx = px - fx0, fy = py - fy0; + // HIP: finite ? (int)max(min(f, w), -1) : (+inf: w, -inf: -1, NaN: 0). The clamp already gives w / -1 for +-inf + // (min / max are IEEE minNum / maxNum), so only NaN needs the select + const int x0 = isnan(fx0) ? 0 : (int)kf_max(kf_min(fx0, (float)w), -1.0f); + const int y0 = isnan(fy0) ? 0 : (int)kf_max(kf_min(fy0, (float)h), -1.0f); + const int xa = ki_max(ki_min(x0, w - 1), 0), xb = ki_max(ki_min(x0 + 1, w - 1), 0); + const int ya = ki_max(ki_min(y0, h - 1), 0), yb = ki_max(ki_min(y0 + 1, h - 1), 0); + const float omfy = 1.0f - fy; + const float4 q00 = kio_texel_h4(img, (uint)ya * (uint)w + (uint)xa), q10 = kio_texel_h4(img, (uint)ya * (uint)w + (uint)xb); + const float4 q01 = kio_texel_h4(img, (uint)yb * (uint)w + (uint)xa), q11 = kio_texel_h4(img, (uint)yb * (uint)w + (uint)xb); + const float4 t0 = q00 + kf_opq4(q10 - q00) * fx; + const float4 t1 = q01 + kf_opq4(q11 - q01) * fx; + return kf_opq4(t0 * omfy) + kf_opq4(t1 * fy); +} + +// (Y, Co, Cg) of max(RGB, 0) scaled by kappa first (spec 42 6.2) or not (spec 43 4.2: kappa after) +void kio_ycocg(float R, float G, float B, out float Y, out float Co, out float Cg) +{ + Y = (0.25f * R + 0.5f * G) + 0.25f * B; + Co = kf_fma(R, 0.5f, kf_opq(0.0f) * G) - 0.5f * B; // 0 G: NaN for an infinite G, as HIP + Cg = (0.5f * G - 0.25f * R) - 0.25f * B; +} +float kio_luma_fma(float R, float G, float B) { return kf_fma(B, 0.25f, kf_fma(R, 0.25f, 0.5f * G)); } diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py index b07dc5f5..82a1a2cb 100644 --- a/clean/k/dx12/k_plan_bridge.py +++ b/clean/k/dx12/k_plan_bridge.py @@ -124,10 +124,11 @@ def chain_step(kernel, grid, block, args, tokens, bufs, cdir, tag): return f"step {kernel} {grid} {block} " + " ".join(out) -def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, fills=None): +def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, d3d12_name=None): """The kernel_replay chain of a plan: buffers (inputs from files, the rest filled with the D3D12 sentinel), one step per launch, a save per output buffer (..d3d12). grid_div: launch with grid // grid_div (the - negative control uses 2). fills: {buffer name: bytes} initial contents that differ from the HIP run's (none).""" + negative control uses 2). d3d12_name: the manifest's name of a plan kernel (default: the part after the code object's + colon; the tagged exact input kernels are kx/). modules: module lines, "[ ]".""" bufs = {n: (p, pdev.sizes[p]) for n, p in plan["bufs"].items()} lines = [f"chain {prefix}{name}"] + [f"module {m}" for m in modules] for n, (p, size) in bufs.items(): @@ -136,7 +137,8 @@ def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", else: lines.append(f"buf b_{n} {size + SLACK} - {D3D12_SENTINEL}") for li, (kernel, grid, block, args) in enumerate(plan["launches"]): - lines.append(chain_step(kernel.split(":")[-1], max(1, grid // grid_div), block, args, tokens_of(kernel), bufs, cdir, + dname = d3d12_name(kernel) if d3d12_name else kernel.split(":")[-1] + lines.append(chain_step(dname, max(1, grid // grid_div), block, args, tokens_of(dname), bufs, cdir, f"{prefix}{name}.l{li}")) for n in outputs: lines.append(f"save b_{n} {prefix}{name}.{n}.d3d12") diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index 95575417..bee8e3f5 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -18,7 +18,9 @@ the half-grid control (the port again with half its workgroups) is used by the families whose grid has more than one workgroup; - Model M's scale_rgb (frame.co, its port already proven) runs through the same replay as a control. -The families: wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's +The families: input = the exact tiled input stage (k_io_exact.co, kx/k_in_tiled) on frames of every flag combination (HDR, +low-resolution motion, inverted depth, reset), odd and tiny base grids, exposure scalars from a device buffer, special values +in the colour, depth, motion and histories (the same three levels as the stage families); wide = the wide network stages (k_block_wide.co: k_stagew_e3 .. _d3), as narrow; narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, +-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). @@ -50,6 +52,7 @@ "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "narrow": (["k_block.co"], NARROW_KERNELS), + "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), "wide": (["k_block_wide.co"], ["k_stagew2_e3", "k_stagew4_e4", "k_stagew2_e4", "k_stagew4_m", "k_stagew4_d4", "k_stagew2_d4", "k_stagew2_d3"])} CONTROL_CO = "frame.co" @@ -381,11 +384,11 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]} - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, [co_name], + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs) if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels neg = name - lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name], outputs, prefix="negative_", grid_div=2) items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size), "offset": list(offset), "specials": special, "outputs": outputs, "hip": got, @@ -445,6 +448,91 @@ def check_narrow(state, read): "failures": [o for o in out if not o["identical"]][:4]}, ok +# ---- family input: the exact tiled input stage (k_io_exact.co) ----------------------------------------------------- + +INPUT_BASES = [(1, 1), (7, 9), (8, 8), (9, 7), (33, 17)] +INPUT_FLAG_SETS = [(h, m, i, r) for h in (True, False) for m in (True, False) for i in (True, False) + for r in (False, True)] + + +def input_specials(x, rng, level): + """level 1: infinity, huge, zero and -0 values in the colour, depth and motion, and non-finite history; level 2: NaN + as well (in the colour, depth, motion and histories).""" + pool = [np.inf, -np.inf, 3.0e38, -3.0e38, 0.0, -0.0, 65504.0, -65504.0, 1e-30] + if level == 2: + pool += [np.nan] + pool = np.array(pool, np.float32) + for k in ("color", "depth", "motion"): + a = x[k] + pick = rng.random(a.shape) < 0.004 + a[pick] = rng.choice(pool, int(pick.sum())) + for k in ("Hc", "Hh"): + a = x[k] + pick = rng.random(a.shape) < 0.01 + a[pick] = rng.choice(pool.astype(np.float16), int(pick.sum())) + + +def run_input(build, rocm, seeds, cdir, hip): + from clean.k.dx12 import k_plan_bridge as PB + from clean.k.runner import k_io, k_stages + from clean.k.tests.test_k_io_emu import frame + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed + 500) + for ci, (hdr, mvlo, inv, reset) in enumerate(INPUT_FLAG_SETS): + base = INPUT_BASES[(ci + si) % len(INPUT_BASES)] + special = (ci // 2) % 3 # 0 none, 1 specials without NaN, 2 with NaN + name = f"inp_s{si}_{ci}" + x = frame(rng, hdr, mvlo, inv, reset=reset, base=base) + x["base"] = base + x.update(E_cur=1.5, E_prev=1.0, E_aux=0.75, pre_exposure=2.0) + if special: + input_specials(x, rng, special) + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, "input@exact", b"", x, None, None) + try: + prm = plan["launches"][-1][3][0] + # the exposure scalars come from a device buffer, as in the runtime (the host values are wrong on purpose) + xs = np.array([prm.f.kappa, prm.f.rho, prm.f.lam, np.float32(1.0) / np.float32(prm.f.kappa)], np.float32) + ptr = pdev.malloc(xs.nbytes) + pdev.upload(ptr, xs.tobytes()) + plan["bufs"]["xs"], plan["sizes"]["xs"] = ptr, xs.nbytes + prm.f.xs = ptr + prm.f.kappa, prm.f.rho, prm.f.lam = -17.0, 7.0, 9.0 + bufs = {n: getattr(prm, n) for n in ("Hc", "Hh", "x0", "cpre", "cpre2")} + plan["launches"] = k_io.input_launches(prm.f, bufs, exact=True, tiled=True) + if len(plan["launches"]) != 1: + continue # unsupported geometry: the runtime falls back to HIP's untiled pair + if reset: # a reset frame reads no history, motion or depth + act = plan["launches"][0][3][0] + act.Hc = act.Hh = None + act.f.motion = act.f.depth = None + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + for k, grid, block, a in plan["launches"]: + co, kernel = k.split(":") + hip.h.launch(hip.h.get_function(hip.mods[co], kernel), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {n: min(int(np.prod(sh)) * 2, sizes[n]) for n, sh in plan["outputs"]} + kw = dict(modules=["k_io_exact.co kx"], outputs=outputs, d3d12_name=lambda k: "kx/" + k.split(":")[-1]) + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, **kw) + if neg is None and plan["launches"][0][1] >= 4 and not reset: + neg = name + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, prefix="negative_", grid_div=2, + **kw) + items.append({"case": name, "kernel": "kx/k_in_tiled", "stage": "input", "size": list(base), + "offset": [0, 0], "specials": special, "outputs": outputs, "hip": got, "defined": defined}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + return items, lines, neg + + # ---- run ----------------------------------------------------------------------------------------------------------- def run(build, rocm, families, seeds, json_out): @@ -465,6 +553,9 @@ def run(build, rocm, families, seeds, json_out): if "narrow" in families: state["narrow"] = run_stages(NARROW, "k_block.co", "nar", build, rocm, seeds, cdir, hip) lines += state["narrow"][1] + if "input" in families: + state["input"] = run_input(build, rocm, seeds, cdir, hip) + lines += state["input"][1] if "wide" in families: state["wide"] = run_stages(WIDE, "k_block_wide.co", "wid", build, rocm, seeds, cdir, hip, WIDE_SIZES) lines += state["wide"][1] @@ -520,7 +611,7 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 - for fam in ("narrow", "wide"): + for fam in ("narrow", "wide", "input"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) ok = ok and good diff --git a/clean/k/dx12/kernels/k_in.hlsl b/clean/k/dx12/kernels/k_in.hlsl new file mode 100644 index 00000000..ff103a3c --- /dev/null +++ b/clean/k/dx12/kernels/k_in.hlsl @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of model K's exact tiled input-stage kernels (clean/k/kernels/k_io.hip ki_tiled_body, HIP build +// k_io_exact.co = WGK_IN_EXACT, wave64; spec 42 sections 3-8). WrongGPU's own code. +// +// One workgroup of 256 lanes per 8 x 8 base positions: lane 4 b + k is display position k of base position b. The +// workgroup first stages the 24 x 24 render-colour footprint of its tile in LDS (Y, Co, Cg, phi(Y) and the HDR weight +// omega per render pixel), then each lane does its display position's 4 fine positions (reprojected colour history, +// motion with the nearest-depth rule, hidden-history position), the 3 x 3 colour neighbourhood (mean, box, phi), the +// disagreement score, the own sample's luma channel, and the hidden-history sample; the 4 display positions of a base +// position exchange their hidden-history samples by two lane swaps (xor 1, xor 2: within groups of 4 lanes, so the +// wave size does not matter; wave64 as the HIP build) and add them in HIP's order. Writes x0 [Bh][Bw][16] FP16. +// Reset frames: only the own sample's channel 0 (channels 1 = 0 's, 2 and 3 zero), no LDS. +// +// Arithmetic: exactly the HIP kernel's operations in its order (k_io.hip has contraction off and every fma written +// out), with the exact path of WGK_IN_EXACT: log2 / 2^x correctly rounded (kf_log2 / kf_exp2, k_fp.hlsli, with their +// binary64 fallback: extended doubles), every division and reciprocal correctly rounded (wg_div; HIP's a / b and +// 1.0f / x), FP16 rounding to nearest even (wg_f2h; never the driver's (half), which rounds toward zero). Where the +// D3D12 compiler could merge two of HIP's roundings (a product by the uniform kappa or rho, a constant chain), the +// intermediate goes through kf_opq (k_fp.hlsli). FP32 denormals are flushed here (HIP keeps them): none occur in the +// port checks' frames (clean/k/dx12/k_port_check.py). +// +// Kernel declared here: kx/k_in_tiled (KInParams; computes KiHost on the GPU). The forms with KiHost in the arguments +// (kx/k_in_tiled_uniform, K_UNIFORM) and the forms reading the game's textures (_direct, WG_DIRECT_TEXTURES) are in this +// source for the runtime's use but are not declared yet: they are declared and checked with the runtime wiring. +// The KInParams pointers cpre / cpre2 are null in the tiled launches (runtime_k.cpp), as in HIP unused here. +// @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 +#include "wg_platform.hlsli" +#ifndef K_UNIFORM +#define K_UNIFORM 0 +#endif +#ifndef WG_DIRECT_TEXTURES +#define WG_DIRECT_TEXTURES 0 +#endif +#if WG_DIRECT_TEXTURES +#include "native_direct.hlsli" +#endif +#include "k_fp.hlsli" +#include "k_frame.hlsli" + +WG_ARGS_BEGIN + K_ARGS_FRAME_IN + K_ARGS_IN_PTRS +#if K_UNIFORM + K_ARGS_HOST +#endif +WG_ARGS_END + +float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } + +// ---- the exact input stage's arithmetic (k_io.hip with KI_FAST = 0) -------------------------------------------------- +float ki_log2(float x) { return kf_log2(x); } +float ki_exp2(float x) { return kf_exp2(x); } +float ki_rcp(float x) { return kf_div(1.0f, x); } +float ki_div(float a, float b) { return kf_div(a, b); } +// spec 42 decode factor D(t) +float ki_decode(float t) +{ + if (t < kf_bits(0x3F7FBE88u)) return ki_rcp(1.0f - t); + const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); + return ki_div(ki_exp2(e) - 1.0f, 9.0f * t); +} +float ki_phi(float Y, bool hdr) +{ + if (hdr) { + const float Yp = kf_max(Y, 0.0f); + return ki_div(ki_log2(kf_fma(Yp, 12.0f, 1.0f)) * kf_bits(0x3F317218u), Yp + kf_bits(0x358637BDu)); + } + return kf_fma(Y, 3.0f, 1.0f); +} +float ki_S(float z) +{ + const float zp = kf_min(kf_max(z, 0.0f), 1.0f); // HIP's NaN test is implied: maxNum(NaN, 0) = 0 + if (!(zp > kf_bits(0x3B4D2E1Cu))) return kf_bits(0x414EB852u) * zp; + const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once + return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu)); +} +// z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart +// (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it) +float ki_T(float z) +{ + return z * kf_max(ki_rcp(z + 1.0f), kf_bits(0x38D1B717u)); +} +float ki_L(float Y, bool hdr) { return hdr ? Y : ki_div(Y, kf_max(1.0f - kf_opq(Y), kf_bits(0x3E4CCCCDu))); } + +struct KiFine { // spec 42 section 5 per fine position + bool valid; + float3 h; // reprojected history (Y, Co, Cg) + float nx, ny; // hidden-history position (normalized) +}; + +KiFine ki_fine(KInParams P, KiHost k, int fx, int fy) +{ + const KFrameIn F = P.f; + const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); + const float px = (float)fx + 0.5f, py = (float)fy + 0.5f; + float mvx, mvy; + if ((F.flags & KIO_MVLO) != 0) { + const int ix = (int)kf_R(F.jx + kf_fma(px, k.a_x, -0.5f)), iy = (int)kf_R(F.jy + kf_fma(py, k.a_y, -0.5f)); + int dx, dy; + kio_nearest(F, ix, iy, dx, dy); + kio_motion(F, kio_mtap(ix, dx, F.mbx, F.rw), kio_mtap(iy, dy, F.mby, F.rh), mvx, mvy); + } else { + const int mx = (int)kf_R(kf_fma(px, k.mp_x, -0.5f)), my = (int)kf_R(kf_fma(py, k.mp_y, -0.5f)); + kio_motion(F, kio_tap(mx, 0, F.mbx, F.W), kio_tap(my, 0, F.mby, F.H), mvx, mvy); + } + // rounded before px + m (HIP); no kf_opq needed: the ISA keeps the products (precaution link dropped, ISA-checked) + // the texture's motion scale folded into the factor and its offset in raw units, as the buffer form gets them in + // WgRtFrame (hlsl/k_frame.hlsli kio_motion; issue #26): direct forms b1's ms / mo, KIO_M16 rmsx .. +#if WG_DIRECT_TEXTURES + const bool dfold = true; + const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy; +#else + const bool dfold = (F.flags & KIO_M16) != 0; + const float dmsx = F.rmsx, dmsy = F.rmsy, dmox = F.rmox, dmoy = F.rmoy; +#endif + float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy; + [branch] if (dfold) { + const bool dmvlo = (F.flags & KIO_MVLO) != 0; + spx = ki_dsp(F.sx, dmsx, F.W, dmvlo ? F.rw : F.W); + spy = ki_dsp(F.sy, dmsy, F.H, dmvlo ? F.rh : F.H); + ofx = ki_dmo(F.mox, dmox, dmsx); + ofy = ki_dmo(F.moy, dmoy, dmsy); + } + const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy); + const float wx = px + mx, wy = py + my; + KiFine r; + const float ux = wx * k.xi_x, uy = wy * k.xi_y; + r.valid = ux >= 0.0f && ux <= 1.0f && uy >= 0.0f && uy <= 1.0f; + const float4 s = kio_bilinear4(P.Hc, W16, H16, wx * k.h_x, wy * k.h_y); + if ((F.flags & KIO_HDR) != 0) { + const float f = ki_decode(s.x); + r.h = F.rho * kf_opq3(s.xyz * f); + } else { + r.h = s.xyz; + } + r.nx = ((float)(4 * (fx >> 2) + 2) + mx) * k.g_x; // fx, fy >= 0: fx / 4 = fx >> 2 + r.ny = ((float)(4 * (fy >> 2) + 2) + my) * k.g_y; + return r; +} + +// kappa max(RGB, 0), each product rounded (kf_opq: no merging with the 0.25 / 0.5 factors that follow) +float3 ki_rgb(KFrameIn F, float4 c) { return kf_opq3(F.kappa * max(c.xyz, 0.0f)); } + +struct KiTiledTap { + float4 cv; // Y, Co, Cg, phi(Y) + float om; // HDR weight max(1 / (Y + 1), 1e-4); 1 for LDR (A = fma(1, v, A) is HIP's A + v, one rounding of the sum) +}; +KiTiledTap ki_tiled_tap_from_raw(KFrameIn F, float4 c) +{ + const float3 rgb = ki_rgb(F, c); + float Y, Co, Cg; + kio_ycocg(rgb.x, rgb.y, rgb.z, Y, Co, Cg); + const bool hdr = (F.flags & KIO_HDR) != 0; + KiTiledTap r; + r.cv = float4(Y, Co, Cg, ki_phi(Y, hdr)); + r.om = hdr ? kf_max(ki_rcp(Y + 1.0f), kf_bits(0x38D1B717u)) : 1.0f; + return r; +} +KiTiledTap ki_tiled_tap(KFrameIn F, int rx, int ry) { return ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); } +float ki_tiled_ch0(KFrameIn F, int rx, int ry) +{ + const float3 rgb = ki_rgb(F, kio_color(F, rx, ry)); + const bool hdr = (F.flags & KIO_HDR) != 0; + const float Yc = kio_luma_fma(rgb.x, rgb.y, rgb.z); + return ki_S(ki_T(F.lam * ki_L(Yc, hdr))); +} +// HIP ki_tiled_transpose: lane bit 0 / bit 1 swaps through ds_bpermute (wg_xor = WaveReadLaneAt(lane ^ d)) +float4 ki_tiled_transpose(float4 v, int lane) +{ + const bool b0 = (lane & 1) != 0, b1 = (lane & 2) != 0; + const float x = wg_xor(b0 ? v.x : v.y, 1u), y = wg_xor(b0 ? v.z : v.w, 1u); + const float4 a = float4(b0 ? x : v.x, b0 ? v.y : x, b0 ? y : v.z, b0 ? v.w : y); + const float z = wg_xor(b1 ? a.x : a.z, 2u), w = wg_xor(b1 ? a.y : a.w, 2u); + return float4(b1 ? z : a.x, b1 ? w : a.y, b1 ? a.z : z, b1 ? a.w : w); +} +void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3) +{ + wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, + uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); +} + +groupshared float4 tile_cv[24 * 24]; +groupshared float tile_om[24 * 24]; + +#if K_UNIFORM +WG_KERNEL(k_in_tiled_uniform) +#else +WG_KERNEL(k_in_tiled) +#endif +{ + WG_SETUP(); + KF_SETUP(); + KInParams P0; + K_LOAD_IN_PARAMS(P0) + const KInParams P = ki_params(P0); + const KFrameIn F = P.f; +#if K_UNIFORM + KiHost hk; + K_LOAD_HOST(hk) +#else + const KiHost hk = ki_host(F); +#endif + const int lane = (int)wg_lane_id(); + const int tiles_x = (F.Bw + 7) / 8, tile = (int)wg_block_id(); + const int bx0 = (tile % tiles_x) * 8, by0 = (tile / tiles_x) * 8; + const int k = lane & 3, local = lane >> 2; + const int bx = bx0 + (local & 7), by = by0 + (local >> 3); + const bool active = bx < F.Bw && by < F.Bh; + const int id = by * F.Bw + bx; + const bool hdr = (F.flags & KIO_HDR) != 0; + const float c1e4 = kf_bits(0x38D1B717u); + float ch0 = 0.0f, ch1 = 0.0f, ch3 = 0.0f; + float4 hid = 0.0f; + if ((F.flags & KIO_RESET) != 0) { + if (active) { + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); + const float h = ki_tiled_ch0(F, rx, ry); + ki_store_x0(P.x0, id, k, h, h, 0.0f, 0.0f); + } + return; + } + const int tx0 = ki_max(ki_min((int)kf_R(F.jx + kf_fma((float)(4 * bx0 + 1), hk.a_x, -0.5f)) - 1, F.rw - 1), 0); + const int ty0 = ki_max(ki_min((int)kf_R(F.jy + kf_fma((float)(4 * by0 + 1), hk.a_y, -0.5f)) - 1, F.rh - 1), 0); + for (int q = lane; q < 24 * 24; q += 256) { + const int raw_x = tx0 + q % 24, raw_y = ty0 + q / 24; + const int rx = ki_min(raw_x, F.rw - 1), ry = ki_min(raw_y, F.rh - 1); + const KiTiledTap value = ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); + tile_cv[q] = value.cv; + tile_om[q] = value.om; + } + wg_barrier_lds(); + if (active) { + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + // colour neighbourhood first, then each fine position scored as it is produced (the same per-position + // operations and the same order of the D sum as HIP; nothing of a fine position stays live but the own + // sample's Yh, nx, ny) + const int cx = (int)kf_R(F.jx + kf_fma((float)(2 * u + 1), hk.a_x, -0.5f)); + const int cy = (int)kf_R(F.jy + kf_fma((float)(2 * v + 1), hk.a_y, -0.5f)); + float3 A = 0.0f, lo = 0.0f, hi = 0.0f; + [unroll] for (int ddy = -1; ddy <= 1; ++ddy) { + [unroll] for (int ddx = -1; ddx <= 1; ++ddx) { + const int rx = kio_tap(cx, ddx, 0, F.rw), ry = kio_tap(cy, ddy, 0, F.rh); + const int lx = rx - tx0, ly = ry - ty0; + const bool cached = (uint)lx < 24u && (uint)ly < 24u; + float4 cv; + float om; + [branch] if (cached) { + cv = tile_cv[ly * 24 + lx]; + om = tile_om[ly * 24 + lx]; + } else { // defensive, as HIP: a footprint wider than the tile (the runtime's guard prevents it) + const KiTiledTap fb = ki_tiled_tap(F, rx, ry); + cv = fb.cv; + om = fb.om; + } + A = float3(kf_fma(om, cv.x, A.x), kf_fma(om, cv.y, A.y), kf_fma(om, cv.z, A.z)); // LDR: om = 1 + const float3 val = cv.xyz * cv.w; + const bool first = ddx == -1 && ddy == -1; + lo = first ? val : min(lo, val); + hi = first ? val : max(hi, val); + } + } + float3 mu = A * kf_bits(0x3DE38E39u); // rounded before 1 - mu (the ISA keeps the products: no kf_opq) + if (hdr) { + const float den = kf_max(1.0f - mu.x, c1e4); + mu = float3(ki_div(mu.x, den), ki_div(mu.y, den), ki_div(mu.z, den)); + } + const float pm = ki_phi(mu.x, hdr); + const float3 mp = kf_opq3(mu * pm); + const float3 eta = 0.5f * (hi - lo); + const float3 nu = float3(kf_fma(hi.x - lo.x, 8.0f, 1.0f), kf_fma(hi.y - lo.y, 8.0f, 1.0f), kf_fma(hi.z - lo.z, 8.0f, 1.0f)); + const int oq = (1 - (v & 1)) + 2 * (u & 1); // (2u + 1 - (v mod 2), 2v + (u mod 2)) as a fine position index + // fine positions of the display position, in the order (2u, 2v), (2u+1, 2v), (2u, 2v+1), (2u+1, 2v+1) + bool valid = true; + float D = 0.0f, own_hx = 0.0f, own_nx = 0.0f, own_ny = 0.0f; + [unroll] for (int q = 0; q < 4; ++q) { + const KiFine fq = ki_fine(P, hk, 2 * u + (q & 1), 2 * v + (q >> 1)); + valid = valid && fq.valid; + const float ph = ki_phi(fq.h.x, hdr); + const float3 g = max(abs(mp - kf_opq3(fq.h * ph)) - eta, 0.0f); + const float3 dl = float3(ki_div(g.x, nu.x), ki_div(g.y, nu.y), ki_div(g.z, nu.z)); + const float score = kf_min(kf_fma(dl.z, 0.5f, kf_fma(dl.x, 2.0f, 0.5f * dl.y)), 1.0f); + D = D + score; + if (q == oq) { + own_hx = fq.h.x; + own_nx = fq.nx; + own_ny = fq.ny; + } + } + D = D * 0.25f; + // own sample + const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); + ch0 = ki_tiled_ch0(F, rx, ry); + const float4 hs = kio_bilinear4(P.Hh, F.Bw, F.Bh, own_nx, own_ny); + hid = valid ? hs : 0.0f; + // channel 1 is channel 0's function of Yh, and Yh = Yc without a valid history: then it equals channel 0 + ch1 = valid ? ki_S(ki_T(F.lam * ki_L(own_hx, hdr))) : ch0; + const float d3 = valid ? D : 0.0f; + ch3 = kf_finite(d3) ? d3 : 0.0f; + } + const float4 hs = ki_tiled_transpose(hid, lane); + if (!active) return; + float hsum = (hs.x + hs.w) + (hs.y + hs.z); + hsum = hsum * 0.25f; + ki_store_x0(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3); +} diff --git a/clean/k/kernels/k_io.hip b/clean/k/kernels/k_io.hip index 7159dd9f..ae119eaa 100644 --- a/clean/k/kernels/k_io.hip +++ b/clean/k/kernels/k_io.hip @@ -25,9 +25,14 @@ enum { KIO_HDR = 1, KIO_MVLO = 2, KIO_INV = 4, KIO_RESET = 8, KIO_C16 = 16, KIO_ // pitch mpitch texels) holding the game's raw values r, the motion is m = ms r + mo (FP32 multiply, then add: the // interop's conversion, clean/kernels/frame.hip input_convert). Both read the interop's input copies (or its compact // input pass) directly; the values equal those of the FP32 inputs the conversion would have written. -// Game-output formats of k_out_pixel (KOutParams.omode): RGBA float32 [H][opitch] (the spec's output, tests); RGBA16F -// rows of opitch texels with RGB rounded to nearest even and alpha 1 (the interop's output copy); dense RGB float32 -// [H][opitch] (the interop's output pass, which writes alpha 1 itself). +// Game-output formats of k_out_pixel (KOutParams.omode): RGBA float32 [H][opitch] (the spec's output, tests, and the +// interop's output pass for model K: FrameIo.out_rgba32f); RGBA16F rows of opitch texels with RGB rounded to nearest +// even (the interop's output copy); dense RGB float32 [H][opitch] (the runtime's own test buffer: no fourth value). +// Game output alpha (issue #26): the colour input's alpha of the current frame, spatial only: cbar[3], the fourth +// channel of the 4.3 bilinear (the stored tap's A = RN16 of the colour texel's fourth value as the game supplies it, the +// four taps and FP16 weights of Y / Co / Cg, RN16(w0 a0) then three FP16 fused multiply-adds). No history, nothing +// from the network, no exposure, no clamp. RGBA16F rows store it as it is, RGBA float32 as its exact widening. The +// D3D12 ports (k_out.hlsl) write the same. enum { KIO_OUT_RGBA32F = 0, KIO_OUT_RGBA16F = 1, KIO_OUT_RGB32F = 2 }; // Frame inputs shared by both stages (must match KFrameIn in clean/k/runner/k_io.py). @@ -79,7 +84,7 @@ struct KOutParams { h16* Hcout; h16* Hhout; // [Bh][Bw][4] int opitch; - int omode; // KIO_OUT_RGBA32F (default), KIO_OUT_RGBA16F (alpha 1), KIO_OUT_RGB32F (runtime interop) + int omode; // KIO_OUT_RGBA32F (default), KIO_OUT_RGBA16F (both: alpha cbar[3]), KIO_OUT_RGB32F (no alpha) h16* ctap; // k_out_color -> k_out_pixel: stored tap (Y, Co, Cg, A) of spec 43 4.2 per render pixel [rh][rw][4] h16* ctapT; // the same: filter-tap value T of spec 43 5.3 per render pixel [rh][rw] KOutUni u; // frame-uniform values (host) @@ -153,12 +158,18 @@ WG_FN v4f kio_texel(const KFrameIn& F, int x, int y) { WG_FN v4f kio_color(const KFrameIn& F, int ix, int iy) { return kio_texel(F, kio_tap(ix, 0, F.cbx, F.rw), kio_tap(iy, 0, F.cby, F.rh)); } -// motion texel (x, y) of the texture (subrect base included by the caller): FP32 pairs, or KIO_M16's ms r + mo +// motion texel (x, y) of the texture (subrect base included by the caller): FP32 pairs, or KIO_M16's raw FP16 value r +// KIO_M16 (the interop's raw-buffer route: input copies or model K's compact input pass, ms / mo in KFrameIn rmsx .., +// WgRtFrame's scale 1): the raw FP16 value r, the scale folded into the motion factor and the offset RN(mo / ms) in raw +// units, as for the direct forms (issue #26; before: ms r + mo per texel, then the factor of scale 1). +// kio_m16_sp / kio_m16_mo: the factor with the scale folded in (the host's expression with s = sx ms) and the offset. +WG_FN float kio_m16_sp(float s, float ms, float den) { return (s * ms) / den; } +WG_FN float kio_m16_mo(float o, float mo, float ms) { return mo == 0.0f ? o : o + mo / ms; } WG_FN void kio_motion(const KFrameIn& F, int x, int y, float* mvx, float* mvy) { if (F.flags & KIO_M16) { const h16* r = kio_ptr((const h16*)F.motion, ((u32)y * F.mpitch + x) * (u32)F.mch); - *mvx = F.rmsx * (float)r[0] + F.rmox; - *mvy = F.rmsy * (float)r[1] + F.rmoy; + *mvx = (float)r[0]; + *mvy = (float)r[1]; return; } const float* m = kio_ptr(F.motion, ((u32)y * F.mpitch + x) * 2u); @@ -421,8 +432,8 @@ WG_FN void ko_motion(const KFrameIn& F, int x, int y, float* mvx, float* mvy) { if (F.flags & KIO_M16) { const u32 i = F.mch == 2 ? t << 1 : (F.mch == 4 ? t << 2 : t * (u32)F.mch); const h16* r = kio_ptr((const h16*)F.motion, i); - *mvx = F.rmsx * (float)r[0] + F.rmox; - *mvy = F.rmsy * (float)r[1] + F.rmoy; + *mvx = (float)r[0]; // KIO_M16: the raw value (kio_m16_sp / kio_m16_mo apply ms / mo at the use) + *mvy = (float)r[1]; return; } const float* m = kio_ptr(F.motion, t * 2u); @@ -801,7 +812,14 @@ WG_FN void ko_pixel(KOutParams P0) { } spx = U.spx; // [MVlo]: sx / rho_x, else sx spy = U.spy; - const float wx = kf_fma(spx, mvx + F.mox, px), wy = kf_fma(spy, mvy + F.moy, py); + float ofx = F.mox, ofy = F.moy; + if (F.flags & KIO_M16) { // KIO_M16: the scale folded into the factor (out_uniforms' spx with s = sx ms), the offset + spx = mvlo ? kio_m16_sp(F.sx, F.rmsx, U.rhox) : F.sx * F.rmsx; + spy = mvlo ? kio_m16_sp(F.sy, F.rmsy, U.rhoy) : F.sy * F.rmsy; + ofx = kio_m16_mo(F.mox, F.rmox, F.rmsx); + ofy = kio_m16_mo(F.moy, F.rmoy, F.rmsy); + } + const float wx = kf_fma(spx, mvx + ofx, px), wy = kf_fma(spy, mvy + ofy, py); // 3.4 validity, 3.5 colour history bool valid = !(F.flags & KIO_RESET) && wx * invW >= 0.0f && wx * invW <= 1.0f && wy * invH >= 0.0f && @@ -1281,8 +1299,8 @@ WG_FN void ko_pixel(KOutParams P0) { Cgg = kinv * (Cgo * d); } const float R = (Yg + Cog) - Cgg, G = Cgg + kf_fma(Cog, 0.0f, Yg), B = (Yg - Cog) - Cgg; - if (P.omode == KIO_OUT_RGBA16F) { // RGB rounded once to FP16 (nearest even), alpha 1 - const v4h o = {kh_round(R), kh_round(G), kh_round(B), (h16)1.0f}; + if (P.omode == KIO_OUT_RGBA16F) { // RGB rounded once to FP16 (nearest even), alpha cbar[3] (an FP16 value) as it is + const v4h o = {kh_round(R), kh_round(G), kh_round(B), cbar[3]}; *(v4h*)kio_ptr((h16*)P.out, ((u32)y * P.opitch + x) * 4u) = o; } else if (P.omode == KIO_OUT_RGB32F) { float* o = kio_ptr(P.out, ((u32)y * P.opitch + x) * 3u); @@ -1436,7 +1454,15 @@ WG_FN KiFine ki_fine(const KInParams& P, const KiHost& k, int fx, int fy) { const int mx = (int)kf_R(kf_fma(px, k.mp_x, -0.5f)), my = (int)kf_R(kf_fma(py, k.mp_y, -0.5f)); kio_motion(F, kio_tap(mx, 0, F.mbx, F.W), kio_tap(my, 0, F.mby, F.H), &mvx, &mvy); } - const float mx = k.sp_x * (mvx + F.mox), my = k.sp_y * (mvy + F.moy); + float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy; + if (F.flags & KIO_M16) { // KIO_M16: the scale folded into the factor (ki_host's sp_x with s = sx ms), the offset + const bool mvlo = (F.flags & KIO_MVLO) != 0; + spx = kio_m16_sp(F.sx, F.rmsx, (1.0f / (float)F.W) * (float)(mvlo ? F.rw : F.W)); + spy = kio_m16_sp(F.sy, F.rmsy, (1.0f / (float)F.H) * (float)(mvlo ? F.rh : F.H)); + ofx = kio_m16_mo(F.mox, F.rmox, F.rmsx); + ofy = kio_m16_mo(F.moy, F.rmoy, F.rmsy); + } + const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy); const float wx = px + mx, wy = py + my; KiFine r; r.valid = wx * k.xi_x >= 0.0f && wx * k.xi_x <= 1.0f && wy * k.xi_y >= 0.0f && wy * k.xi_y <= 1.0f; diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 9f0457d1..e6a54b6b 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -451,7 +451,8 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e !alloc(&expo_, 32, true) || !alloc(&expo_partials_, 4 * (size_t)cdiv(ny, 4)) || !alloc(&expo_computed_, 4, true) || !alloc(&expo_sync_, 4, true) || !alloc(&xs_, 16, true) || !alloc(&kexpo_, 16, true)) return false; - if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 12, true)) return false; + // the own output buffer (tests): dense RGB f32, dense RGBA f32 (FrameIo.out_rgba32f) or RGBA16F rows that fit + if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 16, true)) return false; // the filter-shape table (spec 43 section 7) of every level, computed once; a frame reads its level's set std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); @@ -471,14 +472,18 @@ bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a return dev_->launch(kernel, grid, block, args) || dfail(); } -bool KRuntime::upload_inputs(int rw, int rh, const float* rgb, const float* dep, const float* mot) { +bool KRuntime::upload_inputs(int rw, int rh, const float* color, const float* dep, const float* mot, int channels) { if (rw <= 0 || rh <= 0 || rw > max_rw_ || rh > max_rh_) return fail("render size larger than the maximum"); if (external_io_) return fail("inputs come from the interop (external_io)"); + if (channels != 3 && channels != 4) return fail("model K: the colour has 3 or 4 values per pixel"); if (!dev_->sync()) return dfail(); const size_t n = (size_t)rw * rh; - std::vector c(n * 4, 0.0f); + // the dense colour's fourth value is the colour input's alpha, unmodified (the output alpha follows it through the + // output stage's current-colour bilinear; issue #26); an RGB colour has none: 1, the value a texture fetch + // gives a missing alpha + std::vector c(n * 4, 1.0f); for (size_t i = 0; i < n; ++i) - for (int k = 0; k < 3; ++k) c[4 * i + k] = rgb[3 * i + k]; + for (int k = 0; k < channels; ++k) c[4 * i + k] = color[(size_t)channels * i + k]; return (dev_->upload(color_, c.data(), n * 16) && dev_->upload(depth_, dep, n * 4) && dev_->upload(motion_, mot, n * 8) && dev_->sync()) || dfail(); @@ -789,8 +794,9 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { // network (spec 41 / 44) if (!mark(2) || !network(f)) return false; - // output stage (spec 43) into the caller's output: RGBA16F rows (interop output copy), dense RGB f32 (interop - // output pass, FORMATS.md "Per-frame runtime"), or the own RGB f32 buffer (tests) + // output stage (spec 43) into the caller's output: RGBA16F rows (interop output copy), dense RGBA f32 (the interop's + // output pass for model K, FrameIo.out_rgba32f: the fourth value is the output alpha), dense RGB f32 (FORMATS.md + // "Per-frame runtime"; no alpha), or the own buffer (tests) in one of these layouts if (!mark(3)) return false; // interop option handover=staged: the output stage is next (FrameIo.out_signal) if (io && io->out_signal && !dev_->signal_fence(io->out_signal, io->out_signal_value)) return dfail(); @@ -810,10 +816,16 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { const int hp = io ? io->out_half_pitch : 0; op.out = io && io->out ? io->out : out_; if (!op.out) return fail("frame without an output buffer (interop)"); + // the own output buffer (no FrameIo output) holds RGBA16F rows only if they fit + if (hp > 0 && op.out == out_ && (size_t)hp * H_ > (size_t)W_ * H_ * 16) + return fail("model K: RGBA16F rows do not fit the own output buffer"); if (hp > 0) { if (hp % 8) return fail("model K: output rows must be a multiple of 8 bytes"); op.omode = OUT_RGBA16F; op.opitch = hp / 8; + } else if (io && io->out_rgba32f) { + op.omode = OUT_RGBA32F; + op.opitch = W_; } else { op.omode = OUT_RGB32F; op.opitch = W_; @@ -871,7 +883,7 @@ const char* KRuntime::kernels() { " base=" + std::to_string(Bw_) + "x" + std::to_string(Bh_) + " hdr=" + (hdr_ ? "1" : "0") + " mv=render" + " filter_level=" + std::to_string(level_last_) + - " out=" + (omode_last_ == OUT_RGBA16F ? "rgba16f" : "f32") + + " out=" + (omode_last_ == OUT_RGBA16F ? "rgba16f" : omode_last_ == OUT_RGBA32F ? "rgba32f" : "f32") + std::string(compact_head_ ? " post=k_out_pixel_c filter=fused post_last=k_out_pixel_c filter_last=fused" : " post=k_out_pixel filter=fused post_last=k_out_pixel filter_last=fused") + (diag_input2_ || diag_copyin_ @@ -895,7 +907,7 @@ bool KRuntime::buffer(const std::string& name, DevPtr* p, size_t* bytes) { {"exposure.pair", {kexpo_, 12}}, {"k.scalars", {xs_, 16}}, {"exposure.measured", {expo_computed_, 4}}, {"k.x0", {net_["X0"], net_bytes_["X0"]}}, {"k.N", {net_["OUT"], net_bytes_["OUT"]}}, {"k.Hc", {Hc_[lp], P16 * 8}}, {"k.L", {L_[lp], P16 * 8}}, - {"k.Hh", {Hh_[lp], (size_t)Bw_ * Bh_ * 8}}, {"filter.out", {out_, (size_t)W_ * H_ * 12}}, + {"k.Hh", {Hh_[lp], (size_t)Bw_ * Bh_ * 8}}, {"filter.out", {out_, (size_t)W_ * H_ * (omode_last_ == OUT_RGBA32F ? 16 : 12)}}, {"input.color", {color_, R * 16}}, {"input.depth", {depth_, R * 4}}, {"input.motion", {motion_, R * 8}}, }; diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 1b5a26e9..9e633b99 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -56,8 +56,10 @@ class KRuntime { bool create(const WgRtConfig& cfg, Device* dev, AllocFn alloc, bool external_io); // Enqueues one frame (does not wait). bool frame(const WgRtFrame& fr, const FrameIo* io); - // Test path (wg_rt_upload_inputs): dense RGB float32, depth, motion (render pixels) into the own input buffers. - bool upload_inputs(int rw, int rh, const float* rgb, const float* depth, const float* motion); + // Test path (wg_rt_upload_inputs / wg_rt_upload_inputs_rgba): dense colour float32 of `channels` values per pixel (3: + // RGB, the alpha is 1 as for a colour without an alpha channel; 4: RGBA, the alpha as given), depth, motion (render + // pixels) into the own input buffers. The output alpha follows the colour's alpha (issue #26). + bool upload_inputs(int rw, int rh, const float* color, const float* depth, const float* motion, int channels = 3); const char* kernels(); bool buffer(const std::string& name, DevPtr* p, size_t* bytes); const WgRtFrameInfo& info() const { return info_; } diff --git a/clean/k/tests/test_k_alpha.py b/clean/k/tests/test_k_alpha.py new file mode 100644 index 00000000..62ed38d0 --- /dev/null +++ b/clean/k/tests/test_k_alpha.py @@ -0,0 +1,319 @@ +# SPDX-License-Identifier: MIT +"""Model K: the output alpha follows the colour input's alpha (issue #26). Host checks (no GPU, no HIP run). + +The rule (the spec side's answer, in our terms): the output alpha is purely spatial, from the current frame's colour +input only. + +- Carry: the colour input's fourth channel at render resolution, this frame's, exactly as the game supplies it (no + pre-exposure division, no exposure scale, no clamp: negative values and values above 1 pass through). No history, + nothing from the network. A colour format without an alpha channel: 1, the value a texture fetch gives a missing + component. +- The output stage reads it at the same four render taps as the Y / Co / Cg current-colour 2 x 2 bilinear of spec 43 + section 4.3, with the same edge rule (tap positions clamped to the render rectangle) and the same four FP16 weights. +- Rounding: each tap's alpha is rounded to FP16 first; RN16(w0 a0); then the other three taps with FP16 fused + multiply-adds in the tap order of the colour bilinear ((0,0), (1,0), (0,1), (1,1)). +- The result is the output's fourth channel, unchanged; HDR and LDR alike. + +Checked here: +- alpha_reference (a straightforward CPU form of the rule, FP32 / FP16 steps with NumPy) against exact rational + arithmetic on sampled pixels, and against the fourth channel of the spec 43 reference clean/k/reference/outstage.py + (bit for bit: gradient, random, negative, above-1 and out-of-FP16-range alphas; HDR and LDR; several sizes and + jitters; the render-rectangle edges); +- spatial only: the reference's alpha does not change with the histories, the network output, the motion, the exposure + or the pre-exposure, and over two chained frames with different alpha the second frame's alpha is that of its own + colour; +- no clamp: negative and above-1 results come out; +- sources: every K output path stores cbar3 (D3D12 k_out.hlsl: RGBA16F rows, the direct texture store, dense RGBA + f32; HIP k_io.hip), the direct colour read keeps the texel's alpha, the interop's passes carry it for model K only + (input passes, output pass on an RGBA f32 result, output pass on the result texture, fallback), the runtime's + RGBA f32 result (FrameIo.out_rgba32f) and the test path's alpha. +The GPU checks are clean/k/tests/test_k_alpha_gpu.py (opt-in). + +Run: python -m unittest clean.k.tests.test_k_alpha +""" +import os +import unittest +from fractions import Fraction + +import numpy as np + +from clean.k.reference.f16 import decode_exact, round_f16_exact +from clean.k.reference.fp import F32, R, f32, fma16, fma32, h, sat + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +CLEAN = os.path.join(ROOT, "clean") +TAPS = ((0, 0), (1, 0), (0, 1), (1, 1)) # (dx, dy), the colour bilinear's order + + +def read(*p): + with open(os.path.join(CLEAN, *p), encoding="utf-8") as f: + return f.read().replace("\r\n", "\n") + + +def alpha_taps(render, output, jitter): + """Per output pixel [H, W] of spec 43 sections 4.1 / 4.3: the four taps' render coordinates (clamped to the render + rectangle) and FP16 weights, in the tap order TAPS. -> (ty [4, H, W], tx [4, H, W], w [4, H, W] float16).""" + rw, rh = (int(v) for v in render) + W, H = (int(v) for v in output) + jx, jy = F32(jitter[0]), F32(jitter[1]) + with np.errstate(all="ignore"): + rho_x, rho_y = F32(rw) * (F32(1) / F32(W)), F32(rh) * (F32(1) / F32(H)) + jmx, jmy = jx - F32(0.5), jy - F32(0.5) + yy, xx = np.meshgrid(np.arange(H), np.arange(W), indexing="ij") + px, py = f32(xx) + F32(0.5), f32(yy) + F32(0.5) + # 4.1 local origin of the pixel's 16 x 16 block + o_x = R(fma32(f32(16 * (xx // 16)) + F32(0.5), rho_x, jmx)) - F32(4) + o_y = R(fma32(f32(16 * (yy // 16)) + F32(0.5), rho_y, jmy)) - F32(4) + Ux, Uy = h(fma32(px, rho_x, jmx) - o_x), h(fma32(py, rho_y, jmy) - o_y) + # 4.3 the bilinear's FP16 fractions and weights + fUx, fUy = np.floor(Ux), np.floor(Uy) + fx, fy = sat(h(Ux - fUx)), sat(h(Uy - fUy)) + one = np.float16(1) + w = np.stack([h(h(one - fx) * h(one - fy)), h(fx * h(one - fy)), h(h(one - fx) * fy), h(fx * fy)]) + bx = (o_x + fUx.astype(np.float32)).astype(np.int64) + by = (o_y + fUy.astype(np.float32)).astype(np.int64) + tx = np.stack([np.clip(bx + dx, 0, rw - 1) for dx, _ in TAPS]) + ty = np.stack([np.clip(by + dy, 0, rh - 1) for _, dy in TAPS]) + return ty, tx, w.astype(np.float16) + + +def alpha_reference(alpha, render, output, jitter): + """The rule: the output alpha [H, W] (float16) of a colour alpha [rh, rw] (any float type; None: a colour without + an alpha channel, 1).""" + rw, rh = (int(v) for v in render) + a = np.ones((rh, rw), np.float32) if alpha is None else np.asarray(alpha, np.float32) + assert a.shape == (rh, rw) + ty, tx, w = alpha_taps(render, output, jitter) + with np.errstate(all="ignore"): + a16 = h(a) # each tap's alpha rounded to FP16 first + acc = h(a16[ty[0], tx[0]].astype(np.float64) * w[0].astype(np.float64)) # RN16(w0 a0): the product is exact + for q in (1, 2, 3): # then three FP16 fused multiply-adds + acc = fma16(a16[ty[q], tx[q]], w[q], acc) + return acc.astype(np.float16) + + +def same16(a, b): + a, b = np.asarray(a, np.float16), np.asarray(b, np.float16) + return (a.view(np.uint16) == b.view(np.uint16)) | (np.isnan(a) & np.isnan(b)) + + +def make_alpha(rng, rw, rh, t=0): + """A colour alpha [rh, rw] float32 that changes with t: gradient, random block, negative band, band above 1, far + values (one beyond the FP16 range), other values on the first / last row and column.""" + y, x = np.mgrid[0:rh, 0:rw].astype(np.float32) + a = np.mod((x + 2.0 * y + 5.0 * t) / np.float32(rw + 2 * rh), 1.0).astype(np.float32) + a[rh // 4: rh // 2, rw // 4: rw // 2] = rng.random((rh // 2 - rh // 4, rw // 2 - rw // 4), np.float32) + a[rh // 2, :] = -rng.random(rw, np.float32) * 3.0 - 0.01 + a[:, rw // 2] = 1.0 + rng.random(rh, np.float32) * 3.0 + a[rh // 3, rw // 3] = -37.5 + a[(2 * rh) // 3, (2 * rw) // 3] = 1000.0 + a[0, :] = 0.25 + 0.5 * (t % 2) + a[-1, :] = 2.5 + a[:, 0] = -0.5 + a[:, -1] = 0.875 + return a + + +def exact16(v): + kind = decode_exact(np.float16(v).view(np.uint16)) + assert kind[0] == "num" + return kind[1] * kind[2] + + +class TestAlphaRule(unittest.TestCase): + CASES = [((27, 16), (40, 24), (0.13, -0.27)), ((27, 16), (40, 24), (-0.5, 0.5)), ((20, 12), (40, 24), (0.0, 0.0)), + ((85, 48), (128, 72), (0.31, 0.07)), ((13, 9), (48, 32), (-0.44, 0.21))] + + def test_exact_rational(self): + """alpha_reference's FP16 chain against exact rational arithmetic with one rounding per step.""" + rng = np.random.default_rng(3) + for render, output, jitter in self.CASES: + rw, rh = render + a = make_alpha(rng, rw, rh) + got = alpha_reference(a, render, output, jitter) + ty, tx, w = alpha_taps(render, output, jitter) + a16 = a.astype(np.float16) + H, W = got.shape + for _ in range(400): + y, x = int(rng.integers(H)), int(rng.integers(W)) + taps = [a16[ty[q, y, x], tx[q, y, x]] for q in range(4)] + bits = round_f16_exact(exact16(taps[0]) * exact16(w[0, y, x])) + for q in (1, 2, 3): + acc = exact16(np.uint16(bits).view(np.float16)) + sgn = -1 if bits & 0x8000 else 1 + bits = round_f16_exact(exact16(taps[q]) * exact16(w[q, y, x]) + acc, sign_if_zero=sgn) + want = np.uint16(bits).view(np.float16) + # (a zero sum's sign is not modelled exactly above: compare values, zeros of either sign equal) + self.assertTrue(got[y, x] == want, f"{render} {output} {jitter} ({x}, {y}): {got[y, x]} != {want}") + # the weights are those of a bilinear: each in [0, 1], and every tap inside the render rectangle + self.assertTrue(np.all((w >= 0) & (w <= 1))) + self.assertTrue(np.all((tx >= 0) & (tx < rw) & (ty >= 0) & (ty < rh))) + # both edges of the rectangle are reached (the clamp rule is exercised) + self.assertEqual((int(tx.min()), int(ty.min())), (0, 0)) + self.assertEqual((int(tx.max()), int(ty.max())), (rw - 1, rh - 1)) + + def _frame(self, rng, hdr, render, output, jitter, alpha, reset=False): + from clean.k.reference.layout import base_size + from clean.k.reference.outstage import ceil16 + (rw, rh), (W, H) = render, output + Bw, Bh = base_size(W, H) + W16, H16 = ceil16(W), ceil16(H) + color = rng.random((rh, rw, 4)).astype(np.float32) * (4 if hdr else 1) + color[rng.random((rh, rw)) < 0.03, 1] = -0.5 + if alpha is None: + color = color[..., :3] + else: + color[..., 3] = alpha + N = (rng.standard_normal((Bh, Bw, 40)) * 1.2).astype(np.float16) + return dict(N=N, color=color, motion=(rng.standard_normal((rh, rw, 2)) * 0.8).astype(np.float32), + depth=rng.random((rh, rw)).astype(np.float32), render_size=render, output_size=output, hdr=hdr, + mv_lowres=True, depth_inverted=False, jitter=jitter, mv_scale=(1.0, 0.9), mv_offset=(0.1, 0.0), + reset=reset, E_cur=1.5, E_prev=1.25, E_aux=1.0, e1=1.0, e2=1.1, pre_exposure=0.8, + L=(rng.random((2 * H16, 2 * W16)) * 1.05).astype(np.float16), + Hc=(rng.random((H16, W16, 4)) * 0.6).astype(np.float16), + Hh=(rng.standard_normal((Bh, Bw, 4)) * 0.3).astype(np.float16)) + + def test_against_spec43_reference(self): + """The fourth channel of clean/k/reference/outstage.py's output, bit for bit, HDR and LDR.""" + from clean.k.reference.outstage import output_stage + rng = np.random.default_rng(5) + for render, output, jitter in self.CASES[:4]: + for hdr in (True, False): + a = make_alpha(rng, *render) + a[1, 1] = 70000.0 # beyond FP16: RN16 gives +inf (inf times a zero weight is NaN, as IEEE says) + x = self._frame(rng, hdr, render, output, jitter, a) + out = output_stage(x)["out"] + want = alpha_reference(a, render, output, jitter) + ok = same16(out[..., 3].astype(np.float16), want) + self.assertTrue(ok.all(), f"{render} {output} hdr={hdr}: {int((~ok).sum())} alpha values differ") + # the reference's float32 fourth channel is the exact widening of the FP16 result + fin = np.isfinite(want) + np.testing.assert_array_equal(out[..., 3][fin], want.astype(np.float32)[fin]) + # no clamp: negative and above-1 alphas come out + self.assertLess(float(np.nanmin(want[fin])), -1.0) + self.assertGreater(float(np.nanmax(want[fin])), 2.0) + + def test_no_alpha_channel_is_one(self): + from clean.k.reference.outstage import output_stage + rng = np.random.default_rng(6) + render, output, jitter = self.CASES[0] + x = self._frame(rng, True, render, output, jitter, None) + out = output_stage(x)["out"] + want = alpha_reference(None, render, output, jitter) + self.assertTrue(same16(out[..., 3].astype(np.float16), want).all()) + # the four FP16 weights of a pixel sum to 1 up to FP16 rounding of the chain: alpha 1 comes out as 1 within + # one FP16 step below 1 (the rule stores the chain's result unchanged) + self.assertTrue(np.all((want >= np.float16(0.9990234375)) & (want <= np.float16(1.0)))) + + def test_spatial_only(self): + """Same colour alpha, everything else different (histories, network output, motion, depth, exposure, + pre-exposure, RGB, reset): the same output alpha. Over two chained frames with different alpha the second + frame's alpha is that of its own colour (no temporal carry).""" + from clean.k.reference.outstage import output_stage + rng = np.random.default_rng(7) + render, output, jitter = self.CASES[3] + a = make_alpha(rng, *render, t=0) + want = alpha_reference(a, render, output, jitter) + outs = [] + for k in range(3): + x = self._frame(rng, bool(k % 2), render, output, jitter, a, reset=k == 2) + x.update(E_cur=1.5 + k, E_prev=0.5 + k, pre_exposure=0.8 / (k + 1), e1=1.0 + 0.25 * k) + outs.append(output_stage(x)["out"][..., 3].astype(np.float16)) + self.assertTrue(same16(outs[-1], want).all(), f"variant {k}") + # two chained frames: frame 1 reads frame 0's histories; its alpha is its own colour's + x0 = self._frame(rng, True, render, output, jitter, a, reset=True) + o0 = output_stage(x0) + b = make_alpha(rng, *render, t=1) + self.assertGreater(int(np.count_nonzero(a != b)), a.size // 2) + x1 = self._frame(rng, True, render, output, (0.21, -0.4), b) + x1.update(L=o0["L"], Hc=o0["Hc"], Hh=o0["Hh"]) + o1 = output_stage(x1)["out"][..., 3].astype(np.float16) + self.assertTrue(same16(o1, alpha_reference(b, render, output, (0.21, -0.4))).all()) + self.assertFalse(same16(o1, alpha_reference(a, render, output, (0.21, -0.4))).all()) + + def test_fp32_carry_equals_fp16_carry(self): + """Carrying the alpha as FP32 and rounding at the tap gives the bits of carrying RN16 of it (the buffer forms' + FP32 colour against the FP16 colour of the raw-buffer route and the stored taps).""" + rng = np.random.default_rng(8) + render, output, jitter = self.CASES[3] + a = (make_alpha(rng, *render) * np.float32(1.000123)).astype(np.float32) # not FP16 values + self.assertGreater(int(np.count_nonzero(a.astype(np.float16).astype(np.float32) != a)), a.size // 2) + self.assertTrue(same16(alpha_reference(a, render, output, jitter), + alpha_reference(a.astype(np.float16), render, output, jitter)).all()) + + +class TestAlphaSources(unittest.TestCase): + def test_d3d12_output_kernel(self): + if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): + self.skipTest("the D3D12 input and output ports are not in this tree") + o, oi = read("k", "dx12", "kernels", "k_out.hlsl"), read("k", "dx12", "hlsl", "k_out.hlsli") + self.assertNotIn("KO_OUT_ALPHA0", o) + self.assertNotIn("KO_ALPHA", o) + # the stored tap's A: RN16 of the colour's fourth value; the bilinear: RN16(w0 a0), then three FP16 fmas + self.assertIn("tap = uint2(hY | (hCo << 16), hCg | (ko_f2h(c.w) << 16));", oi) + self.assertIn("hi = q == 0 ? kp_mul_s(t.y, w[q]) : kp_fma_s(t.y, w[q], hi);", o) + self.assertIn("cbar3 = kp_hi(hi);", o) + # every output form stores cbar3 unchanged + self.assertIn("ko_f2h(B) | (kh_u(cbar3) << 16)", o) # RGBA16F rows + self.assertIn("ko_store_out(x, y, R, G, B, cbar3);", o) # the direct texture + self.assertIn("native_direct_output[uint2((uint)x, (uint)y)] = float4(v, (float)A);", o) + self.assertIn("asuint(float4(R, G, B, (float)cbar3))", o) # dense RGBA f32 + # the direct colour read keeps the texel's alpha (not model M's nd_raw_color_xy, alpha +0) + tex = oi[oi.index("float4 kio_texel(int x, int y)"):oi.index("float ko_depth_at")] + self.assertIn("return native_direct_color_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0));", tex) + self.assertNotIn("return nd_raw_color_xy", tex) + # model M's shared header is untouched: its raw colour stays alpha +0 + nd = read("dx12", "hlsl", "native_direct.hlsli") + self.assertIn("return float4(native_direct_color_tex.Load(int3((int)nd_rx + x, (int)nd_ry + y, 0)).rgb, 0.0f);", nd) + + def test_hip_output_kernel(self): + k = read("k", "kernels", "k_io.hip") + self.assertNotIn("KIO_OUT_ALPHA", k) + self.assertIn("t.A = kh_round(c[3]);", k) + self.assertIn("const v4h o = {kh_round(R), kh_round(G), kh_round(B), cbar[3]};", k) + self.assertIn("o[3] = (float)cbar[3];", k) + self.assertIn("hi = q == 0 ? kh2_mul(vh, w) : kh2_fma(vh, w, hi);", k) + self.assertIn("out[3] = hi[1];", k) + + def test_runtime(self): + r, rh, core, api = (read("k", "runtime", "runtime_k.cpp"), read("k", "runtime", "runtime_k.h"), + read("runtime", "runtime_core.h"), read("runtime", "wg_runtime.h")) + self.assertIn("bool out_rgba32f = false;", core) + self.assertIn("} else if (io && io->out_rgba32f) {\n op.omode = OUT_RGBA32F;\n op.opitch = W_;", r) + self.assertIn("if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 16, true)) return false;", r) + # the test path: an RGB colour has alpha 1, an RGBA colour its own + self.assertIn("std::vector c(n * 4, 1.0f);", r) + self.assertIn("int channels = 3);", rh) + self.assertIn("WG_RT_API int wg_rt_upload_inputs_rgba(", api) + + def test_interop(self): + c, ch, b, eh, ex = (read("runtime", "interop", "convert.cpp"), read("runtime", "interop", "convert.h"), + read("runtime", "interop", "backend.cpp"), read("runtime", "interop", "executor_hip.cpp"), + read("runtime", "interop", "executor.h")) + shader = c[c.index('R"HLSL(') + 7:c.index(')HLSL"')] + # input passes: the texel's alpha for model K (flag 8), +0 for model M; the compact pass (model K only) keeps it + self.assertIn("float4 ct = colorTex.Load(p);\n colorBuf[i] = float4(ct.rgb, (flags & 8u) ? ct.a : 0.0);", shader) + self.assertIn("colorBuf[i] = colorTex.Load(p);", shader) + # output passes: model K (flag 4) carries the alpha, model M writes 1 + self.assertIn("uint i = (ka ? 4 : 3) * (id.y * ow + id.x);", shader) + self.assertIn("[branch] if (ka) a = resultBuf[i + 3];", shader) + self.assertIn("outTex[id.xy] = float4(c, (flags & 4u) ? t.a : 1.0);", shader) + self.assertIn("outTex[id.xy] = float4(c, (flags & 4u) ? b.a : 1.0);", shader) + self.assertNotIn("? 0.0 : 1.0", shader) + self.assertIn("constexpr uint32_t kMotionOutputRes = 1, kExposureTexture = 2;", c) + self.assertIn("constexpr uint32_t kOutputAlpha = 4, kInputAlpha = 8;", c) + self.assertIn("(model_k_ ? kInputAlpha : 0);", c) + self.assertEqual(c.count("c.flags = model_k_ ? kOutputAlpha : 0;"), 2) # record_output, record_output_tex + self.assertIn("d.Buffer.NumElements = (model_k_ ? 4 : 3) * f.output_width * f.output_height;", c) + self.assertIn("void set_model_k(bool on) { model_k_ = on; }", ch) + self.assertIn("bool model_k_ = false;", ch) # model M (and every other Converter user): as before + self.assertEqual(b.count("conv_.set_model_k("), 1) + self.assertIn("conv_.set_model_k(model_k_);", b) + self.assertIn("job.out_rgba32f = model_k_ && !halfOut;", b) + self.assertIn("sizes[4] = std::max(O * (model_k_ ? 16 : 12), po * oh_);", b) + self.assertIn("io.out_rgba32f = job.out_rgba32f;", eh) + self.assertIn("bool out_rgba32f = false;", ex) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_direct_motion.py b/clean/k/tests/test_k_direct_motion.py new file mode 100644 index 00000000..ef91c20c --- /dev/null +++ b/clean/k/tests/test_k_direct_motion.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: MIT +"""The direct (in-game, texture) forms of model K's input and output stages apply the motion scale and offset as the +buffer forms and the reference do (issue #26). Host checks (no GPU): + +- cause, reproduced in FP32: with a non-unit motion scale, the previous direct arithmetic (every texel scaled, RN(ms r) + + mo in the input stage, one fused mad(ms, r, mo) in the output stage, then the factor of scale 1) gives a different + warp position than the reference / buffer form (sp = RN(s / RN(RN(1 / n) r)) with the scale folded in, times + RN(r + b)) for a share of the raw values; the new direct arithmetic (the scale folded into the factor from b1's ms, + the offset RN(mo / ms) in raw units) gives the buffer form's bits for every value when mo = 0, and whenever RN(mo / ms) + recovers the buffer drive's raw-unit offset b (mo = RN(ms b)) otherwise; +- sources: the direct ko_motion / kio_motion return the raw texel, the direct factor / offset helpers mirror the host's + expressions (runtime_k.cpp out uniforms, k_input_uniforms.h, k_frame.hlsli ki_host), the buffer forms are unchanged; +- the full-head direct output stage k_out_pixel_direct is declared (the stage tool's --io direct --head full); +- the output alpha: the colour input's alpha through the current-colour bilinear (issue #26, test_k_alpha). + +Run: python -m unittest clean.k.tests.test_k_direct_motion +""" +import os +import re +import unittest + +import numpy as np + +from clean.k.reference.fp import F32, fma32 + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +K = os.path.join(ROOT, "clean", "k") + + +def read(*p): + with open(os.path.join(K, *p), encoding="utf-8") as f: + return f.read().replace("\r\n", "\n") + + +def rn_div(a, b): # correctly rounded FP32 division (wg_div / ko_div / the host's '/') + return (np.asarray(a, np.float32) / np.asarray(b, np.float32)).astype(np.float32) + + +def out_buffer(raw, px, s, b, W, rw): + """Output stage, buffer form: rhox = RN(rw RN(1 / W)), spx = RN(s / rhox), wx = fma(spx, RN(r + b), px).""" + rhox = F32(F32(rw) * F32(F32(1) / F32(W))) + spx = F32(F32(s) / rhox) + return fma32(spx, (raw + F32(b)).astype(np.float32), px) + + +def out_direct_old(raw, px, ms, mo, W, rw): + rhox = F32(F32(rw) * F32(F32(1) / F32(W))) + spx = F32(F32(1) / rhox) + return fma32(spx, fma32(F32(ms), raw, F32(mo)) + F32(0), px) + + +def out_direct_new(raw, px, ms, mo, W, rw): + rhox = F32(F32(rw) * F32(F32(1) / F32(W))) + spx = rn_div(F32(1) * F32(ms), rhox) + off = F32(0) if mo == 0 else F32(F32(0) + rn_div(mo, ms)) + return fma32(spx, (raw + off).astype(np.float32), px) + + +def in_buffer(raw, s, b, W, rw): + """Input stage, buffer form: sp = RN(s / RN(RN(1 / W) rw)), mx = RN(sp RN(r + b)).""" + sp = rn_div(s, F32(F32(F32(1) / F32(W)) * F32(rw))) + return (sp * (raw + F32(b)).astype(np.float32)).astype(np.float32) + + +def in_direct_old(raw, ms, mo, W, rw): + sp = rn_div(F32(1), F32(F32(F32(1) / F32(W)) * F32(rw))) + m = ((F32(ms) * raw).astype(np.float32) + F32(mo)).astype(np.float32) + return (sp * (m + F32(0))).astype(np.float32) + + +def in_direct_new(raw, ms, mo, W, rw): + sp = rn_div(F32(1) * F32(ms), F32(F32(F32(1) / F32(W)) * F32(rw))) + off = F32(0) if mo == 0 else F32(F32(0) + rn_div(mo, ms)) + return (sp * (raw + off).astype(np.float32)).astype(np.float32) + + +class TestKDirectMotion(unittest.TestCase): + CASES = [ # (render w, output W, scale, raw-unit offset b) + (1706, 2560, 1706.0, 0.0), (1280, 2560, 1280.0, 0.0), (340, 512, 0.3, 0.0), (340, 512, 0.7, 0.0), + (960, 1440, -960.0, 0.0), (1920, 3840, 0.5, 0.0), (340, 512, 0.3, 0.1), (192, 288, 0.7, -0.2)] + + def test_cause_and_fix_arithmetic(self): + rng = np.random.default_rng(7) + n = 200000 + old_diff_total = 0 + for rw, W, s, b in self.CASES: + # raw motion as a game's texture holds it: about +-40 render px / s, FP16 values (16-bit textures) + raw = (rng.standard_normal(n) * 40.0 / abs(s)).astype(np.float16).astype(np.float32) + px = (rng.integers(0, W, n) + 0.5).astype(np.float32) + mo = float(F32(F32(s) * F32(b))) # the interop's b1 offset: mo = RN(ms b) + ob, onew, oold = out_buffer(raw, px, s, b, W, rw), out_direct_new(raw, px, s, mo, W, rw), \ + out_direct_old(raw, px, s, mo, W, rw) + ib, inew, iold = in_buffer(raw, s, b, W, rw), in_direct_new(raw, s, mo, W, rw), in_direct_old(raw, s, mo, W, rw) + recovered = b == 0 or float(rn_div(mo, s)) == float(F32(b)) + if recovered: + self.assertTrue(np.array_equal(ob.view(np.uint32), onew.view(np.uint32)), (rw, W, s, b)) + self.assertTrue(np.array_equal(ib.view(np.uint32), inew.view(np.uint32)), (rw, W, s, b)) + if s not in (1.0, 0.5): # a power-of-two scale commutes with the roundings + old_diff_total += int(np.count_nonzero(ob != oold)) + int(np.count_nonzero(ib != iold)) + self.assertGreater(old_diff_total, 0) # the previous direct arithmetic differs (the cause) + + def test_scale_one_unchanged(self): + # ms = 1, mo = 0 (the executor's motion pass, output-size motion): the new direct arithmetic is the old one + rng = np.random.default_rng(3) + raw = (rng.standard_normal(50000) * 10).astype(np.float32) + px = (rng.integers(0, 2560, 50000) + 0.5).astype(np.float32) + self.assertTrue(np.array_equal(out_direct_new(raw, px, 1.0, 0.0, 2560, 1706), + out_direct_old(raw, px, 1.0, 0.0, 2560, 1706))) + self.assertTrue(np.array_equal(in_direct_new(raw, 1.0, 0.0, 2560, 1706), in_direct_old(raw, 1.0, 0.0, 2560, 1706))) + + def test_sources(self): + if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): + self.skipTest("the D3D12 input and output ports are not in this tree") + out_i, out_k = read("dx12", "hlsl", "k_out.hlsli"), read("dx12", "kernels", "k_out.hlsl") + fr, kin = read("dx12", "hlsl", "k_frame.hlsli"), read("dx12", "kernels", "k_in.hlsl") + # direct reads: the raw texel (no per-texel scale; not model M's nd_motion_xy) + self.assertNotIn("nd_motion_xy(", re.sub(r"//[^\n]*", "", out_i)) + for src in (out_i, fr): + self.assertIn("mvx = r.x;\n mvy = r.y;", src) + # factor / offset helpers mirror the host expressions + self.assertIn("float ko_dsp(float s, float ms, float rho, bool mvlo) { const float sm = s * ms; return mvlo ? ko_div(sm, rho) : sm; }", out_i) + self.assertIn("float ko_dmo(float o, float mo, float ms) { return mo == 0.0f ? o : o + ko_div(mo, ms); }", out_i) + self.assertIn("float ki_dsp(float sx, float ms, int n, int r) { return wg_div(sx * ms, wg_div(1.0f, (float)n) * (float)r); }", fr) + self.assertIn("float ki_dmo(float o, float mo, float ms) { return mo == 0.0f ? o : o + wg_div(mo, ms); }", fr) + self.assertIn("k.sp_x = wg_div(F.sx, wg_div(1.0f, (float)F.W) * (float)(mvlo ? F.rw : F.W));", fr) # ki_host + rt = read("runtime", "runtime_k.cpp") + self.assertIn("u.rhox = (float)rw * u.invW;", rt) + self.assertIn("u.spx = fin.sx / u.rhox; // [MVlo]", rt) + self.assertIn("k.sp_x = f.sx / ((1.0f / (float)f.W) * (float)((f.flags & 2) ? f.rw : f.W));", + read("runtime", "k_input_uniforms.h")) + # direct forms: b1's ms / mo folded (the KIO_M16 route shares the code with rmsx ..; test_k_m16_alpha) + for src in (out_k, kin): + self.assertIn("const bool dfold = true;", src) + self.assertIn("const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy;", src) + self.assertIn("spx = ko_dsp(f_sx, dmsx, u_rhox, mvlo);", out_k) + self.assertIn("ofx = ko_dmo(f_mox, dmox, dmsx);", out_k) + self.assertIn("const float wx = mad(spx, mvx + ofx, px), wy = mad(spy, mvy + ofy, py);", out_k) + self.assertIn("spx = ki_dsp(F.sx, dmsx, F.W, dmvlo ? F.rw : F.W);", kin) + self.assertIn("ofx = ki_dmo(F.mox, dmox, dmsx);", kin) + # the plain buffer form (no KIO_M16): the host's uniforms and WgRtFrame's offset + self.assertIn("float spx = u_spx, spy = u_spy, ofx = f_mox, ofy = f_moy;", out_k) + self.assertIn("float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy;", kin) + + def test_full_head_direct_port_and_alpha(self): + if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): + self.skipTest("the D3D12 input and output ports are not in this tree") + out_k = read("dx12", "kernels", "k_out.hlsl") + line = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_direct ")] + self.assertEqual(len(line), 1) + for need in ("entry=k_out_pixel_c", "rs=direct", "KO_COMPACT=0", "WG_DIRECT_TEXTURES=1"): + self.assertIn(need, line[0]) + c = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_c_direct ")][0] + self.assertEqual(c.replace("k_out_pixel_c_direct", "X").replace("KO_COMPACT=1", "C"), + line[0].replace("k_out_pixel_direct", "X").replace("KO_COMPACT=0", "C")) + self.assertIn('"k_out_pixel_direct"', read("equiv", "k_stages.py")) + # the output alpha is cbar3 (the colour input's alpha, issue #26) in both head forms: no constant alpha + self.assertNotIn("KO_OUT_ALPHA0", out_k) + self.assertIn("native_direct_output[uint2((uint)x, (uint)y)] = float4(v, (float)A);", out_k) + self.assertIn("ko_f2h(B) | (kh_u(cbar3) << 16)", out_k) + self.assertEqual(len(re.findall(r"float4\(v, [01]\.0f\)|0x3c00u << 16|0x0000u << 16", out_k)), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_m16_alpha.py b/clean/k/tests/test_k_m16_alpha.py new file mode 100644 index 00000000..eaf07e0f --- /dev/null +++ b/clean/k/tests/test_k_m16_alpha.py @@ -0,0 +1,146 @@ +# SPDX-License-Identifier: MIT +"""Model K: the interop's raw-buffer motion route (KIO_M16) applies the motion scale and offset as the buffer forms and +the reference do (issue #26). Host checks (no GPU, no HIP run): + +- KIO_M16 arithmetic, in FP32: the previous form (every texel ms r + mo, two roundings, then the factor of scale 1 that + WgRtFrame's mv_scale 1 gives) differs from the reference / buffer form (sp = RN(s / RN(RN(1/n) r)) with the scale + folded in, times RN(r + b)) for a share of the raw values; the new form (the raw texel, the scale folded into the + factor, the offset RN(mo / ms) in raw units) gives the buffer form's bits whenever that offset is recovered (always + for mo = 0). The same FP32 operations in HIP (k_io.hip, contract off, correctly rounded division) and D3D12; +- sources: HIP k_io.hip and the D3D12 ports read the raw FP16 texel on KIO_M16 and fold ms / mo with the host's + expressions at both use sites (input ki_fine, output k_out_pixel); +- the Converter's run-time HLSL compiles with d3dcompiler_47 (cs_5_0), every pass. (The output alpha follows the colour input's alpha, issue #26; its checks are + clean/k/tests/test_k_alpha.py.) + +Run: python -m unittest clean.k.tests.test_k_m16_alpha +""" +import ctypes +import os +import re +import sys +import unittest + +import numpy as np + +from clean.k.reference.fp import F32, fma32 +from clean.k.tests.test_k_direct_motion import (in_buffer, in_direct_new, in_direct_old, out_buffer, out_direct_new, + rn_div) + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +CLEAN = os.path.join(ROOT, "clean") + + +def read(*p): + with open(os.path.join(CLEAN, *p), encoding="utf-8") as f: + return f.read().replace("\r\n", "\n") + + +def out_m16_old(raw, px, ms, mo, W, rw): + """Output stage, previous KIO_M16 form: m = RN(RN(ms r) + mo) per texel (HIP contract off: two roundings), the factor + of WgRtFrame's scale 1: spx = RN(1 / rhox), wx = fma(spx, RN(m + 0), px).""" + rhox = F32(F32(rw) * F32(F32(1) / F32(W))) + spx = F32(F32(1) / rhox) + m = ((F32(ms) * raw).astype(np.float32) + F32(mo)).astype(np.float32) + return fma32(spx, (m + F32(0)).astype(np.float32), px) + + +def in_m16_old(raw, ms, mo, W, rw): + return in_direct_old(raw, ms, mo, W, rw) # the same two roundings, then sp of scale 1 + + +class TestKM16Motion(unittest.TestCase): + CASES = [(1706, 2560, 1706.0, 0.0), (1280, 2560, 1280.0, 0.0), (340, 512, 0.3, 0.0), (340, 512, 0.7, 0.0), + (960, 1440, -960.0, 0.0), (1920, 3840, 0.5, 0.0), (340, 512, 0.3, 0.1), (192, 288, 0.7, -0.2)] + + def test_arithmetic(self): + rng = np.random.default_rng(11) + n = 200000 + old_diff = 0 + for rw, W, s, b in self.CASES: + # raw FP16 motion as the route's RG16F copies hold it + raw = (rng.standard_normal(n) * 40.0 / abs(s)).astype(np.float16).astype(np.float32) + px = (rng.integers(0, W, n) + 0.5).astype(np.float32) + mo = float(F32(F32(s) * F32(b))) + ob, ib = out_buffer(raw, px, s, b, W, rw), in_buffer(raw, s, b, W, rw) + if b == 0 or float(rn_div(mo, s)) == float(F32(b)): + # the new KIO_M16 form is the direct forms' arithmetic with (rmsx, rmox) for (ms, mo) + self.assertTrue(np.array_equal(ob.view(np.uint32), out_direct_new(raw, px, s, mo, W, rw).view(np.uint32))) + self.assertTrue(np.array_equal(ib.view(np.uint32), in_direct_new(raw, s, mo, W, rw).view(np.uint32))) + old_diff += int(np.count_nonzero(ob != out_m16_old(raw, px, s, mo, W, rw))) + old_diff += int(np.count_nonzero(ib != in_m16_old(raw, s, mo, W, rw))) + self.assertGreater(old_diff, 0) + + def test_hip_sources(self): + h = read("k", "kernels", "k_io.hip") + self.assertIn("#pragma clang fp contract(off)", h) + self.assertIn("WG_FN float kio_m16_sp(float s, float ms, float den) { return (s * ms) / den; }", h) + self.assertIn("WG_FN float kio_m16_mo(float o, float mo, float ms) { return mo == 0.0f ? o : o + mo / ms; }", h) + # raw reads on KIO_M16 (input kio_motion, output ko_motion): no per-texel scale left + self.assertEqual(len(re.findall(r"\*mvx = \(float\)r\[0\];", h)), 2) + self.assertNotIn("F.rmsx * (float)r[0]", h) + # input: ki_host's sp_x with s = sx ms; output: out_uniforms' spx with s = sx ms + self.assertIn("k.sp_x = F.sx / ((1.0f / (float)F.W) * (float)(mvlo ? F.rw : F.W));", h) + self.assertIn("spx = kio_m16_sp(F.sx, F.rmsx, (1.0f / (float)F.W) * (float)(mvlo ? F.rw : F.W));", h) + self.assertIn("ofx = kio_m16_mo(F.mox, F.rmox, F.rmsx);", h) + self.assertIn("const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy);", h) + self.assertIn("spx = mvlo ? kio_m16_sp(F.sx, F.rmsx, U.rhox) : F.sx * F.rmsx;", h) + self.assertIn("const float wx = kf_fma(spx, mvx + ofx, px), wy = kf_fma(spy, mvy + ofy, py);", h) + rt = read("k", "runtime", "runtime_k.cpp") + self.assertIn("u.spx = fin.sx / u.rhox; // [MVlo]", rt) + self.assertIn("u.rhox = (float)rw * u.invW;", rt) + + def test_d3d12_sources(self): + if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): + self.skipTest("the D3D12 input and output ports are not in this tree") + o, oi = read("k", "dx12", "kernels", "k_out.hlsl"), read("k", "dx12", "hlsl", "k_out.hlsli") + fr, ki = read("k", "dx12", "hlsl", "k_frame.hlsli"), read("k", "dx12", "kernels", "k_in.hlsl") + self.assertNotIn("f_rmsx * r0", oi) + self.assertNotIn("F.rmsx * wg_h2f_lo(r)", fr) + self.assertIn("mvx = wg_h2f(wg_ld16(f_motion, i * 2u));", oi) + self.assertIn("mvx = wg_h2f_lo(r);", fr) + self.assertIn("const bool dfold = (f_flags & KIO_M16) != 0;", o) + self.assertIn("const float dmsx = f_rmsx, dmsy = f_rmsy, dmox = f_rmox, dmoy = f_rmoy;", o) + self.assertIn("const float wx = mad(spx, mvx + ofx, px), wy = mad(spy, mvy + ofy, py);", o) + self.assertIn("const bool dfold = (F.flags & KIO_M16) != 0;", ki) + self.assertIn("const float dmsx = F.rmsx, dmsy = F.rmsy, dmox = F.rmox, dmoy = F.rmoy;", ki) + self.assertIn("const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy);", ki) + + +class TestConverterShader(unittest.TestCase): + @unittest.skipUnless(sys.platform == "win32", "d3dcompiler_47 (Windows)") + def test_converter_hlsl_compiles(self): + c = read("runtime", "interop", "convert.cpp") + shader = c[c.index('R"HLSL(') + 7:c.index(')HLSL"')].encode() + d3dc = ctypes.WinDLL("d3dcompiler_47.dll") + + class Macro(ctypes.Structure): + _fields_ = [("Name", ctypes.c_char_p), ("Definition", ctypes.c_char_p)] + + passes = dict(re.findall(r'"(\w+)", "(PASS_\w+)"', c.replace("\n", " "))) + entries = re.search(r"const char\* entries\[\d+\] = \{([^}]*)\}", c).group(1) + names = re.findall(r'"(\w+)"', entries) + pass_defs = re.findall(r'"(PASS_\w+)"', re.search(r"const char\* passes\[\d+\] = \{([^}]*)\}", c).group(1)) + todo = list(zip(names, pass_defs)) + [("outtex", "PASS_OUTTEX")] + self.assertIn(("output", "PASS_OUTPUT"), todo) + self.assertIn(("fallback", "PASS_FALLBACK"), todo) + for entry, define in todo: + macros = (Macro * 2)(Macro(define.encode(), b"1"), Macro(None, None)) + code, err = ctypes.c_void_p(), ctypes.c_void_p() + hr = d3dc.D3DCompile(shader, len(shader), b"convert", macros, None, entry.encode(), b"cs_5_0", + ctypes.c_uint(1 << 15), 0, ctypes.byref(code), ctypes.byref(err)) + msg = "" + if err.value: + vt = ctypes.cast(ctypes.cast(err, ctypes.POINTER(ctypes.c_void_p))[0], ctypes.POINTER(ctypes.c_void_p)) + ptr = ctypes.WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)(vt[3])(err) + size = ctypes.WINFUNCTYPE(ctypes.c_size_t, ctypes.c_void_p)(vt[4])(err) + msg = ctypes.string_at(ptr, size).decode(errors="replace") + ctypes.WINFUNCTYPE(ctypes.c_ulong, ctypes.c_void_p)(vt[2])(err) + if code.value: + vt = ctypes.cast(ctypes.cast(code, ctypes.POINTER(ctypes.c_void_p))[0], ctypes.POINTER(ctypes.c_void_p)) + ctypes.WINFUNCTYPE(ctypes.c_ulong, ctypes.c_void_p)(vt[2])(code) + self.assertEqual(hr, 0, f"{entry} ({define}): {msg}") + _ = passes + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_route3_motion.py b/clean/k/tests/test_k_route3_motion.py new file mode 100644 index 00000000..786a35a6 --- /dev/null +++ b/clean/k/tests/test_k_route3_motion.py @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: MIT +"""Model K on the interop's FP32 input-conversion pass (the HIP executor's route when the colour or motion texture is +not FP16): the pass writes the raw motion texels for model K and the backend passes motion_scale and the raw-unit +offset in WgRtFrame, so K folds the scale into its motion factor as the reference / buffer form (issue #26). +Host checks (no GPU, no HIP): + +- arithmetic, in FP32: the previous route (the pass's per-texel ms raw + mo, as one mad or two roundings, then K with + scale 1) differs from the buffer form for a share of the raw values; the new route (the pass with scale 1, offset 0: + mad(1, raw, 0); K with mv_scale ms, mv_bias RN(mo / ms)) gives the buffer form's bits on every value (signed zeros, + infinities and NaN included: 1 raw is exact, + 0 maps -0 to +0, which K's raw + b maps to the same value); +- sources: the Converter's run-time HLSL carries the alpha for model K only (issue #26: every change is under a flag + only model K sets or in a pass only model K runs; model M's passes compute what they did, with the same constants), + the raw-motion constants are model K's only, and the backend's K-only + branch; +- the HIP executor job's WgRtFrame reaches K's frame input (fin.sx = mv_scale, fin.mox = mv_bias). + +Run: python -m unittest clean.k.tests.test_k_route3_motion +""" +import os +import subprocess +import unittest + +import numpy as np + +from clean.k.reference.fp import F32, fma32 +from clean.k.tests.test_k_direct_motion import in_buffer, out_buffer, rn_div + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +CLEAN = os.path.join(ROOT, "clean") + + +def read(*p): + with open(os.path.join(CLEAN, *p), encoding="utf-8") as f: + return f.read().replace("\r\n", "\n") + + +def k_out(m, px, s, b, W, rw): # K's output stage on the dense motion m with WgRtFrame (s, b): the buffer form + return out_buffer(m, px, s, b, W, rw) + + +def k_in(m, s, b, W, rw): + return in_buffer(m, s, b, W, rw) + + +class TestKRoute3Motion(unittest.TestCase): + CASES = [(1706, 2560, 1706.0, 0.0), (1280, 2560, 1280.0, 0.0), (340, 512, 0.3, 0.0), (340, 512, 0.7, 0.0), + (960, 1440, -960.0, 0.0), (1920, 3840, 0.5, 0.0), (340, 512, 0.3, 0.1), (192, 288, 0.7, -0.2)] + + def test_arithmetic(self): + rng = np.random.default_rng(23) + n = 200000 + old_diff = 0 + for rw, W, s, b in self.CASES: + # raw FP32 motion (RG32F textures: the FP32 route), with signed zeros, infinities and NaN mixed in + raw = (rng.standard_normal(n) * 40.0 / abs(s)).astype(np.float32) + raw[:64] = -0.0 + raw[64:72] = np.float32(np.inf) + raw[72:80] = -np.float32(np.inf) + raw[80:88] = np.float32(np.nan) + px = (rng.integers(0, W, n) + 0.5).astype(np.float32) + mo = float(F32(F32(s) * F32(b))) # the game's offset in render pixels after the scale + bk = 0.0 if mo == 0 else float(rn_div(mo, s)) # the backend's raw-unit offset RN(mo / ms) + # new route: the pass with scale 1, offset 0 (one mad; 1 raw exact), then K with (ms, bk) + m_new = fma32(F32(1), raw, F32(0)) + with np.errstate(invalid="ignore"): + ob, ib = k_out(raw, px, s, bk, W, rw), k_in(raw, s, bk, W, rw) + on, inn = k_out(m_new, px, s, bk, W, rw), k_in(m_new, s, bk, W, rw) + self.assertTrue(np.array_equal(ob.view(np.uint32), on.view(np.uint32)), (rw, W, s, b)) + self.assertTrue(np.array_equal(ib.view(np.uint32), inn.view(np.uint32)), (rw, W, s, b)) + if b == 0 or bk == float(F32(b)): # and the buffer drive's own b (when RN(mo / ms) recovers it) + self.assertTrue(np.array_equal(out_buffer(raw, px, s, b, W, rw).view(np.uint32), on.view(np.uint32))) + # old route: the pass's per-texel conversion (one mad, d3dcompiler's msx * m + mox; or two roundings), + # then K with scale 1, offset 0 + fin = np.isfinite(raw) + for m_old in (fma32(F32(s), raw, F32(mo)), ((F32(s) * raw).astype(np.float32) + F32(mo)).astype(np.float32)): + old_diff += int(np.count_nonzero((ob != k_out(m_old, px, 1.0, 0.0, W, rw)) & fin)) + old_diff += int(np.count_nonzero((ib != k_in(m_old, 1.0, 0.0, W, rw)) & fin)) + self.assertGreater(old_diff, 0) + + # issue #26 (the output alpha follows the colour input's alpha): the only changes of the run-time HLSL, as (old text, new text, occurrences). Flags 4 / 8 are set for model K only (Converter::set_model_k); + # the compact pass inputs16 is model K's own. + ALPHA_CARRY = [ + (" colorBuf[i] = float4(colorTex.Load(p).rgb, 0.0);\n depthBuf[i] = depthTex.Load(p);\n int3 mp = p;\n" + " if (flags & 1u) { // motion at output size: the texel under the render pixel's centre\n" + " mp = int3(min(uint((id.x + 0.5) * mw / rw), mw - 1), min(uint((id.y + 0.5) * mh / rh), mh - 1), 0);\n" + " }\n float2 m = motionTex.Load(mp).xy;\n", + " // flags & 8u (model K): the texel's alpha, unmodified, as the colour's fourth value (model K's output alpha " + "follows\n // it; a format without an alpha channel loads 1). Else +0 (model M).\n" + " float4 ct = colorTex.Load(p);\n colorBuf[i] = float4(ct.rgb, (flags & 8u) ? ct.a : 0.0);\n" + " depthBuf[i] = depthTex.Load(p);\n int3 mp = p;\n" + " if (flags & 1u) { // motion at output size: the texel under the render pixel's centre\n" + " mp = int3(min(uint((id.x + 0.5) * mw / rw), mw - 1), min(uint((id.y + 0.5) * mh / rh), mh - 1), 0);\n" + " }\n float2 m = motionTex.Load(mp).xy;\n", 1), + ("// exposure as in inputs. The network side applies m = ms raw + mo when it reads the motion.\n", + "// exposure as in inputs. The colour keeps the texel's alpha, unmodified (model K's output alpha follows it; " + "R11G11B10\n// has none and loads 1). The network side applies m = ms raw + mo when it reads the motion.\n", 1), + (" colorBuf[i] = float4(colorTex.Load(p).rgb, 0.0);\n", " colorBuf[i] = colorTex.Load(p);\n", 1), + (" uint i = 3 * (id.y * ow + id.x);\n float3 c = float3(resultBuf[i], resultBuf[i + 1], resultBuf[i + 2]);\n" + " if (srgb) c = to_srgb(c);\n outTex[id.xy] = float4(c, (flags & 4u) ? 0.0 : 1.0);\n", + " // flags & 4u (model K): the result holds 4 x f32 per pixel, the fourth being the output alpha (the colour " + "input's\n // alpha through model K's output stage), stored as it is. Else 3 x f32 and alpha 1 (model M).\n" + " bool ka = (flags & 4u) != 0;\n uint i = (ka ? 4 : 3) * (id.y * ow + id.x);\n" + " float3 c = float3(resultBuf[i], resultBuf[i + 1], resultBuf[i + 2]);\n float a = 1.0;\n" + " [branch] if (ka) a = resultBuf[i + 3];\n if (srgb) c = to_srgb(c);\n outTex[id.xy] = float4(c, a);\n", 1), + (" float3 c = proxyTex.Load(int3(id.xy, 0)).rgb;\n if (srgb) c = to_srgb(c);\n" + " outTex[id.xy] = float4(c, (flags & 4u) ? 0.0 : 1.0);\n", + " float4 t = proxyTex.Load(int3(id.xy, 0));\n float3 c = t.rgb;\n if (srgb) c = to_srgb(c);\n" + " outTex[id.xy] = float4(c, (flags & 4u) ? t.a : 1.0); // model K: the result texture's alpha, as it is\n", 1), + (" float3 c00 = colorTex.Load(int3(p0.x, p0.y, 0)).rgb, c10 = colorTex.Load(int3(p1.x, p0.y, 0)).rgb;\n" + " float3 c01 = colorTex.Load(int3(p0.x, p1.y, 0)).rgb, c11 = colorTex.Load(int3(p1.x, p1.y, 0)).rgb;\n" + " float3 c = lerp(lerp(c00, c10, f.x), lerp(c01, c11, f.x), f.y);\n if (srgb) c = to_srgb(c);\n" + " outTex[id.xy] = float4(c, (flags & 4u) ? 0.0 : 1.0);\n", + " float4 c00 = colorTex.Load(int3(p0.x, p0.y, 0)), c10 = colorTex.Load(int3(p1.x, p0.y, 0));\n" + " float4 c01 = colorTex.Load(int3(p0.x, p1.y, 0)), c11 = colorTex.Load(int3(p1.x, p1.y, 0));\n" + " float4 b = lerp(lerp(c00, c10, f.x), lerp(c01, c11, f.x), f.y);\n float3 c = b.rgb;\n" + " if (srgb) c = to_srgb(c);\n" + " // model K (flags & 4u): the colour's alpha through the same bilinear (a frame without a network result " + "follows the\n // colour input's alpha too); else alpha 1 (model M)\n" + " outTex[id.xy] = float4(c, (flags & 4u) ? b.a : 1.0);\n", 1), + ] + + def test_converter_hlsl_carries_the_alpha_for_model_k_only(self): + """The run-time HLSL has each alpha-carry text above (every one under a flag only model K sets, or in a pass only + model K runs), none of the texts it replaces (model M's constants: colour alpha +0 in, alpha 1 out), and the + motion pass still computes m = ms raw + mo.""" + c = read("runtime", "interop", "convert.cpp") + shader = c[c.index('R"HLSL(') + 7:c.index(')HLSL"')] + for old, new, n in self.ALPHA_CARRY: + self.assertEqual(shader.count(new), n, new) + self.assertNotIn(old, shader) + self.assertIn("motionBuf[i] = float2(msx * m.x + mox, msy * m.y + moy);", shader) + + def test_sources(self): + c, h, b = read("runtime", "interop", "convert.cpp"), read("runtime", "interop", "convert.h"), \ + read("runtime", "interop", "backend.cpp") + for line in ("c.msx = raw_motion ? 1.0f : f.motion_scale[0];", "c.msy = raw_motion ? 1.0f : f.motion_scale[1];", + "c.mox = raw_motion ? 0.0f : f.motion_offset[0];", "c.moy = raw_motion ? 0.0f : f.motion_offset[1];", + "record_inputs_pass(list, f, create_flags, b, false, raw_motion);"): + self.assertIn(line, c) + self.assertIn("const InputBuffers& b, bool compact, bool raw_motion = false);", h) + self.assertIn("bool raw_motion = false);", h) + self.assertIn("const bool kraw = model_k_ && f.motion_scale[0] != 0.0f && f.motion_scale[1] != 0.0f;", b) + self.assertIn("job.frame.mv_scale[k] = f.motion_scale[k];", b) + self.assertIn("job.frame.mv_bias[k] = f.motion_offset[k] == 0.0f ? 0.0f : f.motion_offset[k] / f.motion_scale[k];", b) + self.assertIn("conv_.record_inputs(list, f, flags_, in, kraw);", b) + self.assertEqual(b.count("conv_.record_inputs("), 1) + # the frame the HIP executor passes reaches K's frame input + rt = read("k", "runtime", "runtime_k.cpp") + self.assertIn("fin.sx = fr.mv_scale[0];", rt) + self.assertIn("fin.mox = fr.mv_bias[0];", rt) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_runtime_gpu.py b/clean/k/tests/test_k_runtime_gpu.py index 120b1b27..cadb990e 100644 --- a/clean/k/tests/test_k_runtime_gpu.py +++ b/clean/k/tests/test_k_runtime_gpu.py @@ -11,8 +11,9 @@ and a texel with auto-exposure on, with the exposure rule of spec 42 draft 3 section 4 (clean/k/reference/exposure.py: E_cur = the game's exposure or 1, E_aux = the auto-exposure value; KFrame is given the rule's inputs and the runtime's measured value, and the runtime's [E_cur, E_prev, E_aux] are checked), pre-exposure != 1, both - output layouts of the interop (dense RGB float32; RGBA16F rows with alpha 1) and the input-copy path (raw RGBA16F - colour, R32F depth, FP16 motion converted on the GPU). + output layouts of the interop (dense RGB float32; RGBA16F rows, whose alpha is the colour input's alpha through the + output stage's bilinear, issue #26) and the input-copy path (raw RGBA16F + colour, R32F depth, FP16 motion with ms / mo folded into the motion factor on the GPU). - The input stage: exact (k_io_exact.co) by default, the fast one (k_io.co) with k_fast_in=1. - 1440p Quality (2560 x 1440 from 1706 x 960): the runtime's timing marks (as timing_log=1 in game) per part. @@ -271,8 +272,11 @@ def __del__(self): def inputs(rng, rw, rh, hdr, reset=False, jitter=None): - color = np.zeros((rh, rw, 4), np.float32) # alpha 0, as the interop's input pass writes it + color = np.zeros((rh, rw, 4), np.float32) color[..., :3] = rng.random((rh, rw, 3)) * (6.0 if hdr else 1.0) + # the colour's alpha, as the interop's input passes carry it for model K (its own generator: the other values of + # the sequences stay what they were); negative and above-1 values included + color[..., 3] = np.random.default_rng(1000 * rw + rh).uniform(-1.0, 2.0, (rh, rw)).astype(np.float32) color[rng.random((rh, rw)) < 0.01, 0] = -0.25 # negative values (max(RGB, 0) in specs 42, 43) return dict(color=color, motion=(rng.standard_normal((rh, rw, 2)) * 1.5).astype(np.float32), depth=rng.random((rh, rw)).astype(np.float32), render_size=(rw, rh), @@ -311,7 +315,7 @@ def run_sequence(self, flags, frames, out_size=(200, 120), max_render=(150, 90), if i == 2: # a motion scale and offset (the runtime passes WgRtFrame.mv_scale / mv_bias to K) x["mv_scale"], x["mv_offset"] = (0.75, 1.25), (0.125, -0.25) rawio = None - if raw: # the game's textures as the interop copies them: RGBA16F colour, FP16 motion, m = ms raw + mo + if raw: # the game's textures as the interop copies them: RGBA16F colour, FP16 motion with ms, mo x["color"] = x["color"].astype(np.float16).astype(np.float32) cp = pitch256(8 * rw) // 2 crow = np.zeros((rh, cp), np.float16) @@ -329,6 +333,14 @@ def run_sequence(self, flags, frames, out_size=(200, 120), max_render=(150, 90), runs = flags["hdr"] and (auto or mode == 2) measured = float(rt.buffer("exposure.measured", np.float32)[0]) if runs else None kx = {k: v for k, v in x.items() if k not in ("E_cur", "E_aux", "e1", "e2")} + if raw: # KIO_M16 (issue #26): the raw motion with the scale folded into the factor and the + # offset in raw units, as the runtime applies io.ms / io.mo with WgRtFrame's scale / offset: the + # reference's mv_scale RN(sx ms), mv_offset RN(ox + RN(mo / ms)) (mo != 0), on the raw texels + f32 = np.float32 + kx["motion"] = mraw[..., :2].astype(np.float32) + kx["mv_scale"] = tuple(float(f32(f32(s) * ms[c])) for c, s in enumerate(x["mv_scale"])) + kx["mv_offset"] = tuple(float(f32(f32(o) + f32(mo[c] / ms[c]))) if mo[c] != 0 else float(o) + for c, o in enumerate(x["mv_offset"])) kx.update(game_exposure=None if mode == 2 else x["E_cur"], measured=measured) want = kf.frame(kx) # the runtime's exposure state [E_cur, E_prev, E_aux] (raw values) follows the rule @@ -343,7 +355,11 @@ def run_sequence(self, flags, frames, out_size=(200, 120), max_render=(150, 90), if half: np.testing.assert_array_equal(got.astype(np.float16).view(np.uint16), ref.astype(np.float16).view(np.uint16), err_msg=f"frame {info}") - self.assertTrue(np.all(alpha == 1.0)) + # the output alpha: the colour input's alpha through the 4.3 bilinear (issue #26), the + # reference's fourth channel, as it is + np.testing.assert_array_equal(alpha.astype(np.float16).view(np.uint16), + want["out"][..., 3].astype(np.float16).view(np.uint16), + err_msg=f"alpha frame {info}") else: np.testing.assert_array_equal(got.view(np.uint32), ref.view(np.uint32), err_msg=f"frame {info}") self.assertTrue(np.all(np.isfinite(got)), f"frame {info}") diff --git a/clean/runtime/interop/backend.cpp b/clean/runtime/interop/backend.cpp index ba16de34..1323abfc 100644 --- a/clean/runtime/interop/backend.cpp +++ b/clean/runtime/interop/backend.cpp @@ -780,7 +780,8 @@ class Context { } if (model_k_) { say(WG_LOG_INFO, std::string("network model K (runtime option model=K), ") + - ((flags_ & WG_CREATE_HDR) ? "HDR" : "LDR") + " colour; output alpha 1" + + ((flags_ & WG_CREATE_HDR) ? "HDR" : "LDR") + + " colour; output alpha: the colour input's alpha (1 for a colour format without one)" + ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) ? "; motion at output size is resampled to render pixels by the input pass " "(model K runs its render-resolution motion variant on it)" @@ -952,7 +953,19 @@ class Context { } else if (!copied && model_k_ && k_input16_) { note_path(&input_path_, "input: conversion pass (model K compact pass needs FP16 color and motion)"); } - if (!copied) conv_.record_inputs(list, f, flags_, in); + if (!copied) { + // model K (issue #26): the FP32 pass writes the raw motion texels and K gets motion_scale and the + // raw-unit offset RN(motion_offset / motion_scale) in WgRtFrame, so that it folds the scale into its motion factor + // as the reference and its buffer forms do (the pass's per-texel ms raw + mo with scale 1 rounded differently). + // A zero scale has no raw-unit offset: those frames keep the pass's conversion. Model M: unchanged. + const bool kraw = model_k_ && f.motion_scale[0] != 0.0f && f.motion_scale[1] != 0.0f; + if (kraw) + for (int k = 0; k < 2; ++k) { + job.frame.mv_scale[k] = f.motion_scale[k]; + job.frame.mv_bias[k] = f.motion_offset[k] == 0.0f ? 0.0f : f.motion_offset[k] / f.motion_scale[k]; + } + conv_.record_inputs(list, f, flags_, in, kraw); + } if (motion_hi_ && slot.motion_hi.resource) { // the raw output-size motion for the network (or the capture) conv_.record_motion_hi(list, f, {slot.motion_hi.resource, slot.motion_hi.offset}, ow_, oh_); job.motion_hi = true; @@ -1016,6 +1029,7 @@ class Context { std::string whyNot; const bool halfOut = output_copy_ && Converter::output_copy_ok(f, &whyNot); job.out_half_pitch = halfOut ? Converter::half_rgba_pitch(ow_) : 0; + job.out_rgba32f = model_k_ && !halfOut; // model K's result for the output pass: RGBA f32 (the alpha with it) job.ow = (int)ow_; job.oh = (int)oh_; // 3. the network, enqueued now; GPU-side wait for ready >= v and signal done = v @@ -1398,6 +1412,8 @@ class Context { #endif model_l = model == wgrt::ModelId::L; } + // model K: the output alpha follows the colour input's alpha (issue #26): the passes carry it; model M's 1 + conv_.set_model_k(model_k_); { // model K's input path: k_input=16 (default: the compact pass where exact) or f32 (the FP32 input pass) const std::string ki = option_value(o.runtime, "k_input"); if (!ki.empty() && ki != "16" && ki != "f32") return *err = "unknown k_input " + ki + " (16 or f32)", false; @@ -1576,7 +1592,7 @@ class Context { sizes[1] = std::max(R * 4, p4 * max_rh_); sizes[2] = std::max(R * 8, p8 * max_rh_); sizes[3] = 256; - sizes[4] = std::max(O * 12, po * oh_); + sizes[4] = std::max(O * (model_k_ ? 16 : 12), po * oh_); // model K: RGBA f32 for the output pass (Job.out_rgba32f) // motion at output resolution: O texels of two FP16 and 8 spare bytes (kernels/wg_motion_hi.h); 0 = no buffer. // It comes last in a slot and takes no room when absent, so every other offset is what it was. sizes[5] = motion_hi_ ? O * 4 + 8 : 0; diff --git a/clean/runtime/interop/convert.cpp b/clean/runtime/interop/convert.cpp index f055df8c..e4f63a88 100644 --- a/clean/runtime/interop/convert.cpp +++ b/clean/runtime/interop/convert.cpp @@ -15,6 +15,11 @@ namespace { constexpr UINT kSetSize = 8; // t0..t3, u0..u3 constexpr UINT kRingSets = 512; // descriptor sets before reuse (far more than frames in flight x 3) constexpr uint32_t kMotionOutputRes = 1, kExposureTexture = 2; +// Model K (Converter::set_model_k; issue #26: its output alpha follows the colour input's alpha). +// kOutputAlpha, output passes: the alpha comes with the result (output: the fourth f32 of each result pixel; outtex: the +// result texture's alpha; fallback: the bilinear of the colour's alpha), else alpha 1 (model M). +// kInputAlpha, input pass: the colour texel's alpha goes into the dense colour unmodified, else +0 (model M). +constexpr uint32_t kOutputAlpha = 4, kInputAlpha = 8; constexpr UINT kOptimizationLevel3 = 1u << 15; // D3DCOMPILE_OPTIMIZATION_LEVEL3 (d3dcompiler.h) // Written for WrongGPU from the definitions in spec 03 / spec 20 and clean/FORMATS.md. @@ -77,7 +82,10 @@ void inputs(uint3 id : SV_DispatchThreadID) { if (id.x >= rw || id.y >= rh) return; int3 p = int3(rx + id.x, ry + id.y, 0); uint i = id.y * rw + id.x; - colorBuf[i] = float4(colorTex.Load(p).rgb, 0.0); + // flags & 8u (model K): the texel's alpha, unmodified, as the colour's fourth value (model K's output alpha follows + // it; a format without an alpha channel loads 1). Else +0 (model M). + float4 ct = colorTex.Load(p); + colorBuf[i] = float4(ct.rgb, (flags & 8u) ? ct.a : 0.0); depthBuf[i] = depthTex.Load(p); int3 mp = p; if (flags & 1u) { // motion at output size: the texel under the render pixel's centre @@ -92,13 +100,14 @@ void inputs(uint3 id : SV_DispatchThreadID) { #if defined(PASS_INPUTS16) // Model K's compact input pass (record_inputs16): colour into RGBA16F and the raw motion texel into RG16F elements // (both UAVs typed FP16; the game's colour and motion are FP16-representable, so the stores are exact), depth and -// exposure as in inputs. The network side applies m = ms raw + mo when it reads the motion. +// exposure as in inputs. The colour keeps the texel's alpha, unmodified (model K's output alpha follows it; R11G11B10 +// has none and loads 1). The network side applies m = ms raw + mo when it reads the motion. [numthreads(8, 8, 1)] void inputs16(uint3 id : SV_DispatchThreadID) { if (id.x >= rw || id.y >= rh) return; int3 p = int3(rx + id.x, ry + id.y, 0); uint i = id.y * rw + id.x; - colorBuf[i] = float4(colorTex.Load(p).rgb, 0.0); + colorBuf[i] = colorTex.Load(p); depthBuf[i] = depthTex.Load(p); int3 mp = p; if (flags & 1u) { // motion at output size: the texel under the render pixel's centre @@ -144,10 +153,15 @@ float3 to_srgb(float3 c) { [numthreads(8, 8, 1)] void output(uint3 id : SV_DispatchThreadID) { if (id.x >= ow || id.y >= oh) return; - uint i = 3 * (id.y * ow + id.x); + // flags & 4u (model K): the result holds 4 x f32 per pixel, the fourth being the output alpha (the colour input's + // alpha through model K's output stage), stored as it is. Else 3 x f32 and alpha 1 (model M). + bool ka = (flags & 4u) != 0; + uint i = (ka ? 4 : 3) * (id.y * ow + id.x); float3 c = float3(resultBuf[i], resultBuf[i + 1], resultBuf[i + 2]); + float a = 1.0; + [branch] if (ka) a = resultBuf[i + 3]; if (srgb) c = to_srgb(c); - outTex[id.xy] = float4(c, 1.0); + outTex[id.xy] = float4(c, a); } #endif @@ -157,9 +171,10 @@ void output(uint3 id : SV_DispatchThreadID) { [numthreads(8, 8, 1)] void outtex(uint3 id : SV_DispatchThreadID) { if (id.x >= ow || id.y >= oh) return; - float3 c = proxyTex.Load(int3(id.xy, 0)).rgb; + float4 t = proxyTex.Load(int3(id.xy, 0)); + float3 c = t.rgb; if (srgb) c = to_srgb(c); - outTex[id.xy] = float4(c, 1.0); + outTex[id.xy] = float4(c, (flags & 4u) ? t.a : 1.0); // model K: the result texture's alpha, as it is } #endif @@ -172,11 +187,14 @@ void fallback(uint3 id : SV_DispatchThreadID) { float2 f = s - floor(s); int2 lo = int2(rx, ry), hi = int2(rx + rw - 1, ry + rh - 1); int2 p0 = clamp(int2(rx, ry) + a, lo, hi), p1 = clamp(int2(rx, ry) + a + 1, lo, hi); - float3 c00 = colorTex.Load(int3(p0.x, p0.y, 0)).rgb, c10 = colorTex.Load(int3(p1.x, p0.y, 0)).rgb; - float3 c01 = colorTex.Load(int3(p0.x, p1.y, 0)).rgb, c11 = colorTex.Load(int3(p1.x, p1.y, 0)).rgb; - float3 c = lerp(lerp(c00, c10, f.x), lerp(c01, c11, f.x), f.y); + float4 c00 = colorTex.Load(int3(p0.x, p0.y, 0)), c10 = colorTex.Load(int3(p1.x, p0.y, 0)); + float4 c01 = colorTex.Load(int3(p0.x, p1.y, 0)), c11 = colorTex.Load(int3(p1.x, p1.y, 0)); + float4 b = lerp(lerp(c00, c10, f.x), lerp(c01, c11, f.x), f.y); + float3 c = b.rgb; if (srgb) c = to_srgb(c); - outTex[id.xy] = float4(c, 1.0); + // model K (flags & 4u): the colour's alpha through the same bilinear (a frame without a network result follows the + // colour input's alpha too); else alpha 1 (model M) + outTex[id.xy] = float4(c, (flags & 4u) ? b.a : 1.0); } #endif )HLSL"; @@ -445,12 +463,12 @@ void Converter::record_inputs16(ID3D12GraphicsCommandList* list, const WgFrame& } void Converter::record_inputs(ID3D12GraphicsCommandList* list, const WgFrame& f, uint32_t create_flags, - const InputBuffers& b) { - record_inputs_pass(list, f, create_flags, b, false); + const InputBuffers& b, bool raw_motion) { + record_inputs_pass(list, f, create_flags, b, false, raw_motion); } void Converter::record_inputs_pass(ID3D12GraphicsCommandList* list, const WgFrame& f, uint32_t create_flags, - const InputBuffers& b, bool compact) { + const InputBuffers& b, bool compact, bool raw_motion) { const UINT set = next_set(); D3D12_CPU_DESCRIPTOR_HANDLE cpu = heap_->GetCPUDescriptorHandleForHeapStart(); cpu.ptr += (SIZE_T)set * kSetSize * increment_; @@ -497,13 +515,17 @@ void Converter::record_inputs_pass(ID3D12GraphicsCommandList* list, const WgFram c.ry = f.render_y; c.rw = f.render_width; c.rh = f.render_height; - c.msx = f.motion_scale[0]; - c.msy = f.motion_scale[1]; - c.mox = f.motion_offset[0]; - c.moy = f.motion_offset[1]; + // raw_motion (model K): scale 1, offset 0, so the pass's msx * m + mox writes the raw texel m (1 m exact; + 0 only + // turns -0 into +0, which K's own m + b turns into the same value); the caller passes motion_scale / offset in + // WgRtFrame (issue #26). Model M: the frame's motion_scale / offset, as before. + c.msx = raw_motion ? 1.0f : f.motion_scale[0]; + c.msy = raw_motion ? 1.0f : f.motion_scale[1]; + c.mox = raw_motion ? 0.0f : f.motion_offset[0]; + c.moy = raw_motion ? 0.0f : f.motion_offset[1]; c.mw = f.motion.width; c.mh = f.motion.height; - c.flags = ((create_flags & WG_CREATE_MOTION_OUTPUT_RES) ? kMotionOutputRes : 0) | (expoTex ? kExposureTexture : 0); + c.flags = ((create_flags & WG_CREATE_MOTION_OUTPUT_RES) ? kMotionOutputRes : 0) | (expoTex ? kExposureTexture : 0) | + (model_k_ ? kInputAlpha : 0); Barriers bar; const auto srvState = D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE; @@ -535,7 +557,7 @@ void Converter::record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, d.Format = DXGI_FORMAT_R32_FLOAT; d.ViewDimension = D3D12_UAV_DIMENSION_BUFFER; d.Buffer.FirstElement = range.offset / 4; - d.Buffer.NumElements = 3 * f.output_width * f.output_height; + d.Buffer.NumElements = (model_k_ ? 4 : 3) * f.output_width * f.output_height; // model K: RGBA f32 (kOutputAlpha) device_->CreateUnorderedAccessView(result, nullptr, &d, u1); } else { D3D12_SHADER_RESOURCE_VIEW_DESC d = {}; @@ -558,6 +580,7 @@ void Converter::record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, c.ow = f.output_width; c.oh = f.output_height; c.srgb = is_srgb(ofmt) ? 1 : 0; + c.flags = model_k_ ? kOutputAlpha : 0; Barriers bar; bar.need(out, f.output.state, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); if (!result) @@ -612,6 +635,7 @@ bool Converter::record_output_tex(ID3D12GraphicsCommandList* list, const WgFrame c.ow = f.output_width; c.oh = f.output_height; c.srgb = is_srgb(ofmt) ? 1 : 0; + c.flags = model_k_ ? kOutputAlpha : 0; Barriers bar; bar.need(out, f.output.state, D3D12_RESOURCE_STATE_UNORDERED_ACCESS); bar.need(proxy, D3D12_RESOURCE_STATE_UNORDERED_ACCESS, D3D12_RESOURCE_STATE_NON_PIXEL_SHADER_RESOURCE); diff --git a/clean/runtime/interop/convert.h b/clean/runtime/interop/convert.h index 5d1fff25..52a173d1 100644 --- a/clean/runtime/interop/convert.h +++ b/clean/runtime/interop/convert.h @@ -7,6 +7,10 @@ // - output: the network's result buffer (O, 3 x f32 RGB) into the game's output texture (alpha 1, sRGB encoded for // sRGB formats); // - fallback: a bilinear upscale of the game's color into the output texture, for frames without a network result. +// Model K (set_model_k; issue #26): its output alpha is the colour input's alpha of the frame, so the input passes +// keep the colour texel's alpha in the dense colour's fourth value (model M: +0), the output pass reads 4 x f32 per +// result pixel and stores the fourth as the alpha, the output pass on a result texture stores that texture's alpha, +// and the fallback stores the bilinear of the colour's alpha (model M: alpha 1 in all three). // HLSL compiled at run time with the Windows component d3dcompiler_47.dll (cs_5_0). The game's resource states are // transitioned as needed and restored to the states the front end reported. #pragma once @@ -38,8 +42,11 @@ class Converter { public: ~Converter(); bool init(ID3D12Device* device, std::string* err); - // Records the input conversion of frame f into list (barriers on the game's textures and back). - void record_inputs(ID3D12GraphicsCommandList* list, const WgFrame& f, uint32_t create_flags, const InputBuffers& b); + // Records the input conversion of frame f into list (barriers on the game's textures and back). raw_motion (model K, + // issue #26): the motion buffer gets the raw texels (the pass's scale 1, offset 0; the caller passes + // motion_scale / offset in WgRtFrame, and K folds the scale into its motion factor as the reference does). + void record_inputs(ID3D12GraphicsCommandList* list, const WgFrame& f, uint32_t create_flags, const InputBuffers& b, + bool raw_motion = false); // Model K's compact input pass: dense rows of RGBA16F colour (rw * 8 bytes), R32F depth and the raw RG16F motion // texel (rw * 4 bytes; the network applies motion_scale / motion_offset), into the same buffers. Exact when // inputs16_ok: colour RGBA16F or R11G11B10, motion RG16F or RGBA16F (FP16-representable values). @@ -60,8 +67,8 @@ class Converter { // render_height), UNORDERED_ACCESS) at the render pixel's texture coordinate. The game's motion texture must be in a // shader-resource state; no barriers are recorded. False (*err) if its shader fails to compile. bool record_motion_point(ID3D12GraphicsCommandList* list, const WgFrame& f, ID3D12Resource* dst, std::string* err); - // Records the output write: from the result buffer (R32_FLOAT elements, 3 per pixel) or, when result is null, the - // bilinear fallback from the game's color texture (result.resource null). + // Records the output write: from the result buffer (R32_FLOAT elements, 3 per pixel; model K: 4 per pixel, RGBA) or, + // when result is null, the bilinear fallback from the game's color texture (result.resource null). void record_output(ID3D12GraphicsCommandList* list, const WgFrame& f, const BufferRange& result); // The output pass from an RGBA32F texture holding the result (executor=dx12; the texture in UNORDERED_ACCESS before // and after): the operations of record_output's pass on the same FP32 values. False (*err) if its shader fails. @@ -91,8 +98,16 @@ class Converter { // before it), e.g. from a rescue list. void record_spin(ID3D12GraphicsCommandList* list, const BufferRange& flag, uint32_t value, uint32_t polls); void record_flag_immediate(ID3D12GraphicsCommandList* list, const BufferRange& flag, uint32_t value); + // Model K (issue #26): the alpha is carried (see the file comment): the input passes keep the colour's alpha, + // record_output reads an RGBA f32 result (FrameIo.out_rgba32f), record_output_tex and the fallback store the result + // texture's / the colour's alpha. Off (model M, the default): colour alpha +0 in, alpha 1 out. Set by the backend + // once the model is known. + void set_model_k(bool on) { model_k_ = on; } + // Bytes per pixel of the result buffer record_output reads (dense f32 rows): 12, model K 16. + uint32_t result_pixel_bytes() const { return model_k_ ? 16u : 12u; } private: + bool model_k_ = false; struct Constants { uint32_t rx, ry, rw, rh; float msx, msy, mox, moy; @@ -101,7 +116,7 @@ class Converter { }; UINT next_set(); void record_inputs_pass(ID3D12GraphicsCommandList* list, const WgFrame& f, uint32_t create_flags, - const InputBuffers& b, bool compact); + const InputBuffers& b, bool compact, bool raw_motion = false); void bind(ID3D12GraphicsCommandList* list, ID3D12PipelineState* pso, const Constants& c, UINT set); ID3D12Device* device_ = nullptr; diff --git a/clean/runtime/interop/executor.h b/clean/runtime/interop/executor.h index 22588a04..c9a0d881 100644 --- a/clean/runtime/interop/executor.h +++ b/clean/runtime/interop/executor.h @@ -94,6 +94,9 @@ struct Job { int color_pitch = 0, depth_pitch = 0, motion_pitch = 0, motion_channels = 4; float ms[2] = {1, 1}, mo[2] = {0, 0}; int out_half_pitch = 0; + // Model K with out_half_pitch == 0: the result is dense RGBA f32 (16 bytes per pixel, the fourth value the output + // alpha; FrameIo.out_rgba32f) for the output pass, which stores it (Converter::set_model_k). + bool out_rgba32f = false; int ow = 0, oh = 0; // output size (sparkle guard) // Motion at output resolution: the slot's motion_hi holds this frame's raw motion (mw x mh texels of two FP16, rows // of motion_hi_pitch bytes); mv_scale_hi converts it to pixels of that size, b_mv is added to it first. diff --git a/clean/runtime/interop/executor_hip.cpp b/clean/runtime/interop/executor_hip.cpp index b164a739..856b8253 100644 --- a/clean/runtime/interop/executor_hip.cpp +++ b/clean/runtime/interop/executor_hip.cpp @@ -157,6 +157,7 @@ class HipExecutor : public Executor { io.motion_channels = job.motion_channels; for (int k = 0; k < 2; ++k) io.ms[k] = job.ms[k], io.mo[k] = job.mo[k]; io.out_half_pitch = job.out_half_pitch; + io.out_rgba32f = job.out_rgba32f; if (job.motion_hi && job.motion_hi_use && p[5]) { // motion at output resolution: the raw texels (two FP16) io.motion_hi = p[5]; io.motion_hi_pitch = job.motion_hi_pitch; @@ -202,6 +203,7 @@ class HipExecutor : public Executor { // copied after the pass. A failed allocation or launch leaves the frame as the network wrote it. void sparkle_guard(const Job& job, const wgrt::FrameIo& io) { if (job.raw_inputs || !job.ow || !job.oh) return; // dense f32 depth / motion only (not the input_copy path) + if (job.out_rgba32f) return; // model K's RGBA f32 result: the guard kernel reads 3 x f32 or RGBA16F rows only wgrt::Device* d = wgrt::runtime_device(rt_); int ow = job.ow, oh = job.oh, rw = job.frame.rw, rh = job.frame.rh; const size_t pbytes = (size_t)ow * oh * 2, dbytes = (size_t)rw * rh * 4; diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index be8374b4..572718e4 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -1490,6 +1490,19 @@ struct Runtime { return ok; } + // Test path with an RGBA colour: model K keeps the alpha (its output alpha follows it); Model M reads RGB only + bool upload_inputs_rgba(int rw, int rh, const float* rgba, const float* dep, const float* mot) { +#ifndef WG_NO_MODEL_K + if (k) return k->upload_inputs(rw, rh, rgba, dep, mot, 4) || fail(k->err()); +#endif + if (rw <= 0 || rh <= 0 || rw > max_rw || rh > max_rh) return fail("render size larger than the maximum"); + const size_t npix = (size_t)rw * rh; + std::vector rgb(npix * 3); + for (size_t i = 0; i < npix; ++i) + for (int c = 0; c < 3; ++c) rgb[3 * i + c] = rgba[4 * i + c]; + return upload_inputs(rw, rh, rgb.data(), dep, mot); + } + bool upload_inputs(int rw, int rh, const float* rgb, const float* dep, const float* mot) { #ifndef WG_NO_MODEL_K if (k) return k->upload_inputs(rw, rh, rgb, dep, mot) || fail(k->err()); @@ -2672,6 +2685,10 @@ Runtime* runtime_create(const WgRtConfig& cfg, std::unique_ptr dev, std: } void runtime_destroy(Runtime* rt) { delete rt; } bool runtime_frame(Runtime* rt, const WgRtFrame& frame, const FrameIo* io) { return rt->frame(frame, io); } +bool runtime_upload_inputs_rgba(Runtime* rt, int rw, int rh, const float* color_rgba, const float* depth, + const float* motion) { + return rt->upload_inputs_rgba(rw, rh, color_rgba, depth, motion); +} bool runtime_direct_preflight(Runtime* rt, const WgRtFrame& frame, const FrameIo* io) { return rt->direct_preflight(frame, io); } @@ -2810,6 +2827,9 @@ WG_RT_API const char* wg_rt_error(void* rt) { return rt ? ((Runtime*)rt)->err.c_ WG_RT_API int wg_rt_upload_inputs(void* rt, int rw, int rh, const float* c, const float* d, const float* m) { return ((Runtime*)rt)->upload_inputs(rw, rh, c, d, m) ? 0 : 1; } +WG_RT_API int wg_rt_upload_inputs_rgba(void* rt, int rw, int rh, const float* c, const float* d, const float* m) { + return ((Runtime*)rt)->upload_inputs_rgba(rw, rh, c, d, m) ? 0 : 1; +} WG_RT_API int wg_rt_upload_motion_hi(void* rt, int mw, int mh, int wide, const void* texels, const float* mv_scale, const float* b_mv) { return ((Runtime*)rt)->upload_motion_hi(mw, mh, wide, texels, mv_scale, b_mv) ? 0 : 1; diff --git a/clean/runtime/runtime_core.h b/clean/runtime/runtime_core.h index 3784052e..73e0da32 100644 --- a/clean/runtime/runtime_core.h +++ b/clean/runtime/runtime_core.h @@ -26,6 +26,11 @@ struct FrameIo { // The caller's output pass reads `out` as RGBA16F (8 bytes per pixel) instead of 3 x f32: needed for the post kernels // with FP16 output (post_fused*_o16); the runtime refuses those otherwise. bool out_rgba16f = false; + // Model K (issue #26): `out` is written as dense RGBA f32 [oh][ow][4] (16 bytes per pixel) instead of 3 x f32, + // the fourth value being the output alpha (the colour input's alpha of this frame through the output stage's + // current-colour bilinear), for a caller whose output pass carries it (the interop's output pass for model K). + // Not used with out_half_pitch > 0 (the RGBA16F rows hold the alpha themselves). Model M ignores it. + bool out_rgba32f = false; // Interop copy paths (options input_copy / output_copy). raw_inputs: color, depth and motion above are the game's // textures copied as they are into rows of the given pitches (bytes): color RGBA16F, depth R32F, motion with // motion_channels FP16 channels, raw values (m = ms * raw + mo, here); the runtime converts them into its own dense @@ -75,6 +80,10 @@ Runtime* runtime_create(const WgRtConfig& cfg, std::unique_ptr dev, std: void runtime_destroy(Runtime* rt); // Enqueues one whole frame (does not wait). io may be null. bool runtime_frame(Runtime* rt, const WgRtFrame& frame, const FrameIo* io); +// Test path (as wg_rt_upload_inputs_rgba): the colour as dense RGBA f32 [rh][rw][4]; model K keeps the fourth value +// (the colour input's alpha), Model M ignores it. +bool runtime_upload_inputs_rgba(Runtime* rt, int rw, int rh, const float* color_rgba, const float* depth, + const float* motion); // Same mode and direct-frame checks used by runtime_frame, before any caller texture transitions or queue work. // Only runtime_error may change: no GPU calls and no frame/history/reset mutation. Buffer-mode frames are unchanged. bool runtime_direct_preflight(Runtime* rt, const WgRtFrame& frame, const FrameIo* io); diff --git a/clean/runtime/wg_runtime.h b/clean/runtime/wg_runtime.h index 3ab67b04..7c424d02 100644 --- a/clean/runtime/wg_runtime.h +++ b/clean/runtime/wg_runtime.h @@ -127,6 +127,11 @@ WG_RT_API const char* wg_rt_error(void* rt); /* Test path: copy this frame's inputs (dense [rh][rw] arrays: RGB float32, depth float32, motion float32 x 2). */ WG_RT_API int wg_rt_upload_inputs(void* rt, int rw, int rh, const float* color_rgb, const float* depth, const float* motion); +/* The same with the colour as dense RGBA float32 [rh][rw][4]. Model K: the fourth value is the colour input's alpha, + * which its output alpha follows (wg_rt_upload_inputs gives model K alpha 1, the value of a colour without an alpha + * channel). Model M ignores the fourth value. */ +WG_RT_API int wg_rt_upload_inputs_rgba(void* rt, int rw, int rh, const float* color_rgba, const float* depth, + const float* motion); /* Test path for motion vectors at output resolution (spec 03 draft 4 "Motion at output resolution", spec 06 section * 3; the runtime must have been created with the runtime option motion_outres=original): the motion of the region * M = mw x mh (the output size) as dense rows, a texel being two FP16 (wide = 0) or two f32 (wide = 1). Every later