diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 725dd18a..492c509f 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -300,6 +300,10 @@ "k_out_color", "k_io.co" ], + [ + "k_out_color_direct", + "k_io.co" + ], [ "k_out_pixel", "k_io.co" @@ -308,6 +312,14 @@ "k_out_pixel_c", "k_io.co" ], + [ + "k_out_pixel_c_direct", + "k_io.co" + ], + [ + "k_out_pixel_direct", + "k_io.co" + ], [ "k_stage2_d0_hidden", "k_block.co" @@ -380,9 +392,37 @@ "k_warm", "k_block_wide.co" ], + [ + "kx/k_in_base", + "k_io_exact.co" + ], + [ + "kx/k_in_base_direct", + "k_io_exact.co" + ], + [ + "kx/k_in_color", + "k_io_exact.co" + ], + [ + "kx/k_in_color_direct", + "k_io_exact.co" + ], + [ + "kx/k_in_reset", + "k_io_exact.co" + ], [ "kx/k_in_tiled", "k_io_exact.co" + ], + [ + "kx/k_in_tiled_direct", + "k_io_exact.co" + ], + [ + "kx/k_in_tiled_uniform", + "k_io_exact.co" ] ] } diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 595fb21a..96d123f2 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -265,6 +265,7 @@ bool bind_direct(NativeD3D12Ext* ext, Case& c, const std::string& base) { if (t.slot == "color") f.color = t.tex, f.color_format = (DXGI_FORMAT)t.format; else if (t.slot == "depth") f.depth = t.tex, f.depth_format = (DXGI_FORMAT)t.format; else if (t.slot == "motion") f.motion = t.tex, f.motion_format = (DXGI_FORMAT)t.format; + else if (t.slot == "exposure") f.exposure = t.tex, f.exposure_format = (DXGI_FORMAT)t.format; else if (t.slot == "output") f.output = t.tex, f.output_format = (DXGI_FORMAT)t.format; } f.params = c.dparams; @@ -275,6 +276,55 @@ bool bind_direct(NativeD3D12Ext* ext, Case& c, const std::string& base) { return true; } +// The three direct lines of a case or a chain (see the header comment); true when `w` is one of them (*bad set on error). +bool parse_direct_line(const std::string& w, std::istringstream& s, Case& c, bool* bad) { + *bad = false; + if (w == "direct") { + DirectTex t; + s >> t.slot >> t.format >> t.width >> t.height >> t.file; + if (t.slot != "color" && t.slot != "depth" && t.slot != "motion" && t.slot != "exposure" && t.slot != "output") { + std::printf("unknown direct slot %s\n", t.slot.c_str()); + *bad = true; + } + c.direct.push_back(t); + return true; + } + if (w == "direct_params") { + s >> c.dparams.rx >> c.dparams.ry >> c.dparams.rw >> c.dparams.rh; + std::string tok; + int pos = 0; + while (s >> tok) { // [pi [rne16]] then key=value words: ms=, mo=, expo= + const size_t eq = tok.find('='); + if (eq == std::string::npos) { + if (pos == 0) c.dparams.color_pi = std::stof(tok); + else if (pos == 1) c.dparams.output_rne16 = (unsigned)std::stoul(tok); + ++pos; + } else { + const std::string key = tok.substr(0, eq), val = tok.substr(eq + 1); + const size_t comma = val.find(','); + if (key == "ms" && comma != std::string::npos) { + c.dparams.msx = std::stof(val.substr(0, comma)); + c.dparams.msy = std::stof(val.substr(comma + 1)); + } else if (key == "mo" && comma != std::string::npos) { + c.dparams.mox = std::stof(val.substr(0, comma)); + c.dparams.moy = std::stof(val.substr(comma + 1)); + } else if (key == "expo") { + c.dparams.exposure_mode = (unsigned)std::stoul(val); + } else { + std::printf("unknown direct_params word %s\n", tok.c_str()); + *bad = true; + } + } + } + return true; + } + if (w == "direct_out") { + s >> c.direct_out; + return true; + } + return false; +} + std::string dir_of(const std::string& path) { const size_t s = path.find_last_of("/\\"); return s == std::string::npos ? std::string(".") : path.substr(0, s); @@ -375,6 +425,7 @@ struct Chain { std::vector steps; std::vector> saves; // (id, file) int time_runs = 0; + Case dcase; // direct textures bound around the steps (rs=direct kernels): direct / direct_params / direct_out lines }; int run_chain(Device* d, Chain& c, const std::string& base) { @@ -484,7 +535,33 @@ int run_chain(Device* d, Chain& c, const std::string& base) { if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return false; } return true; }; + NativeD3D12Ext* dext = nullptr; + if (rc == 0 && !c.dcase.direct.empty()) { + c.dcase.name = c.name; + dext = d3d12_ext(d); + if (!dext || !bind_direct(dext, c.dcase, base)) { + std::string e; + if (dext) dext->sync(&e); // the upload pass may still be queued: the textures are released only after it ran + release_direct(c.dcase); + rc = 1; + } + } if (rc == 0 && !run_steps()) rc = 1; + if (dext) { // the direct frame ends after the first run; an output texture is read back + std::string derr; + if (rc == 0 && (!dext->end_direct_frame(&derr) || !dext->sync(&derr))) { + std::printf("%s: direct: %s\n", c.name.c_str(), derr.c_str()); + rc = 1; + } + if (rc == 0 && !c.dcase.direct_out.empty()) { + std::vector texels; + if (!read_direct_output(dext, c.dcase, texels, derr) || !write_file(base + "/" + c.dcase.direct_out, texels)) { + std::printf("%s: direct output: %s\n", c.name.c_str(), derr.c_str()); + rc = 1; + } + } + release_direct(c.dcase); + } for (size_t i = 0; rc == 0 && i < c.saves.size(); ++i) { ChainBuf* b = find(c.saves[i].first); std::vector data(b ? b->bytes : 0); @@ -551,6 +628,10 @@ int main(int argc, char** argv) { ch.saves.emplace_back(id, file); } else if (w == "time") { s >> ch.time_runs; + } else if (w == "direct" || w == "direct_params" || w == "direct_out") { + bool bad = false; + parse_direct_line(w, s, ch.dcase, &bad); + if (bad) return 2; } else { std::printf("unknown chain line: %s\n", line.c_str()); return 2; @@ -565,22 +646,10 @@ int main(int argc, char** argv) { c.modules.emplace_back(m, tag); } else if (w == "kernel") { s >> c.kernel >> c.grid >> c.block; - } else if (w == "direct") { - DirectTex t; - s >> t.slot >> t.format >> t.width >> t.height >> t.file; - if (t.slot != "color" && t.slot != "depth" && t.slot != "motion" && t.slot != "output") { - std::printf("unknown direct slot %s\n", t.slot.c_str()); - return 2; - } - c.direct.push_back(t); - } else if (w == "direct_params") { - s >> c.dparams.rx >> c.dparams.ry >> c.dparams.rw >> c.dparams.rh; - float pi = 1.0f; - unsigned rne16 = 0; - if (s >> pi) c.dparams.color_pi = pi; // optional: color_pi, then output_rne16 - if (s >> rne16) c.dparams.output_rne16 = rne16; - } else if (w == "direct_out") { - s >> c.direct_out; + } else if (w == "direct" || w == "direct_params" || w == "direct_out") { + bool bad = false; + parse_direct_line(w, s, c, &bad); + if (bad) return 2; } else if (w == "arg") { Arg a; s >> a.kind; diff --git a/clean/fsr/tests/test_frontend.cpp b/clean/fsr/tests/test_frontend.cpp index 3bff71db..2db49e57 100644 --- a/clean/fsr/tests/test_frontend.cpp +++ b/clean/fsr/tests/test_frontend.cpp @@ -23,6 +23,7 @@ #include "../src/frontend.h" #include "../src/param_map.h" #include "wg_settings_io.h" // pure part: describe() +#include "../../runtime/interop/executor_select.h" // the network runtime's refused-frame policy (header only) using namespace wgfsr; @@ -143,6 +144,7 @@ static int g_failDispatchAt = -1; static int g_notReadyUntil = 0; // FakeDispatch answers WG_ERROR_NOT_READY for frames below this count static int g_parameterErrorAt = -1; // FakeDispatch answers WG_ERROR_PARAMETER for this frame (inputs it cannot take) static int g_unsupportedFrom = -1; // FakeDispatch answers WG_ERROR_UNSUPPORTED from this frame on (frames it does not take) +static std::set g_unsupportedAt; // FakeDispatch answers WG_ERROR_UNSUPPORTED at these frames only (refused frames) static int g_inputErrorFrom = -1; // FakeDispatch answers WG_ERROR_INPUT from this frame on (inputs it never takes) static std::set g_inputErrorAt; // ...and at these frames (a menu or loading frame with an odd texture) static int g_deviceErrorFrom = -1; // FakeDispatch answers WG_ERROR_DEVICE from this frame on (bug hunt F1) @@ -171,6 +173,7 @@ static WgResult FakeDispatch(WgUpscaler*, const WgFrame* frame) { g_frames.push_back(*frame); if (int(g_frames.size()) - 1 == g_parameterErrorAt) return WG_ERROR_PARAMETER; if (g_unsupportedFrom >= 0 && int(g_frames.size()) - 1 >= g_unsupportedFrom) return WG_ERROR_UNSUPPORTED; + if (g_unsupportedAt.count(int(g_frames.size()) - 1)) return WG_ERROR_UNSUPPORTED; if (g_inputErrorFrom >= 0 && int(g_frames.size()) - 1 >= g_inputErrorFrom) return WG_ERROR_INPUT; if (g_inputErrorAt.count(int(g_frames.size()) - 1)) return WG_ERROR_INPUT; if (g_deviceErrorFrom >= 0 && int(g_frames.size()) - 1 >= g_deviceErrorFrom) return WG_ERROR_DEVICE; @@ -343,6 +346,7 @@ static void Reset() { g_notReadyUntil = 0; g_parameterErrorAt = -1; g_unsupportedFrom = -1; + g_unsupportedAt.clear(); g_inputErrorFrom = -1; g_inputErrorAt.clear(); g_deviceErrorFrom = -1; @@ -2388,6 +2392,60 @@ static void TestUnsupportedFrames() { CHECK(fe->DestroyContext(&ctx, kMemCb) == FFX_API_RETURN_OK); } +// REVIEW-215c: a refused frame of model K takes the same front-end path as model M's. The backend's answer comes from the +// real policy (wgi::refused_frame_action, executor_select.h: a refused K frame, and a refused M frame with hip_switch=off, +// are answered WG_ERROR_UNSUPPORTED, never a switch to HIP and never a bilinear fallback), the fake upscaler answers +// that for the refused frames, and the front end must give every frame an upscaler: AMD's FSR when it exists (it takes +// the context over at the first refusal, as for model M), else the stand-in for each refused frame (any number in a row, +// the context stays WrongGPU's and takes the next frame it can). No HIP runtime is ever started. +static void TestRefusedFrames() { + for (const bool model_k : {true, false}) { // model K, and model M as the control + const bool unsupported = + wgi::refused_frame_action(model_k, /*by_default=*/true, /*hip_switch=*/false) == wgi::RefusedAction::kNoSwitch; + CHECK(unsupported); // the backend's answer for a refused frame is WG_ERROR_UNSUPPORTED + const std::set pattern = unsupported ? std::set{1, 2, 3, 5, 6} : std::set{}; + // with AMD's FSR: the first refused frame hands the context to it; no later frame is lost + Reset(); + g_resources[kOutput] = {dxgi::R16G16B16A16_FLOAT, 2560, 1440}; + Config config; + config.hosted = true; + auto fe = MakeFrontEnd(config); + UpscaleCreate own(FFX_UPSCALE_ENABLE_HIGH_DYNAMIC_RANGE, /*withOverride=*/true); + own.version.versionId = kWrongGpuProviderVersionId; + ffxContext ctx = nullptr; + CHECK(fe->CreateContext(&ctx, &own.upscale.header, kMemCb) == FFX_API_RETURN_OK && fe->OwnContextCount() == 1); + g_unsupportedAt = pattern; + ffxDispatchDescUpscale d = MakeDispatch(); + for (int i = 0; i < 8; ++i) CHECK(fe->Dispatch(&ctx, &d.header) == FFX_API_RETURN_OK); // every frame has an upscaler + CHECK(g_frames.size() == 2 && g_standInFrames.empty()); // frame 0 the network's, frame 1 refused: AMD from there on + CHECK(!g_amd.empty() && g_amd.back().fn == "dispatch" && g_statuses.back().state == UpscaleState::Amd); + CHECK(CountLog("HIP") == 0); // no HIP runtime on the way + CHECK(fe->DestroyContext(&ctx, kMemCb) == FFX_API_RETURN_OK); + + // without AMD's FSR: each refused frame goes to the stand-in (five in a row, then a taken frame, then two more + // refused ones), every frame gets an upscaler, the network takes the frames between + Reset(); + g_resources[kOutput] = {dxgi::R16G16B16A16_FLOAT, 2560, 1440}; + g_originalAvailable = false; + fe = MakeFrontEnd(config); + ctx = nullptr; + CHECK(fe->CreateContext(&ctx, &own.upscale.header, kMemCb) == FFX_API_RETURN_OK && fe->OwnContextCount() == 1); + g_unsupportedAt = unsupported ? std::set{1, 2, 3, 4, 5, 8, 9} : std::set{}; + const std::vector want = {UpscaleState::Network, UpscaleState::StandIn, UpscaleState::StandIn, + UpscaleState::StandIn, UpscaleState::StandIn, UpscaleState::StandIn, + UpscaleState::Network, UpscaleState::Network, UpscaleState::StandIn, + UpscaleState::StandIn, UpscaleState::Network}; + for (size_t i = 0; i < want.size(); ++i) { + CHECK(fe->Dispatch(&ctx, &d.header) == FFX_API_RETURN_OK); + CHECK(g_statuses.back().state == want[i]); + } + CHECK(g_frames.size() == want.size() && g_standInFrames.size() == 7 && g_amd.empty()); + CHECK(CountLog("HIP") == 0); + CHECK(CountLog("AMD FSR is not available for this context; the stand-in upscales the frames the upscaler does not take") == 1); + CHECK(fe->DestroyContext(&ctx, kMemCb) == FFX_API_RETURN_OK); + } +} + static void TestSplitBlocked() { Reset(); g_resources[kOutput] = {dxgi::R16G16B16A16_FLOAT, 2560, 1440}; @@ -3434,6 +3492,7 @@ int main() { #endif TestRegistryEntries(); TestUnsupportedFrames(); + TestRefusedFrames(); TestSplitBlocked(); TestRuntimeProviderName(); TestWithoutModelKItems(); diff --git a/clean/k/README.md b/clean/k/README.md index 8f104911..6aaf3146 100644 --- a/clean/k/README.md +++ b/clean/k/README.md @@ -195,6 +195,24 @@ Model K ships in the same `wronggpu_runtime.dll` as model M; nothing changes for Exposure (spec 42 draft 3 section 4): E_cur = the game's exposure texture or value, 1 if it passes none; the auto-exposure value (spec 07, run when the game asks for it at creation, HDR only) is E_aux and acts only through λ. `k_expo_rule=v051` (A/B only) restores the 0.5.1 rule (the measured value as E_cur and E_aux). + Direct3D 12 devices (`backend=2`, the hybrid device, and `backend=3`, the native device): the exact default path runs + on the D3D12 ports of `clean/k/dx12` (byte-identical to the HIP kernels, `clean/k/dx12/README.md`); a context whose + options leave that path (`network=fused16` / `fused8`, `k_fast=1`, `k_fast_in=1`, `k_in_exact=0`, `k_diag`) is + refused on the native device (those options run on the HIP backend only). The legacy exact input kernels + (`kx/k_in_color`, `kx/k_in_reset`, `kx/k_in_base`: a render size above 0.68 x the output up to 1.0, a jitter outside + [-0.5, 0.5], `input_tiled=0`) are ported too, so a native context covers every ratio and every jitter. An input-copy + frame (`raw_inputs`, the interop's HIP input copies) is refused on the D3D12 devices. On the native device `create` + first checks that + `dx12/manifest.json` has a port of every kernel the context can launch and refuses it, naming the missing kernels, + before the model is loaded. `k_in_uniforms=0` (diagnostic): the tiled input kernel that computes its frame values on the + GPU instead of taking the host's. `k_fill=<0..255>` (test hook, changes no value a frame writes): every buffer a frame writes and both history copies start with this byte after allocation, so that a test with a different byte per backend proves first-frame and reset-only writes. Latency options of the D3D12 devices that change no value: `k_color_at=` + (`k_out_color` runs unordered right after that stage; default: before the input stage, HIP's order) and `k_warm=` / + `off` (backend 3: `k_warm`, a cache warm-up of the weight blobs of the stages after it, launched unordered after that + stage, default `e0`; `k_warm_to`, `k_warm_line`, `k_warm_groups`). The D3D12 executor of the interop (the game's own + device) runs model K with the forms that read the game's textures and write the output texture (`_direct`, also + declared for the legacy input kernels). When it cannot take a context (a kernel without a port in the build, an option + off the exact path) it refuses K at create with one log line and the game keeps its own upscaler: model K has no HIP + route in a game (`executor=hip` selects the HIP executor as a diagnostic). In game (Cyberpunk 2077 benchmark, 1440p Quality, RX 9070 XT; the project's measurements): model K GPU time about 1.8 ms per frame (`fused`; 1.72 ms with `fused16`), the game at ~120 fps against ~117 fps with model M diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index b0a942aa..ec514023 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -44,13 +44,27 @@ and inputs, the stage runner's launch plans run on HIP and replayed on D3D12 (`k third case carry special values (infinity, +-65504, zero, -0, subnormals) and of every third case NaN as well; the half-grid control launches a stage with half its workgroups. `wide` (the wide stages of `k_block_wide.co`: `k_stagew2_e3`, `k_stagew4_e4`, `k_stagew2_e4`, `k_stagew4_m`, `k_stagew4_d4`, `k_stagew2_d4`, `k_stagew2_d3`: one window -per 128 NG lanes) is checked the same way. `expo` (`k_expo_scalars`) against `k_io.co`; `warm` +per 128 NG lanes) is checked the same way. Not ported, because no context can reach them: the NG = 4 forms of `e3` and `d3` (`k_stagew4_e3`, `k_stagew4_d3`), which the HIP build has. `wide_ng` in `clean/k/runtime/runtime_k.cpp` gives `e3` and `d3` NG = 2 on the exact network whatever the window count, the sizes or the options (only `e4` and `d4` look at the window count), and the FP16 networks, which take other kernels, are refused on the native device; `clean/k/tests/test_k_wide_ng.py` pins the rule, the two places that build the kernel names and the declared kernel set. A replay of `k_stagew4_d3` on D3D12 was reported to differ from the HIP capture; if a change ever makes the form reachable, that test fails until the form is ported and checked. `expo` (`k_expo_scalars`) against `k_io.co`; `warm` (`k_warm`, a cache warm-up with no HIP kernel: checked against its definition, through the struct argument path). -NaN: IEEE 754 does not define the sign or payload of a NaN, and where a NaN is generated or propagated the two devices -can differ in it (the decoder stages, with NaN in the input). For the cases with NaN in the input the check compares the -raw bytes and, where they differ, reports the elements that are NaN on both sides separately; every other byte must be -identical. Without NaN in the input the raw bytes are identical. +`legacy` (the legacy exact input kernels of `k_io_exact.co`, `kx/k_in_color`, `kx/k_in_reset`, `kx/k_in_base`, in +`kernels/k_in_legacy.hlsl`; the runtime takes them where the tiled kernel does not apply: a render size above 0.68 x +the output up to 1.0 (native AA, DLAA), a jitter outside [-0.5, 0.5], `input_tiled=0`): the same flag sets as `input`, +every fourth case at render size = output size or with a jitter outside the guard; the inputs, the intermediates `cpre` +and `cpre2` and the output `x0` are compared raw; the half-grid control launches `k_in_base` with half its workgroups). +`direct` (the forms for the native executor, which read the game's colour, depth, motion and exposure textures and write +the output texture: `kx/k_in_tiled_direct`, `kx/k_in_color_direct`, `kx/k_in_base_direct`, `k_out_color_direct`, +`k_out_pixel_c_direct`, `k_out_pixel_direct`, `k_expo_scalars_direct`, `exposure_auto_diet_direct`): the check binds +real textures (RGBA16F colour, R32F depth, RG16F motion, exposure, an RGBA16F output) through `kernel_replay`'s direct +mode and compares the output texture and the buffers with the HIP kernels' run on the raw buffer route +(`KIO_C16` / `KIO_M16`, with the motion scale and offset folded as the interop passes them). + +NaN: IEEE 754 does not define the sign of a NaN that a stage generates or propagates, and where the input holds NaN the +two devices can differ in it (the decoder stages). For the cases with NaN in the input the check compares the raw bytes +and, where they differ, accepts only the elements that are NaN on both sides and differ in the sign bit alone (the +exponent, the quiet bit and the payload equal), counted separately; every other differing byte, a NaN with another +payload included, fails the case (`clean/k/tests/test_k_port_check_nan.py`). Without NaN in the input the raw bytes are +identical. Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero and HIP keeps them. `k_expo_scalars` only meets one with a pre-exposure above about 8.5e37, so the check's inputs keep clear of them. diff --git a/clean/k/dx12/hlsl/k_in_common.hlsli b/clean/k/dx12/hlsl/k_in_common.hlsli new file mode 100644 index 00000000..4e88f455 --- /dev/null +++ b/clean/k/dx12/hlsl/k_in_common.hlsli @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT +// The exact input stage's scalar arithmetic and per-fine-position reprojection, shared by the tiled kernels (k_in.hlsl) +// and the legacy kernels (k_in_legacy.hlsl). WrongGPU's own code. Include after the kernel's argument block, +// k_fp.hlsli and k_frame.hlsli, and after kf_opq3 is defined. +#pragma once + +// ---- the exact input stage's arithmetic (k_io.hip with KI_FAST = 0) -------------------------------------------------- +float ki_log2(float x) { return kf_log2(x); } +float ki_exp2(float x) { return kf_exp2(x); } +float ki_rcp(float x) { return kf_div(1.0f, x); } +float ki_div(float a, float b) { return kf_div(a, b); } +// spec 42 decode factor D(t) +float ki_decode(float t) +{ + if (t < kf_bits(0x3F7FBE88u)) return ki_rcp(1.0f - t); + const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); + return ki_div(ki_exp2(e) - 1.0f, 9.0f * t); +} +float ki_phi(float Y, bool hdr) +{ + if (hdr) { + const float Yp = kf_max(Y, 0.0f); + return ki_div(ki_log2(kf_fma(Yp, 12.0f, 1.0f)) * kf_bits(0x3F317218u), Yp + kf_bits(0x358637BDu)); + } + return kf_fma(Y, 3.0f, 1.0f); +} +float ki_S(float z) +{ + const float zp = kf_min(kf_max(z, 0.0f), 1.0f); // HIP's NaN test is implied: maxNum(NaN, 0) = 0 + if (!(zp > kf_bits(0x3B4D2E1Cu))) return kf_bits(0x414EB852u) * zp; + const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once + return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu)); +} +// z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart +// (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it) +float ki_T(float z) +{ + return z * kf_max(ki_rcp(z + 1.0f), kf_bits(0x38D1B717u)); +} +float ki_L(float Y, bool hdr) { return hdr ? Y : ki_div(Y, kf_max(1.0f - kf_opq(Y), kf_bits(0x3E4CCCCDu))); } + +struct KiFine { // spec 42 section 5 per fine position + bool valid; + float3 h; // reprojected history (Y, Co, Cg) + float nx, ny; // hidden-history position (normalized) +}; + +KiFine ki_fine(KInParams P, KiHost k, int fx, int fy) +{ + const KFrameIn F = P.f; + const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); + const float px = (float)fx + 0.5f, py = (float)fy + 0.5f; + float mvx, mvy; + if ((F.flags & KIO_MVLO) != 0) { + const int ix = (int)kf_R(F.jx + kf_fma(px, k.a_x, -0.5f)), iy = (int)kf_R(F.jy + kf_fma(py, k.a_y, -0.5f)); + int dx, dy; + kio_nearest(F, ix, iy, dx, dy); + kio_motion(F, kio_mtap(ix, dx, F.mbx, F.rw), kio_mtap(iy, dy, F.mby, F.rh), mvx, mvy); + } else { + const int mx = (int)kf_R(kf_fma(px, k.mp_x, -0.5f)), my = (int)kf_R(kf_fma(py, k.mp_y, -0.5f)); + kio_motion(F, kio_tap(mx, 0, F.mbx, F.W), kio_tap(my, 0, F.mby, F.H), mvx, mvy); + } + // rounded before px + m (HIP); no kf_opq needed: the ISA keeps the products (precaution link dropped, ISA-checked) + // the texture's motion scale folded into the factor and its offset in raw units, as the buffer form gets them in + // WgRtFrame (hlsl/k_frame.hlsli kio_motion; issue #26): direct forms b1's ms / mo, KIO_M16 rmsx .. +#if WG_DIRECT_TEXTURES + const bool dfold = true; + const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy; +#else + const bool dfold = (F.flags & KIO_M16) != 0; + const float dmsx = F.rmsx, dmsy = F.rmsy, dmox = F.rmox, dmoy = F.rmoy; +#endif + float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy; + [branch] if (dfold) { + const bool dmvlo = (F.flags & KIO_MVLO) != 0; + spx = ki_dsp(F.sx, dmsx, F.W, dmvlo ? F.rw : F.W); + spy = ki_dsp(F.sy, dmsy, F.H, dmvlo ? F.rh : F.H); + ofx = ki_dmo(F.mox, dmox, dmsx); + ofy = ki_dmo(F.moy, dmoy, dmsy); + } + const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy); + const float wx = px + mx, wy = py + my; + KiFine r; + const float ux = wx * k.xi_x, uy = wy * k.xi_y; + r.valid = ux >= 0.0f && ux <= 1.0f && uy >= 0.0f && uy <= 1.0f; + const float4 s = kio_bilinear4(P.Hc, W16, H16, wx * k.h_x, wy * k.h_y); + if ((F.flags & KIO_HDR) != 0) { + const float f = ki_decode(s.x); + r.h = F.rho * kf_opq3(s.xyz * f); + } else { + r.h = s.xyz; + } + r.nx = ((float)(4 * (fx >> 2) + 2) + mx) * k.g_x; // fx, fy >= 0: fx / 4 = fx >> 2 + r.ny = ((float)(4 * (fy >> 2) + 2) + my) * k.g_y; + return r; +} + +// kappa max(RGB, 0), each product rounded (kf_opq: no merging with the 0.25 / 0.5 factors that follow) +float3 ki_rgb(KFrameIn F, float4 c) { return kf_opq3(F.kappa * max(c.xyz, 0.0f)); } +void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3) +{ + wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, + uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); +} diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py index 82a1a2cb..fe3d1f54 100644 --- a/clean/k/dx12/k_plan_bridge.py +++ b/clean/k/dx12/k_plan_bridge.py @@ -124,7 +124,7 @@ def chain_step(kernel, grid, block, args, tokens, bufs, cdir, tag): return f"step {kernel} {grid} {block} " + " ".join(out) -def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, d3d12_name=None): +def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", grid_div=1, d3d12_name=None, extra=()): """The kernel_replay chain of a plan: buffers (inputs from files, the rest filled with the D3D12 sentinel), one step per launch, a save per output buffer (..d3d12). grid_div: launch with grid // grid_div (the negative control uses 2). d3d12_name: the manifest's name of a plan kernel (default: the part after the code object's @@ -136,6 +136,7 @@ def chain_lines(name, plan, pdev, tokens_of, cdir, modules, outputs, prefix="", lines.append(f"buf b_{n} {size + SLACK} {name}.{n}.in {D3D12_SENTINEL}") else: lines.append(f"buf b_{n} {size + SLACK} - {D3D12_SENTINEL}") + lines += list(extra) # e.g. the direct texture lines of kernel_replay for li, (kernel, grid, block, args) in enumerate(plan["launches"]): dname = d3d12_name(kernel) if d3d12_name else kernel.split(":")[-1] lines.append(chain_step(dname, max(1, grid // grid_div), block, args, tokens_of(dname), bufs, cdir, diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index f24b35ea..4f0aae70 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -55,7 +55,12 @@ FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), "narrow": (["k_block.co"], NARROW_KERNELS), "input": (["k_io_exact.co"], ["kx/k_in_tiled"]), + "legacy": (["k_io_exact.co"], ["kx/k_in_color", "kx/k_in_reset", "kx/k_in_base"]), "output": (["k_io.co"], ["k_out_color", "k_out_pixel_c", "k_out_pixel"]), + "direct": (["k_io.co", "k_io_exact.co"], ["kx/k_in_tiled_direct", "k_out_color_direct", + "k_out_pixel_c_direct", "k_out_pixel_direct", + "k_expo_scalars_direct"]), + "directleg": (["k_io_exact.co"], ["kx/k_in_color_direct", "kx/k_in_reset", "kx/k_in_base_direct"]), "wide": (["k_block_wide.co"], ["k_stagew2_e3", "k_stagew4_e4", "k_stagew2_e4", "k_stagew4_m", "k_stagew4_d4", "k_stagew2_d4", "k_stagew2_d3"])} CONTROL_CO = "frame.co" @@ -231,12 +236,13 @@ def run_expo(build, rocm, seeds, cdir, hip): return items, lines, {"case": "expo_ctl", "hip": outs} -def chain_lines(name, prefix, grid, sizes, kernel, co): +def chain_lines(name, prefix, grid, sizes, kernel, co, extra=()): p = prefix + name return [f"chain {p}", f"module {co}", f"buf st {sizes['st']} {name}.st {D3D12_SENTINEL}", f"buf src {sizes['src']} {name}.src {D3D12_SENTINEL}", f"buf meas {sizes['measured']} {name}.meas {D3D12_SENTINEL}", f"buf xs {sizes['xs']} - {D3D12_SENTINEL}", - f"step {kernel} {grid} 32 @st @src @meas @xs B:{name}.args", f"save st {p}.out_st", f"save xs {p}.out_xs", ""] + *extra, f"step {kernel} {grid} 32 @st @src @meas @xs B:{name}.args", f"save st {p}.out_st", + f"save xs {p}.out_xs", ""] # ---- family warm: k_warm ------------------------------------------------------------------------------------------ @@ -403,28 +409,34 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non def nan_equivalent_differ(got, want, defined, width=2): - """Differing bytes over the first `defined` bytes after ignoring the elements (2 bytes: FP16) that are NaN in both: - IEEE 754 does not define a NaN's sign or payload, and the two devices may generate or propagate a different one.""" + """(differing elements outside the accepted exception, elements that differ only in the NaN sign) over the first + `defined` bytes. The only accepted exception is the sign of a NaN (IEEE 754 does not define it for a NaN a stage + generates or propagates): an element is a sign-only difference when both are NaN and the two differ in the sign bit + alone, so the exponent, the quiet bit and the payload are equal. Any other difference, a NaN with another payload + included, is outside.""" dt = np.uint16 if width == 2 else np.uint32 a = np.frombuffer(got[:defined], dt) b = np.frombuffer(want[:defined], dt) exp, mant = (0x7C00, 0x03FF) if width == 2 else (0x7F800000, 0x007FFFFF) + sign = dt(0x8000 if width == 2 else 0x80000000) nan = lambda v: ((v & exp) == exp) & ((v & mant) != 0) - return int(np.sum((a != b) & ~(nan(a) & nan(b)))), int(np.sum((a != b) & nan(a) & nan(b))) + sign_only = (a != b) & nan(a) & nan(b) & ((a ^ b) == sign) + return int(np.sum((a != b) & ~sign_only)), int(np.sum(sign_only)) def check_narrow(state, read): items, _, neg = state out = [] for it in items: - per = {n: compare(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n], D3D12_SENTINEL, HIP_SENTINEL) + fn = lambda n: it["case"] + ("." + n + ".d3d12" if "d3d12_files" not in it else it["d3d12_files"][n]) + per = {n: compare(read(fn(n)), it["hip"][n], it["defined"][n], D3D12_SENTINEL, HIP_SENTINEL) for n in it["outputs"]} raw = all(v["identical"] for v in per.values()) nan_only = 0 if not raw and it["specials"] == 2: # NaN inputs: a NaN's sign / payload is not defined (see nan_equivalent_differ) for n, v in per.items(): if not v["identical"] and v["slack_bad"] == 0: - d, same = nan_equivalent_differ(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n], + d, same = nan_equivalent_differ(read(fn(n)), it["hip"][n], it["defined"][n], it.get("widths", {}).get(n, 2)) v["differ_outside_nan"], v["nan_elements_differing_in_sign"] = d, same nan_only += same @@ -606,6 +618,280 @@ def run_output(build, rocm, seeds, cdir, hip): return items, lines, neg +# ---- family direct: the forms that read the game's textures (rs=direct; native executor) --------------------------- +# The reference is HIP's raw-buffer route (KIO_C16 colour rows, KIO_M16 raw motion with ms / mo folded, R32F depth): the +# bytes the HIP executor feeds model K for 16-bit float game textures. D3D12 reads the same texels as textures. + +DXGI_RGBA16F, DXGI_RGBA32F, DXGI_R32F, DXGI_RG16F = 10, 2, 41, 34 +KIO_C16, KIO_M16 = 16, 32 + + +def direct_inputs(x, rng): + """The frame's textures as arrays: colour RGBA16F, depth R32F, motion RG16F (render size, or the output's with + full-size motion); ms, mo of the motion texture.""" + c = x["color"].astype(np.float16) + m = x["motion"].astype(np.float16) + ms = np.array([0.5, 2.0], np.float32) if rng.random() < 0.5 else np.array([1.0, 1.0], np.float32) + mo = np.array([0.25, -0.5], np.float32) if ms[0] != 1.0 else np.array([0.0, 0.0], np.float32) + return c, np.ascontiguousarray(x["depth"], np.float32), m, ms, mo + + +def direct_lines(name, c, m, ms, mo, rne16, out_fmt=None, out_wh=(0, 0)): + """kernel_replay direct lines of a frame: textures from files, the frame params, the output texture.""" + rh, rw = c.shape[:2] + mh, mw = m.shape[:2] + lines = [f"direct color {DXGI_RGBA16F} {rw} {rh} {name}.color.tex", f"direct depth {DXGI_R32F} {rw} {rh} {name}.depth.tex", + f"direct motion {DXGI_RG16F} {mw} {mh} {name}.motion.tex", + f"direct_params 0 0 {rw} {rh} 1.0 {rne16} ms={float(ms[0])!r},{float(ms[1])!r} " + f"mo={float(mo[0])!r},{float(mo[1])!r}"] + if out_fmt: + lines += [f"direct output {out_fmt} {out_wh[0]} {out_wh[1]} {name}.outinit.tex", f"direct_out {name}.out.d3d12"] + else: # a direct frame binds all four textures: the input stage's frame gets an output it never writes + lines += [f"direct output {DXGI_RGBA16F} {rw} {rh} -"] + return lines + + +def run_direct(build, rocm, seeds, cdir, hip, stages=("input", "output"), tag="dir", with_expo=True): + """stages: "input" (kx/k_in_tiled_direct), "legacy" (kx/k_in_color_direct + kx/k_in_base_direct, and kx/k_in_reset, the + buffer form that a direct frame runs on a reset frame), "output" (k_out_color_direct + k_out_pixel_c_direct, or + k_out_pixel_direct with the head map).""" + from clean.k.dx12 import k_plan_bridge as PB + from clean.k.runner import k_io, k_stages + from clean.k.runner.k_io import KExpoArgs + from clean.k.tests.test_k_io_emu import frame + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed + 1300 + (400 if stages == ("legacy",) else 0)) + for ci, (hdr, mvlo, inv, reset) in enumerate(INPUT_FLAG_SETS): + base = INPUT_BASES[(ci + si) % len(INPUT_BASES)] + special = (ci // 2) % 3 + for stage in stages: + if stage == "output" and reset and ci % 4: # a reset frame's output stage on a few cases only + continue + name = f"{tag}_{stage[:3]}_s{si}_{ci}" + compact = (ci // 3 + si) % 2 == 0 # output stage: the compact pixel head, else the head map (k_head=full) + x = frame(rng, hdr, mvlo, inv, reset=reset, base=base) + x["base"] = base + x.update(E_cur=1.5, E_prev=1.0, E_aux=0.75, pre_exposure=2.0) + if stage == "legacy": # every fourth case: render size = output size (DLAA 1.0), or a jitter outside [-0.5, 0.5] + legacy_frame(x, rng, ci % 4 == 1, ci % 4 == 2) + x["color"][..., 3] = rng.uniform(-1.0, 2.0, x["color"].shape[:2]).astype(np.float32) + if special: + input_specials(x, rng, special) + if stage == "output": + x["N"][rng.random(x["N"].shape) < 0.002] = np.array(np.inf if special == 1 else np.nan, np.float16) + c16, d32, m16, ms, mo = direct_inputs(x, rng) + x["color"] = c16.astype(np.float32) # the reference sees the same texel values + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, "input@exact" if stage in ("input", "legacy") else + ("output@compact" if compact else "output"), b"", x, None, None) + try: + launches = [ln for ln in plan["launches"] if ln[0] != "k_out_hidden"] + prm = launches[0][3][0] + f = prm.f + xs = np.array([f.kappa, f.rho, f.lam, np.float32(1.0) / np.float32(f.kappa)], np.float32) + ptr = pdev.malloc(xs.nbytes) + pdev.upload(ptr, xs.tobytes()) + plan["bufs"]["xs"], plan["sizes"]["xs"] = ptr, xs.nbytes + fs = [f] + if stage == "legacy": # k_in_color and k_in_base / k_in_reset each carry their own parameter block + names = [ln[0].split(":")[-1] for ln in launches] + assert names == ["k_in_color", "k_in_reset" if reset else "k_in_base"], names + fs = [ln[3][0].f for ln in launches] + for ff in fs: + ff.xs = ptr + ff.kappa, ff.rho, ff.lam = -17.0, 7.0, 9.0 + # the raw-buffer route: RGBA16F colour rows, raw RG16F motion with ms / mo, R32F depth + rh, rw = c16.shape[:2] + mh, mw = m16.shape[:2] + cb, mbuf = pdev.malloc(c16.nbytes), pdev.malloc(m16.nbytes) + pdev.upload(cb, c16.tobytes()) + pdev.upload(mbuf, m16.tobytes()) + plan["bufs"]["color16"], plan["sizes"]["color16"] = cb, c16.nbytes + plan["bufs"]["motion16"], plan["sizes"]["motion16"] = mbuf, m16.nbytes + for ff in fs: + ff.color, ff.motion = cb, mbuf + ff.cpitch, ff.mpitch, ff.mch = rw, mw, 2 + ff.flags |= KIO_C16 | KIO_M16 + ff.rmsx, ff.rmsy, ff.rmox, ff.rmoy = (float(v) for v in (*ms, *mo)) + omode = 0 + if stage == "legacy": + n_px = f.rw * f.rh + plan["outputs"] = list(plan["outputs"]) + [("cpre", (n_px, 4)), ("cpre2", (n_px, 2))] + modules, hipco = ["k_io_exact.co kx"], "k_io_exact.co" + d3name = lambda k: "kx/" + k.split(":")[-1] + ("" if k.endswith("k_in_reset") else "_direct") + out_fmt, rne16 = None, 0 + elif stage == "input": + bufs = {n: getattr(prm, n) for n in ("Hc", "Hh", "x0", "cpre", "cpre2")} + plan["launches"] = k_io.input_launches(f, bufs, exact=True, tiled=True) + if len(plan["launches"]) != 1: + continue + if reset: + act = plan["launches"][0][3][0] + act.Hc = act.Hh = None + act.f.motion = act.f.depth = None + modules, hipco = ["k_io_exact.co kx"], "k_io_exact.co" + d3name = lambda k: "kx/" + k.split(":")[-1] + "_direct" + out_fmt, rne16 = None, 0 + else: + omode = (ci + si) % 2 # 0 RGBA32F, 1 RGBA16F rows + prm.omode, prm.opitch = omode, f.W + plan["launches"] = launches + plan["outputs"] = [(n, sh) for n, sh in plan["outputs"] if n != "Hhout"] + plan["outputs"] += [("ctap", (f.rh, f.rw, 4)), ("ctapT", (f.rh, f.rw))] + modules, hipco = ["k_io.co"], "k_io.co" + d3name = lambda k: k + "_direct" + out_fmt, rne16 = (DXGI_RGBA16F if omode else DXGI_RGBA32F), (1 if omode else 0) + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + write(os.path.join(cdir, f"{name}.color.tex"), c16.tobytes()) + write(os.path.join(cdir, f"{name}.depth.tex"), d32.tobytes()) + write(os.path.join(cdir, f"{name}.motion.tex"), m16.tobytes()) + for k, grid, block, a in plan["launches"]: + co, kernel = (k.split(":") if ":" in k else (hipco, k)) + hip.h.launch(hip.h.get_function(hip.mods[co], kernel), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {n: min(int(np.prod(sh)) * 2, sizes[n]) for n, sh in plan["outputs"]} + widths, saves, out_wh = {}, outputs, (0, 0) + if stage == "legacy": + defined.update({"x0": min(f.Bw * f.Bh * 32, sizes["x0"]), "cpre": n_px * 16, "cpre2": n_px * 8}) + widths = {"cpre": 4, "cpre2": 4} + if stage == "output": + out_wh = (f.W, f.H) + bpp = 8 if omode else 16 + defined["out"] = f.W * f.H * bpp + got["out"] = got["out"][:defined["out"]] # the texture holds the image only + widths["out"] = 2 if omode else 4 + write(os.path.join(cdir, f"{name}.outinit.tex"), bytes([D3D12_SENTINEL]) * defined["out"]) + saves = [n for n in outputs if n != "out"] # the image comes back through direct_out + extra = direct_lines(name, c16, m16, ms, mo, rne16, out_fmt, out_wh) + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, modules=modules, outputs=saves, + d3d12_name=d3name, extra=extra) + if neg is None and plan["launches"][-1][1] >= 4 and not reset: + neg = name + extra_neg = [l.replace(f"direct_out {name}.out.d3d12", f"direct_out negative_{name}.out.d3d12") + for l in extra] + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, modules=modules, + outputs=saves, d3d12_name=d3name, extra=extra_neg, prefix="negative_", + grid_div=2) + kernel = {"input": "kx/k_in_tiled_direct", + "legacy": "kx/k_in_reset" if reset else "kx/k_in_base_direct", + "output": "k_out_pixel_c_direct" if compact else "k_out_pixel_direct"}[stage] + items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(base), "offset": [0, 0], + "specials": special, "outputs": outputs, "hip": got, "defined": defined, + "widths": widths}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + # the exposure texel: k_expo_scalars_direct reads E_cur from the game's exposure texture (src_mode 1) + sizes = expo_sizes() + for si, seed in enumerate(seeds if with_expo else []): + rng = np.random.default_rng(seed + 1700) + for ci, (a, st, src, meas) in enumerate([c for c in expo_cases(rng) if c[0]["src_mode"] == 1 and c[0]["hdr"]]): + name = f"dir_exp_s{si}_{ci}" + outs = hip.run("k_io.co", "k_expo_scalars", 1, 32, + [("buf", 0), ("buf", 1), ("buf", 2), ("buf", 3), KExpoArgs(**a)], + [(st, sizes["st"]), (src, sizes["src"]), (meas, sizes["measured"]), (None, sizes["xs"])]) + write(os.path.join(cdir, name + ".st"), st) + write(os.path.join(cdir, name + ".src"), src) + write(os.path.join(cdir, name + ".meas"), meas) + write(os.path.join(cdir, name + ".args"), expo_struct(a)) + write(os.path.join(cdir, name + ".expo.tex"), src[:4]) + lines += chain_lines(name, "", 1, sizes, "k_expo_scalars_direct", "k_io.co", + extra=[f"direct exposure {DXGI_R32F} 1 1 {name}.expo.tex", + f"direct color {DXGI_RGBA16F} 1 1 -", f"direct depth {DXGI_R32F} 1 1 -", + f"direct motion {DXGI_RG16F} 1 1 -", f"direct output {DXGI_RGBA16F} 1 1 -", + "direct_params 0 0 1 1 1.0 0 expo=1"]) + items.append({"case": name, "kernel": "k_expo_scalars_direct", "stage": "expo", "size": [1, 1], + "offset": [0, 0], "specials": 0, "outputs": ["st", "xs"], "hip": {"st": outs[0], "xs": outs[3]}, + "defined": {"st": 16, "xs": 16}, "widths": {}, "d3d12_files": {"st": ".out_st", "xs": ".out_xs"}}) + return items, lines, neg + + +# ---- family legacy: the legacy exact input kernels k_in_color + k_in_reset / k_in_base (k_io_exact.co) ----------------- +# The runtime takes them where the tiled kernel does not apply: render size above 0.68 x the output (up to 1.0, DLAA), +# a jitter outside [-0.5, 0.5]. Inputs, intermediates (cpre, cpre2) and the output x0 are compared raw. + +def legacy_frame(x, rng, ratio_one, wide_jitter): + """x with the render size the output's (DLAA) and/or a jitter outside [-0.5, 0.5].""" + from clean.k.tests.test_k_io_emu import W, H + if ratio_one: + x["render_size"] = (W, H) + x["color"] = (rng.random((H, W, 4)) * (4 if x["hdr"] else 1)).astype(np.float32) + x["depth"] = rng.random((H, W)).astype(np.float32) + x["motion"] = (rng.standard_normal((H, W, 2)) * 0.8).astype(np.float32) + if wide_jitter: + x["jitter"] = (0.75, -0.8) + return x + + +def run_legacy(build, rocm, seeds, cdir, hip): + from clean.k.dx12 import k_plan_bridge as PB + from clean.k.runner import k_io, k_stages + from clean.k.tests.test_k_io_emu import frame + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed + 2100) + for ci, (hdr, mvlo, inv, reset) in enumerate(INPUT_FLAG_SETS): + base = INPUT_BASES[(ci + si) % len(INPUT_BASES)] + special = (ci // 2) % 3 + ratio_one, wide_jitter = ci % 4 == 1, ci % 4 == 2 # every fourth case: ratio 1.0, jitter outside the guard + name = f"leg_s{si}_{ci}" + x = frame(rng, hdr, mvlo, inv, reset=reset, base=base) + x["base"] = base + x.update(E_cur=1.5, E_prev=1.0, E_aux=0.75, pre_exposure=2.0) + legacy_frame(x, rng, ratio_one, wide_jitter) + if special: + input_specials(x, rng, special) + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, "input@exact", b"", x, None, None) # k_in_color + k_in_base / k_in_reset + try: + prm = plan["launches"][-1][3][0] + f = prm.f + xs = np.array([f.kappa, f.rho, f.lam, np.float32(1.0) / np.float32(f.kappa)], np.float32) + ptr = pdev.malloc(xs.nbytes) + pdev.upload(ptr, xs.tobytes()) + plan["bufs"]["xs"], plan["sizes"]["xs"] = ptr, xs.nbytes + f.xs = ptr + f.kappa, f.rho, f.lam = -17.0, 7.0, 9.0 + names = [ln[0].split(":")[-1] for ln in plan["launches"]] + assert names == ["k_in_color", "k_in_reset" if reset else "k_in_base"], names + n_px = f.rw * f.rh + plan["outputs"] = list(plan["outputs"]) + [("cpre", (n_px, 4)), ("cpre2", (n_px, 2))] + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + for k, grid, block, a in plan["launches"]: + co, kernel = k.split(":") + hip.h.launch(hip.h.get_function(hip.mods[co], kernel), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {"x0": min(f.Bw * f.Bh * 32, sizes["x0"]), "cpre": n_px * 16, "cpre2": n_px * 8} + widths = {"cpre": 4, "cpre2": 4} + kw = dict(modules=["k_io_exact.co kx"], outputs=outputs, d3d12_name=lambda k: "kx/" + k.split(":")[-1]) + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, **kw) + if neg is None and plan["launches"][0][1] >= 4 and not reset: + neg = name + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, prefix="negative_", grid_div=2, + **kw) + items.append({"case": name, "kernel": "kx/k_in_base" if not reset else "kx/k_in_reset", + "stage": "input", "size": list(base), "offset": [0, 0], "specials": special, + "outputs": outputs, "hip": got, "defined": defined, "widths": widths, + "ratio_one": ratio_one, "wide_jitter": wide_jitter}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + return items, lines, neg + + # ---- run ----------------------------------------------------------------------------------------------------------- def run(build, rocm, families, seeds, json_out): @@ -626,9 +912,18 @@ def run(build, rocm, families, seeds, json_out): if "narrow" in families: state["narrow"] = run_stages(NARROW, "k_block.co", "nar", build, rocm, seeds, cdir, hip) lines += state["narrow"][1] + if "legacy" in families: + state["legacy"] = run_legacy(build, rocm, seeds, cdir, hip) + lines += state["legacy"][1] if "input" in families: state["input"] = run_input(build, rocm, seeds, cdir, hip) lines += state["input"][1] + if "direct" in families: + state["direct"] = run_direct(build, rocm, seeds, cdir, hip) + lines += state["direct"][1] + if "directleg" in families: + state["directleg"] = run_direct(build, rocm, seeds, cdir, hip, stages=("legacy",), tag="dirl", with_expo=False) + lines += state["directleg"][1] if "output" in families: state["output"] = run_output(build, rocm, seeds, cdir, hip) lines += state["output"][1] @@ -687,7 +982,7 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 - for fam in ("narrow", "wide", "input", "output"): + for fam in ("narrow", "wide", "input", "legacy", "output", "direct", "directleg"): if fam in state: res["results"][fam], good = check_narrow(state[fam], read) ok = ok and good diff --git a/clean/k/dx12/kernels/k_in.hlsl b/clean/k/dx12/kernels/k_in.hlsl index ff103a3c..41178a92 100644 --- a/clean/k/dx12/kernels/k_in.hlsl +++ b/clean/k/dx12/kernels/k_in.hlsl @@ -19,11 +19,18 @@ // intermediate goes through kf_opq (k_fp.hlsli). FP32 denormals are flushed here (HIP keeps them): none occur in the // port checks' frames (clean/k/dx12/k_port_check.py). // -// Kernel declared here: kx/k_in_tiled (KInParams; computes KiHost on the GPU). The forms with KiHost in the arguments -// (kx/k_in_tiled_uniform, K_UNIFORM) and the forms reading the game's textures (_direct, WG_DIRECT_TEXTURES) are in this -// source for the runtime's use but are not declared yet: they are declared and checked with the runtime wiring. +// Kernels (the names the runtime launches; runtime_k.cpp selects k_in_tiled_uniform when input_uniforms_safe accepts +// the frame's FP32 values, else k_in_tiled, which computes the same KiHost on the GPU): +// kx/k_in_tiled_uniform KInUniformParams (KInParams + KiHost) +// kx/k_in_tiled KInParams +// kx/k_in_tiled_direct the same kernel for the native executor (rs=direct, model K's direct I/O: colour / depth / +// motion from the game's textures t0 / t1 / t2, k_frame.hlsli kio_texel / kio_motion / +// kio_depth; WG_DIRECT_TEXTURES); the uniform form has no direct form (the native executor has +// no host frame values) // The KInParams pointers cpre / cpre2 are null in the tiled launches (runtime_k.cpp), as in HIP unused here. +// @wg-kernel kx/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1 // @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 +// @wg-kernel kx/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1 #include "wg_platform.hlsli" #ifndef K_UNIFORM #define K_UNIFORM 0 @@ -47,99 +54,7 @@ WG_ARGS_END float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } -// ---- the exact input stage's arithmetic (k_io.hip with KI_FAST = 0) -------------------------------------------------- -float ki_log2(float x) { return kf_log2(x); } -float ki_exp2(float x) { return kf_exp2(x); } -float ki_rcp(float x) { return kf_div(1.0f, x); } -float ki_div(float a, float b) { return kf_div(a, b); } -// spec 42 decode factor D(t) -float ki_decode(float t) -{ - if (t < kf_bits(0x3F7FBE88u)) return ki_rcp(1.0f - t); - const float e = kf_opq((t + kf_bits(0x3C4FAE29u)) * 9.0f) * kf_bits(0x3FB8AA3Bu); - return ki_div(ki_exp2(e) - 1.0f, 9.0f * t); -} -float ki_phi(float Y, bool hdr) -{ - if (hdr) { - const float Yp = kf_max(Y, 0.0f); - return ki_div(ki_log2(kf_fma(Yp, 12.0f, 1.0f)) * kf_bits(0x3F317218u), Yp + kf_bits(0x358637BDu)); - } - return kf_fma(Y, 3.0f, 1.0f); -} -float ki_S(float z) -{ - const float zp = kf_min(kf_max(z, 0.0f), 1.0f); // HIP's NaN test is implied: maxNum(NaN, 0) = 0 - if (!(zp > kf_bits(0x3B4D2E1Cu))) return kf_bits(0x414EB852u) * zp; - const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once - return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu)); -} -// z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart -// (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it) -float ki_T(float z) -{ - return z * kf_max(ki_rcp(z + 1.0f), kf_bits(0x38D1B717u)); -} -float ki_L(float Y, bool hdr) { return hdr ? Y : ki_div(Y, kf_max(1.0f - kf_opq(Y), kf_bits(0x3E4CCCCDu))); } - -struct KiFine { // spec 42 section 5 per fine position - bool valid; - float3 h; // reprojected history (Y, Co, Cg) - float nx, ny; // hidden-history position (normalized) -}; - -KiFine ki_fine(KInParams P, KiHost k, int fx, int fy) -{ - const KFrameIn F = P.f; - const int W16 = kio_ceil16(F.W), H16 = kio_ceil16(F.H); - const float px = (float)fx + 0.5f, py = (float)fy + 0.5f; - float mvx, mvy; - if ((F.flags & KIO_MVLO) != 0) { - const int ix = (int)kf_R(F.jx + kf_fma(px, k.a_x, -0.5f)), iy = (int)kf_R(F.jy + kf_fma(py, k.a_y, -0.5f)); - int dx, dy; - kio_nearest(F, ix, iy, dx, dy); - kio_motion(F, kio_mtap(ix, dx, F.mbx, F.rw), kio_mtap(iy, dy, F.mby, F.rh), mvx, mvy); - } else { - const int mx = (int)kf_R(kf_fma(px, k.mp_x, -0.5f)), my = (int)kf_R(kf_fma(py, k.mp_y, -0.5f)); - kio_motion(F, kio_tap(mx, 0, F.mbx, F.W), kio_tap(my, 0, F.mby, F.H), mvx, mvy); - } - // rounded before px + m (HIP); no kf_opq needed: the ISA keeps the products (precaution link dropped, ISA-checked) - // the texture's motion scale folded into the factor and its offset in raw units, as the buffer form gets them in - // WgRtFrame (hlsl/k_frame.hlsli kio_motion; issue #26): direct forms b1's ms / mo, KIO_M16 rmsx .. -#if WG_DIRECT_TEXTURES - const bool dfold = true; - const float dmsx = nd_msx, dmsy = nd_msy, dmox = nd_mox, dmoy = nd_moy; -#else - const bool dfold = (F.flags & KIO_M16) != 0; - const float dmsx = F.rmsx, dmsy = F.rmsy, dmox = F.rmox, dmoy = F.rmoy; -#endif - float spx = k.sp_x, spy = k.sp_y, ofx = F.mox, ofy = F.moy; - [branch] if (dfold) { - const bool dmvlo = (F.flags & KIO_MVLO) != 0; - spx = ki_dsp(F.sx, dmsx, F.W, dmvlo ? F.rw : F.W); - spy = ki_dsp(F.sy, dmsy, F.H, dmvlo ? F.rh : F.H); - ofx = ki_dmo(F.mox, dmox, dmsx); - ofy = ki_dmo(F.moy, dmoy, dmsy); - } - const float mx = spx * (mvx + ofx), my = spy * (mvy + ofy); - const float wx = px + mx, wy = py + my; - KiFine r; - const float ux = wx * k.xi_x, uy = wy * k.xi_y; - r.valid = ux >= 0.0f && ux <= 1.0f && uy >= 0.0f && uy <= 1.0f; - const float4 s = kio_bilinear4(P.Hc, W16, H16, wx * k.h_x, wy * k.h_y); - if ((F.flags & KIO_HDR) != 0) { - const float f = ki_decode(s.x); - r.h = F.rho * kf_opq3(s.xyz * f); - } else { - r.h = s.xyz; - } - r.nx = ((float)(4 * (fx >> 2) + 2) + mx) * k.g_x; // fx, fy >= 0: fx / 4 = fx >> 2 - r.ny = ((float)(4 * (fy >> 2) + 2) + my) * k.g_y; - return r; -} - -// kappa max(RGB, 0), each product rounded (kf_opq: no merging with the 0.25 / 0.5 factors that follow) -float3 ki_rgb(KFrameIn F, float4 c) { return kf_opq3(F.kappa * max(c.xyz, 0.0f)); } +#include "k_in_common.hlsli" struct KiTiledTap { float4 cv; // Y, Co, Cg, phi(Y) @@ -173,11 +88,6 @@ float4 ki_tiled_transpose(float4 v, int lane) const float z = wg_xor(b1 ? a.x : a.z, 2u), w = wg_xor(b1 ? a.y : a.w, 2u); return float4(b1 ? z : a.x, b1 ? w : a.y, b1 ? a.z : z, b1 ? a.w : w); } -void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3) -{ - wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, - uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); -} groupshared float4 tile_cv[24 * 24]; groupshared float tile_om[24 * 24]; diff --git a/clean/k/dx12/kernels/k_in_legacy.hlsl b/clean/k/dx12/kernels/k_in_legacy.hlsl new file mode 100644 index 00000000..94bd4f65 --- /dev/null +++ b/clean/k/dx12/kernels/k_in_legacy.hlsl @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of model K's legacy exact input-stage kernels (clean/k/kernels/k_io.hip k_in_color, k_in_reset, k_in_base; +// HIP build k_io_exact.co = WGK_IN_EXACT, wave64, 128 lanes per workgroup; spec 42). WrongGPU's own code. The runtime +// takes them when the tiled input stage does not apply: a render size above 0.68 x the output (native AA, DLAA-like +// ratios up to 1.0), or a jitter outside [-0.5, 0.5], and for a base size outside the tiled guard. +// +// k_in_color: per render pixel (i, j) the colour tap values (Y, Co, Cg, phi(Y)) into cpre and the HDR weight and the +// own sample's channel 0 into cpre2. k_in_reset: a reset frame's x0 from the own sample's channel 0 only. k_in_base: one +// lane per display position, 32 base positions per workgroup; the four display positions of a base position meet in LDS +// for the hidden-history channel. Same arithmetic, same order and same roundings as HIP, with the exact path of +// WGK_IN_EXACT (hlsl/k_in_common.hlsli: correctly rounded log2 / 2^x / divisions and reciprocals, FP16 rounding to +// nearest even); where the D3D12 compiler could merge two of HIP's roundings (a product by a uniform feeding an add or +// a subtraction) the intermediate goes through kf_opq, exactly where the tiled kernel (k_in.hlsl) does. A luminance +// weight of LDR is 0 in cpre2 (HIP), and the LDR sum is a plain addition (HIP's A + v), not a fused multiply-add. +// The direct forms (rs=direct, the native executor: colour / depth / motion from the game's textures) are the same +// source with WG_DIRECT_TEXTURES; k_in_reset reads only buffers and has no direct form. +// FP32 denormals are flushed here (HIP keeps them), as in every D3D12 port. +// +// @wg-kernel kx/k_in_color co=k_io_exact.co entry=k_in_color block=128 wave=64 args=s200:p4.b128.p5 +// @wg-kernel kx/k_in_reset co=k_io_exact.co entry=k_in_reset block=128 wave=64 args=s200:p4.b128.p5 +// @wg-kernel kx/k_in_base co=k_io_exact.co entry=k_in_base block=128 wave=64 args=s200:p4.b128.p5 +// @wg-kernel kx/k_in_color_direct co=k_io_exact.co entry=k_in_color block=128 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1 +// @wg-kernel kx/k_in_base_direct co=k_io_exact.co entry=k_in_base block=128 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1 +#include "wg_platform.hlsli" +#ifndef WG_DIRECT_TEXTURES +#define WG_DIRECT_TEXTURES 0 +#endif +#if WG_DIRECT_TEXTURES +#include "native_direct.hlsli" +#endif +#include "k_fp.hlsli" +#include "k_frame.hlsli" + +WG_ARGS_BEGIN + K_ARGS_FRAME_IN + K_ARGS_IN_PTRS +WG_ARGS_END + +float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } + +#include "k_in_common.hlsli" + +groupshared float4 lds_hid[128]; + +// Per render pixel: the operations k_in_base would do for a colour tap or an own sample at clamped position (i, j) +WG_KERNEL(k_in_color) +{ + WG_SETUP(); + KF_SETUP(); + KInParams P0; + K_LOAD_IN_PARAMS(P0) + const KInParams P = ki_params(P0); + const KFrameIn F = P.f; + const int id = (int)(wg_block_id() * 128u + wg_lane_id()); + if (id >= F.rw * F.rh) return; + const int i = id % F.rw, j = id / F.rw; + const bool hdr = (F.flags & KIO_HDR) != 0; + const float4 c = kio_texel(F, F.cbx + i, F.cby + j); + const float3 rgb = ki_rgb(F, c); + float Y, Co, Cg; + kio_ycocg(rgb.x, rgb.y, rgb.z, Y, Co, Cg); + wg_st128(P.cpre, (uint)id * 16u, asuint(float4(Y, Co, Cg, ki_phi(Y, hdr)))); + const float Yc = kio_luma_fma(rgb.x, rgb.y, rgb.z); + wg_stf(P.cpre2, (uint)id * 8u, hdr ? kf_max(ki_rcp(Y + 1.0f), kf_bits(0x38D1B717u)) : 0.0f); + wg_stf(P.cpre2, (uint)id * 8u + 4u, ki_S(ki_T(F.lam * ki_L(Yc, hdr)))); +} + +// A reset frame: channel 1 equals channel 0 and channels 2 / 3 are zero (spec 42 sections 6.1, 6.5 and 7) +WG_KERNEL(k_in_reset) +{ + WG_SETUP(); + KF_SETUP(); + KInParams P; + K_LOAD_IN_PARAMS(P) + const KFrameIn F = P.f; + const int lane = (int)wg_lane_id(); + const int id = (int)(wg_block_id() * 32u) + (lane >> 2), k = lane & 3; + if (id >= F.Bw * F.Bh) return; + const int bx = id % F.Bw, by = id / F.Bw; + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + const KiHost hk = ki_host(F); + const int fx = 2 * u + 1 - (v & 1), fy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)fx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)fy + 0.5f, hk.a_y, -0.5f)); + const float ch0 = wg_ldf(P.cpre2, (((uint)kio_tap(ry, 0, 0, F.rh) * (uint)F.rw + (uint)kio_tap(rx, 0, 0, F.rw)) * 2u + 1u) * 4u); + ki_store_x0(P.x0, id, k, ch0, ch0, 0.0f, 0.0f); +} + +// One lane per display position: lanes 4b .. 4b + 3 are the display positions k = 0..3 of base position b +WG_KERNEL(k_in_base) +{ + WG_SETUP(); + KF_SETUP(); + KInParams P0; + K_LOAD_IN_PARAMS(P0) + const KInParams P = ki_params(P0); + const KFrameIn F = P.f; + const int lane = (int)wg_lane_id(); + const int id = (int)(wg_block_id() * 32u) + (lane >> 2), k = lane & 3; + const bool active = id < F.Bw * F.Bh; // inactive lanes still reach the barrier + const int bx = active ? id % F.Bw : 0, by = active ? id / F.Bw : 0; + const bool hdr = (F.flags & KIO_HDR) != 0; + const KiHost hk = ki_host(F); + const float c1e4 = kf_bits(0x38D1B717u); + float ch0 = 0.0f, ch1 = 0.0f, ch3 = 0.0f; + float4 hid = 0.0f; + if (active) { + const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1); + // colour neighbourhood + const int cx = (int)kf_R(F.jx + kf_fma((float)(2 * u + 1), hk.a_x, -0.5f)); + const int cy = (int)kf_R(F.jy + kf_fma((float)(2 * v + 1), hk.a_y, -0.5f)); + float3 A = 0.0f, lo = 0.0f, hi = 0.0f; + [unroll] for (int ddy = -1; ddy <= 1; ++ddy) { + [unroll] for (int ddx = -1; ddx <= 1; ++ddx) { + // the tap's values from k_in_color (render pixel kio_tap(.., 0, n) of the subrect) + const uint t = (uint)kio_tap(cy, ddy, 0, F.rh) * (uint)F.rw + (uint)kio_tap(cx, ddx, 0, F.rw); + const float4 cv = wg_ld4f(P.cpre, t * 16u); + if (hdr) { + const float om = wg_ldf(P.cpre2, t * 8u); + A = float3(kf_fma(om, cv.x, A.x), kf_fma(om, cv.y, A.y), kf_fma(om, cv.z, A.z)); + } else { + A = float3(A.x + cv.x, A.y + cv.y, A.z + cv.z); // HIP's A + v (no weight) + } + const float3 val = cv.xyz * cv.w; + const bool first = ddx == -1 && ddy == -1; + lo = first ? val : min(lo, val); + hi = first ? val : max(hi, val); + } + } + float3 mu = A * kf_bits(0x3DE38E39u); + if (hdr) { + const float den = kf_max(1.0f - mu.x, c1e4); + mu = float3(ki_div(mu.x, den), ki_div(mu.y, den), ki_div(mu.z, den)); + } + const float pm = ki_phi(mu.x, hdr); + const float3 mp = kf_opq3(mu * pm); + const float3 eta = 0.5f * (hi - lo); + const float3 nu = float3(kf_fma(hi.x - lo.x, 8.0f, 1.0f), kf_fma(hi.y - lo.y, 8.0f, 1.0f), kf_fma(hi.z - lo.z, 8.0f, 1.0f)); + const int oq = (1 - (v & 1)) + 2 * (u & 1); // (2u + 1 - (v mod 2), 2v + (u mod 2)) as a fine position index + // fine positions in the order (2u, 2v), (2u+1, 2v), (2u, 2v+1), (2u+1, 2v+1); each scored as it is produced + bool valid = (F.flags & KIO_RESET) == 0; + float D = 0.0f, own_hx = 0.0f, own_nx = 0.0f, own_ny = 0.0f; + [unroll] for (int q = 0; q < 4; ++q) { + const KiFine fq = ki_fine(P, hk, 2 * u + (q & 1), 2 * v + (q >> 1)); + valid = valid && fq.valid; + const float ph = ki_phi(fq.h.x, hdr); + const float3 g = max(abs(mp - kf_opq3(fq.h * ph)) - eta, 0.0f); + const float3 dl = float3(ki_div(g.x, nu.x), ki_div(g.y, nu.y), ki_div(g.z, nu.z)); + const float score = kf_min(kf_fma(dl.z, 0.5f, kf_fma(dl.x, 2.0f, 0.5f * dl.y)), 1.0f); + D = D + score; + if (q == oq) { + own_hx = fq.h.x; + own_nx = fq.nx; + own_ny = fq.ny; + } + } + D = D * 0.25f; + // own sample: channel 0 from k_in_color's cpre2 + const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); + const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); + const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); + ch0 = wg_ldf(P.cpre2, (((uint)kio_tap(ry, 0, 0, F.rh) * (uint)F.rw + (uint)kio_tap(rx, 0, 0, F.rw)) * 2u + 1u) * 4u); + const float4 hs = kio_bilinear4(P.Hh, F.Bw, F.Bh, own_nx, own_ny); + hid = valid ? hs : 0.0f; + // channel 1 is channel 0's function of Yh, and Yh = Yc without a valid history: then it equals channel 0 + ch1 = valid ? ki_S(ki_T(F.lam * ki_L(own_hx, hdr))) : ch0; + const float d3 = valid ? D : 0.0f; + ch3 = kf_finite(d3) ? d3 : 0.0f; + } + lds_hid[lane] = hid; + wg_barrier_lds(); + if (!active) return; + // slot k, channel 2: 0.25 ((s0 + s3) + (s1 + s2)), s_m = channel k of slot m's hidden-history sample + const int g4 = lane & ~3; + float hsum = (lds_hid[g4][k] + lds_hid[g4 + 3][k]) + (lds_hid[g4 + 1][k] + lds_hid[g4 + 2][k]); + hsum = hsum * 0.25f; + ki_store_x0(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3); +} diff --git a/clean/k/dx12/kernels/k_out.hlsl b/clean/k/dx12/kernels/k_out.hlsl index dd19d394..da700b4b 100644 --- a/clean/k/dx12/kernels/k_out.hlsl +++ b/clean/k/dx12/kernels/k_out.hlsl @@ -1,7 +1,7 @@ // SPDX-License-Identifier: MIT // D3D12 ports of model K's output stage (spec 43): k_out_color, k_out_pixel_c (compact pixel head, the default) -// and k_out_pixel (k_head=full). The forms that read the game's textures (_direct, WG_DIRECT_TEXTURES) are in this source -// for the runtime's use but are not declared yet: they are declared and checked with the runtime wiring. WrongGPU's own code. +// and k_out_pixel (k_head=full). The forms that read the game's textures and write its output texture (_direct, +// rs=direct, WG_DIRECT_TEXTURES) are declared beside them (see k_in.hlsl). WrongGPU's own code. // Basis: our HIP kernels in clean/k/kernels/k_io.hip (HIP build k_io.co, wave64, 128 lanes per workgroup), exact forms // only (FAST = false; the k_fast=1 "_f" kernels are not ported and keep running on HIP). Same names, same argument // struct (KOutParams, see hlsl/k_out.hlsli for the s-token and why each skipped field is dead), same launch geometry @@ -44,6 +44,9 @@ // @wg-kernel k_out_color co=k_io.co block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 // @wg-kernel k_out_pixel_c co=k_io.co block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 defines=KO_COMPACT=1 // @wg-kernel k_out_pixel co=k_io.co entry=k_out_pixel_c block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 defines=KO_COMPACT=0 +// @wg-kernel k_out_color_direct co=k_io.co entry=k_out_color block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 rs=direct defines=WG_DIRECT_TEXTURES=1 +// @wg-kernel k_out_pixel_c_direct co=k_io.co entry=k_out_pixel_c block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 rs=direct defines=KO_COMPACT=1,WG_DIRECT_TEXTURES=1 +// @wg-kernel k_out_pixel_direct co=k_io.co entry=k_out_pixel_c block=128 wave=64 args=s312:p4.b12.x24.b52.x12.b24.x4.p4.x8.p4.b8.p2.b56 rs=direct defines=KO_COMPACT=0,WG_DIRECT_TEXTURES=1 #include "wg_platform.hlsli" #if defined(WG_DIRECT_TEXTURES) #include "native_direct.hlsli" diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index e6a54b6b..f01d01c4 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -10,6 +10,7 @@ #include #include +#include "../../converter/src/json.h" #include "../../runtime/embedded.h" #include "../../runtime/fpconv.h" #include "../../runtime/runtime_core.h" @@ -75,6 +76,13 @@ struct KBlkParams { int w, h, ox, oy, nwx, nwy, cw, ch; }; static_assert(sizeof(KBlkParams) == 184, "KBlkParams"); +// k_warm (clean/k/dx12/kernels/k_warm.hlsl, D3D12 only): up to twelve byte ranges r[i] of n[i] bytes, one 4-byte load +// per `line` bytes of each, the XOR of the loaded words stored to dummy only when it equals key (a word nobody reads) +struct KWarmParams { + DevPtr r[12], dummy; + uint32_t n[12], line, key; +}; +static_assert(sizeof(KWarmParams) == 160, "KWarmParams"); struct ExpParams { // clean/kernels/exposure.hip WgExpParams int rw, rh, nx, ny, nparts, pad0; float inv_pi, K, inv_n; @@ -249,6 +257,11 @@ const FusedIo kFusedIo[11] = { // f16 (variant "fused16", WIDE_DEFAULT16): k_stagew4h_ for e4, m, d4; 0 = k_stageh_. // FP32: e4 and d4 with NG = 4 when they have at most 24 windows (k_launch.py WIDE4_SMALL_MAX_WINDOWS; 1440p output: 24, // faster with 4; 4K output: 40, faster with 2). +// Defaults of the D3D12 latency options k_color_at / k_warm (KRuntime::create). +const char* const kColorAtDefault = "input"; +const char* const kWarmAtDefault = "e0"; +const char* const kWarmToDefault = "d0"; + int wide_ng(const std::string& s, bool f16, int nwin) { if (f16) return s == "e4" || s == "m" || s == "d4" ? 4 : 0; if (s == "m") return 4; @@ -301,7 +314,8 @@ const char* KRuntime::timing_info() { "(exposure_auto_diet, or exposure_auto_diet_c16 with input_copy, if auto-exposure, else k_expo_scalars) -> [1] start " "of input/color preparation -> [2] start of the first neural kernel -> [3] start of k_out_pixel (k_out_pixel_c with the compact pixel head) -> [4] end of " "an empty kernel after the final output-stage kernel; exposure = [0]-[1] (input conversion, exposure, k_expo_scalars), input " - "= [1]-[2] (k_in_tiled_color[_uniform], or separate k_out_color followed by k_in_tiled[_uniform] or k_in_color plus k_in_base/k_in_reset), network = [2]-[3] (eleven stage kernels, including hidden-history output in d0), output = [3]-[4] (" + "= [1]-[2] (k_in_tiled_color[_uniform], or separate k_out_color followed by k_in_tiled[_uniform] or k_in_color plus k_in_base/k_in_reset), network = [2]-[3] (eleven stage kernels, including hidden-history output in d0; on the D3D12 devices also " + "k_out_color, launched unordered after the stage k_color_at names, and k_warm when k_warm is on), output = [3]-[4] (" "k_out_pixel writing the game output)"; } @@ -312,11 +326,14 @@ bool KRuntime::alloc(DevPtr* p, size_t bytes, bool zero) { return true; } -bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool external_io) { +bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool external_io, bool native_direct) { cfg_ = c; dev_ = dev; alloc_ = std::move(alloc_fn); external_io_ = external_io; + native_direct_ = native_direct; + if (native_direct_ && (!external_io_ || c.backend != 3)) + return fail("model K: native direct resources need external I/O on the native D3D12 device (backend 3)"); build_dir_ = c.build_dir ? c.build_dir : "."; W_ = c.ow; H_ = c.oh; @@ -358,6 +375,12 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e const std::string tiled = option(c, "input_tiled"); if (!tiled.empty() && tiled != "0" && tiled != "1") return fail("input_tiled must be 0 or 1"); tiled_input_ = tiled != "0"; + // k_in_uniforms (1, the default: the tiled input kernel takes the host's frame-uniform values, k_in_tiled_uniform, + // where input_uniforms_safe allows; 0: diagnostic, always k_in_tiled, which computes the same values on the GPU: the + // same output, used by the D3D12 port check of k_in_tiled) + const std::string kuni = option(c, "k_in_uniforms"); + if (!kuni.empty() && kuni != "0" && kuni != "1") return fail("model K: k_in_uniforms must be 0 or 1"); + input_uniforms_ = kuni != "0"; const std::string diag = option(c, "k_diag"); if (!diag.empty() && diag != "input2" && diag != "copyin" && diag != "input2+copyin" && diag != "copyin+input2") return fail("model K: unknown k_diag " + diag + " (input2, copyin, input2+copyin)"); @@ -383,6 +406,54 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!krule.empty() && krule != "spec" && krule != "v051") return fail("model K: unknown k_expo_rule " + krule + " (spec or v051)"); expo_v051_ = krule == "v051"; + // Latency of the small-grid stages on the D3D12 devices (no kernel arithmetic and no data flow changes): + // k_color_at (D3D12 devices, backends 2 and 3; default kColorAtDefault): launch k_out_color unordered right after + // the named stage (dev_->unordered_next(): no UAV barrier between that stage and k_out_color; the stage after it + // waits for both), so it runs beside a latency-bound stage instead of alone before the input stage. Nothing the + // network reads is written by k_out_color (ctap, ctapT: read by k_out_pixel_c only) and nothing it reads is written + // by the network. "input": before the input stage (HIP's order; HIP and the host emulator always keep it). + // k_warm (backend 3 only; default kWarmAtDefault; "off"): a k_warm dispatch launched unordered right after the + // named stage ("input": after the input stage, before e0) reads every 128-byte line (k_warm_line) of the weight + // blobs of the stages after it up to k_warm_to (default kWarmToDefault) into the caches, so the stages do not wait + // for cold weights. k_warm_groups: its grid (default 64). + const std::string kcol = option(c, "k_color_at"); + if (!kcol.empty() && kcol != "input" && !k_stage(kcol)) return fail("model K: k_color_at: unknown stage " + kcol); + const std::string cat = kcol.empty() ? std::string(kColorAtDefault) : kcol; + color_at_ = c.backend >= 2 && cat != "input" ? cat : std::string(); + const std::string kwarm = option(c, "k_warm"); + if (!kwarm.empty() && kwarm != "off" && kwarm != "input" && + (!k_stage(kwarm) || kwarm == "d2" || kwarm == "d1" || kwarm == "d0")) + return fail("model K: k_warm: " + kwarm + " is not off, input or a stage before d2"); + warm_to_ = option(c, "k_warm_to"); + if (warm_to_.empty()) warm_to_ = kWarmToDefault; + if (!k_stage(warm_to_)) return fail("model K: k_warm_to: unknown stage " + warm_to_); + const std::string wat = kwarm.empty() ? std::string(kWarmAtDefault) : kwarm; + warm_at_ = c.backend == 3 && wat != "off" ? wat : std::string(); + const std::string kwl = option(c, "k_warm_line"), kwg = option(c, "k_warm_groups"); + if (!kwl.empty()) warm_line_ = (unsigned)std::atoi(kwl.c_str()); + if (!kwg.empty()) warm_groups_ = (unsigned)std::atoi(kwg.c_str()); + if (warm_line_ < 4 || warm_line_ > 4096 || (warm_line_ & (warm_line_ - 1))) + return fail("model K: k_warm_line must be a power of two in 4..4096"); + if (warm_groups_ < 1 || warm_groups_ > 1024) return fail("model K: k_warm_groups must be 1..1024"); + // k_fill=<0..255> (test use, changes no value a frame writes): every buffer a frame writes and both history copies are + // filled with this byte after allocation, before the first frame, so that a test with a different byte per backend + // sees a write a frame did not make (first-frame and reset-only writes included) as a difference + const std::string kfill = option(c, "k_fill"); + if (!kfill.empty()) { + char* end = nullptr; + const long v = std::strtol(kfill.c_str(), &end, 10); + if (end == kfill.c_str() || *end || v < 0 || v > 255) return fail("model K: k_fill must be 0..255"); + fill_ = (int)v; + } + // The native D3D12 device (backend 3: the D3D12 executor, or k_bench.py's own-buffer timing) runs the exact default + // path only, and only when every kernel this context can launch has a port in /dx12/manifest.json; both are + // checked here, before the model is loaded, so a refused context costs nothing more (for model K the context ends and + // the game keeps its own upscaler: there is no HIP route in a game). + if (c.backend == 3) { + if (!options_ok_direct()) return false; + const std::string missing = d3d12_missing(); + if (!missing.empty()) return fail("model K: the D3D12 build has no port of " + missing); + } map_path_ = option(c, "model_k_map"); if (map_path_.empty()) { map_path_ = join(build_dir_, "preset-K.v1.map.json"); // next to the library, else built into it @@ -417,7 +488,6 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e // k_in_exact=1: the same input kernels with exact arithmetic, addressed as "kx/" if (in_exact_ && !dev_->load_module_tagged(join(build_dir_, "k_io_exact.co"), "kx")) return fail("model K: " + dev_->err()); - // buffers for (auto& kv : blobs_) { DevPtr p; @@ -444,15 +514,30 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e !alloc(&Hh_[p], (size_t)Bw_ * Bh_ * 4 * 2, true)) return false; const int ny = std::max(256, max_rh_ / 2); - if (!alloc(&color_, R * 16) || !alloc(&depth_, R * 4) || !alloc(&motion_, R * 8) || !alloc(&cpre_, R * 16) || - !alloc(&cpre2_, R * 8) || !alloc(&ctap_, R * 8) || !alloc(&ctapT_, R * 2) || + // native_direct: no dense inputs (the kernels read the game's textures) and no legacy input-stage maps (cpre, cpre2: + // the tiled input stage, the only one with direct forms, reads neither) + if (!(native_direct_ || (alloc(&color_, R * 16) && alloc(&depth_, R * 4) && alloc(&motion_, R * 8) && + alloc(&cpre_, R * 16) && alloc(&cpre2_, R * 8))) || + !alloc(&ctap_, R * 8) || !alloc(&ctapT_, R * 2) || // exp2tab: read only by the host emulator's k_out_pixel (the GPU uses v_exp_f16); a valid, zeroed buffer !alloc(&exp2tab_, 65536 * 2, true) || !alloc(&tables_, 8 * kTableBytes) || !alloc(&expo_, 32, true) || !alloc(&expo_partials_, 4 * (size_t)cdiv(ny, 4)) || !alloc(&expo_computed_, 4, true) || - !alloc(&expo_sync_, 4, true) || !alloc(&xs_, 16, true) || !alloc(&kexpo_, 16, true)) + !alloc(&expo_sync_, 4, true) || !alloc(&xs_, 16, true) || !alloc(&kexpo_, 16, true) || + (!warm_at_.empty() && !alloc(&warm_dummy_, 16, true))) return false; // the own output buffer (tests): dense RGB f32, dense RGBA f32 (FrameIo.out_rgba32f) or RGBA16F rows that fit if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 16, true)) return false; + if (fill_ >= 0) { // k_fill (test hook): the sentinel in everything a frame writes, before the first frame + auto fillb = [&](DevPtr p, size_t bytes) { return !p || dev_->memset(p, fill_, std::max(bytes, 256)); }; + bool ok = true; + for (const auto& kv : net_) ok = ok && fillb(kv.second, net_bytes_[kv.first]); + for (int p = 0; p < 2; ++p) + ok = ok && fillb(Hc_[p], P16 * 4 * 2) && fillb(L_[p], 4 * P16 * 2) && fillb(Hh_[p], (size_t)Bw_ * Bh_ * 4 * 2); + ok = ok && fillb(ctap_, R * 8) && fillb(ctapT_, R * 2) && fillb(cpre_, R * 16) && fillb(cpre2_, R * 8) && + fillb(xs_, 16) && fillb(kexpo_, 16) && + fillb(color_, R * 16) && fillb(depth_, R * 4) && fillb(motion_, R * 8); // the own input copies (unused with caller buffers) + if (!ok) return dfail(); + } // the filter-shape table (spec 43 section 7) of every level, computed once; a frame reads its level's set std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); @@ -491,7 +576,7 @@ bool KRuntime::upload_inputs(int rw, int rh, const float* color, const float* de // The eleven stages for frame index f (k_net.py KNet.stage_plan, variant "fused" or "fused16"; k_launch.py // stage_fused_launch). -bool KRuntime::network(int f) { +bool KRuntime::network(int f, const std::function& after) { int off[11][2]; k_stage_offsets(f, offsets_table_, off); const auto& table = k_stage_table(); @@ -573,6 +658,7 @@ bool KRuntime::network(int f) { : (f16_ ? "k_stage2h_d0_hidden" : "k_stage2_d0_hidden"); if (!launch(k0, grid, kThreads, &p, &hidden)) return false; } else if (!launch(k.c_str(), grid, (unsigned)(kThreads * (ng ? ng : 1)), &p)) return false; + if (after && !after(st.name)) return false; if (i < 10) { info_.offsets[i][0] = ox; // WgRtFrameInfo has ten rows: e0 ... d1 (d0's offset is (4, 4)) info_.offsets[i][1] = oy; @@ -581,6 +667,32 @@ bool KRuntime::network(int f) { return true; } +// k_warm (backend 3): unordered after stage `at` (or the input stage), the weight blobs of the stages after it up to +// warm_to_ into the caches. +bool KRuntime::warm(const char* at) { + KWarmParams w = {}; + w.dummy = warm_dummy_; + w.line = warm_line_; + w.key = 0x9e3779b9u; + int n = 0; + bool past = std::strcmp(at, "input") == 0; // "input": from e0 on + uint64_t lines = 0; + for (const KStageInfo& st : k_stage_table()) { + if (past && n < 12) { + const size_t bytes = blobs_[st.name].size() & ~(size_t)3; + w.r[n] = wdev_[st.name]; + w.n[n++] = (uint32_t)bytes; + lines += (bytes + warm_line_ - 1) / warm_line_; + } + if (std::strcmp(st.name, at) == 0) past = true; + if (warm_to_ == st.name) break; + } + if (!n) return true; + const unsigned grid = (unsigned)std::min(warm_groups_, (lines + 255) / 256); + dev_->unordered_next(); + return launch("k_warm", grid, 256, &w); +} + bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { // reset = 2 (interop: a new context of the game took this upscaler over, wronggpu_upscaler.h): this frame is the // first frame of a newly created K runtime, as for model M. The host state a frame leaves for the next one starts @@ -593,6 +705,7 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { } const int rw = fr.rw, rh = fr.rh; if (rw <= 0 || rh <= 0 || rw > max_rw_ || rh > max_rh_) return fail("render size larger than the maximum"); + if (native_direct_ && !direct_preflight(fr, io)) return false; // Model K resets on the game's flag and on the first frame only (spec 44: the reset restarts the frame index). A // render-size change keeps the history: the histories are at output and base resolution, and the input and output // stages reproject them with this frame's render size (model M's spec 10 rule does not apply to K). @@ -619,14 +732,18 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { info_.jitter_prev[1] = reset ? fr.jitter[1] : jprev_[1]; if (!mark(0)) return false; - // inputs: the caller's dense buffers (FrameIo), the interop's raw copies converted here, or the own test buffers + // inputs: the caller's dense buffers (FrameIo), the interop's raw copies converted here, or the own test buffers; + // native_direct: none (the "_direct" kernels read the bound textures; their pointer arguments stay 0) DevPtr cin = io && io->color ? io->color : color_, din = io && io->depth ? io->depth : depth_; DevPtr min_ = io && io->motion ? io->motion : motion_; + if (native_direct_) cin = din = min_ = 0; // input_copy (io->raw_inputs): the interop's copies of the game's textures in pitched rows, read directly (KIO_C16: // RGBA16F colour, not divided by pi: K applies kappa; R32F depth; KIO_M16: FP16 motion of motion_channels channels // with m = ms r + mo applied on load, as input_convert would have written it) const bool raw = io && io->raw_inputs; int cpitch = rw, mpitch = rw, zpitch = rw, mch = 2; + if (raw && cfg_.backend >= 2) // the interop's HIP input copies: the D3D12 devices read dense buffers or the game's textures + return fail("model K: input-copy frames (raw_inputs) run on the HIP backend only"); if (raw) { mch = io->motion_channels; if ((mch != 2 && mch != 4) || io->color_pitch % 8 || io->depth_pitch % 4 || io->motion_pitch % (2 * mch)) @@ -635,7 +752,7 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { zpitch = io->depth_pitch / 4; mpitch = io->motion_pitch / (2 * mch); } - if (!cin || !din || !min_) return fail("frame without input buffers (interop)"); + if (!native_direct_ && (!cin || !din || !min_)) return fail("frame without input buffers (interop)"); if (diag_copyin_ && !raw && cin != color_) { // k_diag=copyin: the inputs from K's own allocations (dense, as FrameIo) const size_t n = (size_t)rw * rh; if (!dev_->copy(color_, cin, n * 16) || !dev_->copy(depth_, din, n * 4) || !dev_->copy(motion_, min_, n * 8)) @@ -668,8 +785,10 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { ep.inv_n = (float)(1.0 / ((double)ep.nx * ep.ny)); if (raw ? !launch("exposure_auto_diet_c16", (unsigned)ep.nparts, 256, &cin, &cpitch, &expo_partials_, &expo_sync_, &expo_computed_, &expo_, &ep, &first) - : !launch("exposure_auto_diet", (unsigned)ep.nparts, 256, &cin, &expo_partials_, &expo_sync_, - &expo_computed_, &expo_, &ep, &first)) + // native_direct: model M's direct form (dx12/kernels/exposure.hlsl: the same sums on the colour texels, + // nd_raw_color_xy, which are the input pass's dense f32 colour; inv_pi as here) + : !launch(native_direct_ ? "exposure_auto_diet_direct" : "exposure_auto_diet", (unsigned)ep.nparts, 256, + &cin, &expo_partials_, &expo_sync_, &expo_computed_, &expo_, &ep, &first)) return false; } KExpoArgs xa = {}; @@ -684,10 +803,15 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { xa.e2 = 1.0f; xa.inv_pi = 1.0f / pi; DevPtr src = mode == 1 && io ? io->expo_src : expo_computed_; + // native_direct, mode 1: k_expo_scalars_direct reads E_cur from the bound exposure texture (t3, the texel the input + // pass copies into expo_src) instead of *src; src is then a valid but unread pointer + const bool expo_tex = native_direct_ && mode == 1; + if (expo_tex) src = expo_computed_; if (mode == 1 && !src) return fail("exposure from the game's texture without a source buffer"); // k_expo_rule=v051 (A/B only): the 0.5.1 rule, the auto-exposure value as E_cur (so also as E_aux and E_prev) if (expo_v051_ && mode == 2 && auto_run) xa.src_mode = 1, xa.aux = 0, src = expo_computed_; - if (!launch("k_expo_scalars", 1, 32, &kexpo_, &src, &expo_computed_, &xs_, &xa)) return false; + if (!launch(expo_tex ? "k_expo_scalars_direct" : "k_expo_scalars", 1, 32, &kexpo_, &src, &expo_computed_, &xs_, &xa)) + return false; // the frame's inputs (k_io.py frame_in): dense render-size images, motion in render pixels ([MVlo]; the interop // converts output-resolution motion to render pixels in its input pass) @@ -734,22 +858,26 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { const char* input_kernel = in_exact_ ? (reset ? "kx/k_in_reset" : "kx/k_in_base") : (reset ? "k_in_reset" : "k_in_base"); // k_fast=1: the "_f" variant of an input / output stage kernel (k_io.hip KIO_FAST), else the exact kernel itself std::string kname_buf; + // native_direct: the kernels that read the inputs or write the output (k_out_color, the tiled input kernels, + // k_out_pixel[_c]) run as their "_direct" forms (same arguments; t0..t3 / u3 / b1 instead of the dense pointers) auto kname = [&](const char* n) -> const char* { + const bool io_kernel = std::strncmp(n, "k_in_", 5) == 0 || std::strncmp(n, "k_out_", 6) == 0; if (in_exact_ && std::strncmp(n, "k_in_", 5) == 0) { // input-stage kernels from k_io_exact.co kname_buf = std::string("kx/") + n; - return kname_buf.c_str(); + } else if (fast_) { + kname_buf = std::string(n) + "_f"; + } else { + kname_buf = n; } - if (!fast_) return n; - kname_buf = std::string(n) + "_f"; + if (native_direct_ && io_kernel) kname_buf += "_direct"; return kname_buf.c_str(); }; - const bool use_tiled = tiled_input_ && cfg_.backend == 0 && fin.s == 0.5f && W_ > 0 && W_ <= 16384 && H_ > 0 && H_ <= 16384 && - Bw_ > 0 && Bw_ <= 4096 && Bh_ > 0 && Bh_ <= 4096 && rw > 0 && rw <= .68 * W_ && rh > 0 && rh <= .68 * H_ && - std::isfinite(fin.jx) && std::isfinite(fin.jy) && std::fabs(fin.jx) <= .5f && std::fabs(fin.jy) <= .5f; + // backend 1 (the host emulator) has no tiled input kernels; HIP (0) and the D3D12 devices (2 hybrid, 3 native) do + const bool use_tiled = tiled_input_ && cfg_.backend != 1 && fin.s == 0.5f && tiled_guard(rw, rh, fin.jx, fin.jy); // k_in_exact: the output colour taps by k_out_color (KFrame's exact plan), not by the tiled input kernel const bool use_tiled_color = use_tiled && !in_exact_ && !reset && 4LL * Bw_ >= W_ && 4LL * Bh_ >= H_; KiHost input_host = {}; - const bool use_uniforms = use_tiled && input_uniforms_safe(fin, &input_host); + const bool use_uniforms = use_tiled && !native_direct_ && input_uniforms_ && input_uniforms_safe(fin, &input_host); auto input_stage = [&]() { if (use_tiled) { KInParams local = ip; @@ -768,10 +896,12 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { if (use_uniforms) ++input_uniform_launches_; ++tiled_launches_; } else { - if (!launch(in_exact_ ? "kx/k_in_color" : "k_in_color", cdiv((long long)rw * rh, kThreads), kThreads, &ip)) - return false; + const std::string color_name = std::string(in_exact_ ? "kx/k_in_color" : "k_in_color") + (native_direct_ ? "_direct" : ""); + if (!launch(color_name.c_str(), cdiv((long long)rw * rh, kThreads), kThreads, &ip)) return false; ++legacy_color_launches_; - if (!launch(input_kernel, cdiv(4LL * Bw_ * Bh_, kThreads), kThreads, &ip)) return false; + // native_direct: k_in_color / k_in_base read the game's textures (their "_direct" forms); k_in_reset reads buffers only + const std::string base_name = native_direct_ && !reset ? std::string(input_kernel) + "_direct" : input_kernel; + if (!launch(base_name.c_str(), cdiv(4LL * Bw_ * Bh_, kThreads), kThreads, &ip)) return false; ++legacy_base_launches_; } return true; @@ -785,14 +915,27 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { color.ctap = ctap_; color.ctapT = ctapT_; if (!mark(1)) return false; - if (!use_tiled_color && !launch(kname("k_out_color"), cdiv((long long)rw * rh, kThreads), kThreads, &color)) return false; + // k_color_at (D3D12 devices): k_out_color runs inside network() instead, unordered after that stage + const bool color_late = !use_tiled_color && !color_at_.empty(); + if (!use_tiled_color && !color_late && + !launch(kname("k_out_color"), cdiv((long long)rw * rh, kThreads), kThreads, &color)) + return false; if (!input_stage()) return false; tiled_last_ = use_tiled; tiled_color_last_ = use_tiled_color; input_uniforms_last_ = use_uniforms; + if (warm_at_ == "input" && !warm("input")) return false; // k_warm=input: before the first stage // network (spec 41 / 44) - if (!mark(2) || !network(f)) return false; + auto after = [&](const char* s) -> bool { + if (warm_at_ == s && !warm(s)) return false; + if (color_late && color_at_ == s) { + dev_->unordered_next(); + if (!launch(kname("k_out_color"), cdiv((long long)rw * rh, kThreads), kThreads, &color)) return false; + } + return true; + }; + if (!mark(2) || !network(f, after)) return false; // output stage (spec 43) into the caller's output: RGBA16F rows (interop output copy), dense RGBA f32 (the interop's // output pass for model K, FrameIo.out_rgba32f: the fourth value is the output alpha), dense RGB f32 (FORMATS.md @@ -815,8 +958,11 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { op.ctapT = ctapT_; const int hp = io ? io->out_half_pitch : 0; op.out = io && io->out ? io->out : out_; - if (!op.out) return fail("frame without an output buffer (interop)"); - // the own output buffer (no FrameIo output) holds RGBA16F rows only if they fit + // native_direct: k_out_pixel[_c]_direct writes the bound output texture (u3; omode below stays RGB f32: the + // RGBA16F output-copy bytes are chosen by the b1 flag output_rne16, as for model M's direct post kernel) + if (native_direct_) op.out = 0; + if (!op.out && !native_direct_) return fail("frame without an output buffer (interop)"); + // the own output buffer (no FrameIo output) holds RGBA16F rows only if they fit (k_bench.py's backend 3 timing) if (hp > 0 && op.out == out_ && (size_t)hp * H_ > (size_t)W_ * H_ * 16) return fail("model K: RGBA16F rows do not fit the own output buffer"); if (hp > 0) { @@ -864,8 +1010,107 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { return true; } +bool KRuntime::tiled_guard(int rw, int rh, float jx, float jy) const { + return W_ > 0 && W_ <= 16384 && H_ > 0 && H_ <= 16384 && Bw_ > 0 && Bw_ <= 4096 && Bh_ > 0 && Bh_ <= 4096 && + rw > 0 && rw <= .68 * W_ && rh > 0 && rh <= .68 * H_ && std::isfinite(jx) && std::isfinite(jy) && + std::fabs(jx) <= .5f && std::fabs(jy) <= .5f; +} + +// Backend 3 (native D3D12): only the exact default path has D3D12 ports (owner 2026-10-09: K exact to the original; +// the relaxed variants stay on HIP). k_head=full and k_expo_rule=v051 keep exact kernels and are allowed. +bool KRuntime::options_ok_direct() { + const char* why = f16_ ? "network=fused16 / fused8" + : fast_ ? "k_fast=1" + : !in_exact_ ? "the fast input stage (k_fast_in=1 / k_in_exact=0)" + : (diag_input2_ || diag_copyin_) ? "k_diag" + : nullptr; + if (why) + return fail(std::string("model K on the native D3D12 device runs the exact default path only (") + why + + " runs on HIP)"); + return true; +} + +std::vector KRuntime::d3d12_kernels() const { + const std::string d = native_direct_ ? "_direct" : ""; + std::vector v; + if (hdr_) v.push_back("exposure_auto_diet" + d); // auto-exposure (HDR only): at create or per frame (mode 2) + v.push_back("k_expo_scalars"); + if (native_direct_) v.push_back("k_expo_scalars_direct"); // exposure from the game's texture (mode 1) + v.push_back("k_out_color" + d); + if (!native_direct_) v.push_back("kx/k_in_tiled_uniform"); // (no direct form: the direct frame uses k_in_tiled_direct) + v.push_back("kx/k_in_tiled" + d); // input_uniforms_safe refusing the host uniforms + // the legacy input kernels: a render size above 0.68 x the output (up to 1.0, native AA / DLAA), a jitter outside + // [-0.5, 0.5], or input_tiled=0 (k_in_reset reads buffers only: no direct form) + v.push_back("kx/k_in_color" + d); + v.push_back("kx/k_in_reset"); + v.push_back("kx/k_in_base" + d); + // the eleven stages as network() names them; e4 / d4 take NG = 2 above 24 windows, which depends on the frame's + // window offset (even, 0..6): every form this output size can launch + for (const KStageInfo& st : k_stage_table()) { + int lw, lh; + k_level_size(Bw_, Bh_, st.level, &lw, &lh); + for (int o = 0; o <= 6; o += 2) { + const std::string n = st.name; + const int ng = wide_ng(n, false, k_window_count(lw, lh, o, o)); + std::string k; + if (st.head) k = compact_head_ ? "k_stage2_d0_hp" : "k_stage2_d0_hidden"; + else if (ng) k = "k_stagew" + std::to_string(ng) + "_" + n; + else k = (n == "e0" || n == "e1" || n == "d1" ? "k_stage2_" : "k_stage_") + n; + if (std::find(v.begin(), v.end(), k) == v.end()) v.push_back(k); + } + } + v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); + if (!warm_at_.empty()) v.push_back("k_warm"); // k_warm (option k_warm, backend 3) + v.push_back("timing_barrier"); // the timing marks (timing_log=1) + return v; +} + +std::string KRuntime::d3d12_missing() const { + const std::string path = join(build_dir_, "dx12/manifest.json"); + std::vector bytes; + wgc::Json m; + std::string jerr; + if (!wg_read_file(path, &bytes) || !wgc::parse_json(std::string(bytes.begin(), bytes.end()), m, jerr)) + return "any kernel (" + path + " missing or unreadable)"; + std::map co; // port name -> the code object it stands for + if (const auto* ks = m.find("kernels"); ks && ks->is_array()) + for (const auto& k : ks->items) { + const auto* n = k.find("name"); + const auto* c = k.find("co"); + if (n && n->is_string() && c && c->is_string()) co[n->text] = c->text; + } + std::string missing; + for (const std::string& k : d3d12_kernels()) { + auto it = co.find(k); + // the code object the port stands for must be one this runtime loads (else the device refuses it at launch) + const bool tagged = k.rfind("kx/", 0) == 0; + const bool co_ok = it != co.end() && + (tagged ? it->second == "k_io_exact.co" + : (it->second == "k_block.co" || it->second == "k_block_wide.co" || + it->second == "k_io.co" || it->second == "exposure.co" || it->second == "frame.co")); + if (co_ok) continue; + missing += (missing.empty() ? "" : ", ") + k + (it == co.end() ? "" : " (port of " + it->second + ")"); + } + return missing; +} + +bool KRuntime::direct_preflight(const WgRtFrame& fr, const FrameIo* io) { + if (!native_direct_) return fail("model K: not a native direct runtime"); + if (!io || !io->native_direct) return fail("model K: the native direct runtime needs FrameIo.native_direct"); + if (io->raw_inputs || io->out_half_pitch || io->out_rgba16f || io->motion_hi || io->motion_hi_direct || io->out_signal) + return fail("model K direct frame: no input copies, output rows, output-resolution motion buffer or staged hand-over"); + if (io->exposure_mode < -1 || io->exposure_mode > 2) return fail("model K direct frame: unsupported exposure mode"); + if (fr.reset < 0 || fr.reset > 2) return fail("model K direct frame: reset must be 0, 1 or 2"); + if (fr.rw <= 0 || fr.rh <= 0 || fr.rw > max_rw_ || fr.rh > max_rh_) + return fail("model K direct frame: render size outside the configured maximum"); + if (!std::isfinite(fr.jitter[0]) || !std::isfinite(fr.jitter[1])) return fail("model K direct frame: non-finite jitter"); + // every render size up to the maximum and every finite jitter run on D3D12: the tiled input stage where its guard + // holds, else the legacy input kernels (both have direct forms) + return true; +} + const char* KRuntime::kernels() { - kernels_text_ = std::string("model=K network=") + (f8_ ? "fused8" : f16_ ? "fused16" : "fused") + + kernels_text_ = std::string(native_direct_ ? "io=direct_textures " : "") + "model=K network=" + (f8_ ? "fused8" : f16_ ? "fused16" : "fused") + (f8_ ? " fp8_stages=" + std::to_string(f8data_.size()) + (f8why_.empty() ? "" : " fp8_kept:" + f8why_) : std::string()) + (in_exact_ ? " input=k_io_exact.co" : " input=k_io.co") + " output=k_io.co exposure=k_expo_scalars" + @@ -910,6 +1155,13 @@ bool KRuntime::buffer(const std::string& name, DevPtr* p, size_t* bytes) { {"k.Hh", {Hh_[lp], (size_t)Bw_ * Bh_ * 8}}, {"filter.out", {out_, (size_t)W_ * H_ * (omode_last_ == OUT_RGBA32F ? 16 : 12)}}, {"input.color", {color_, R * 16}}, {"input.depth", {depth_, R * 4}}, {"input.motion", {motion_, R * 8}}, + // the network's intermediate maps after the frame (D3D12 port checks, clean/k/dx12/k_compare.py): the skip maps + // S0..S4 (every encoder stage's block output), the ping-pong maps A / B (their last writers' contents) and the + // output colour taps of k_out_color / the tiled colour kernel + {"k.S0", {net_["S0"], net_bytes_["S0"]}}, {"k.S1", {net_["S1"], net_bytes_["S1"]}}, + {"k.S2", {net_["S2"], net_bytes_["S2"]}}, {"k.S3", {net_["S3"], net_bytes_["S3"]}}, + {"k.S4", {net_["S4"], net_bytes_["S4"]}}, {"k.A", {net_["A"], net_bytes_["A"]}}, + {"k.B", {net_["B"], net_bytes_["B"]}}, {"k.ctap", {ctap_, R * 8}}, {"k.ctapT", {ctapT_, R * 2}}, }; // the head map exists only with k_head=full; the default frame holds the compact pixel head in the same buffer if (name == "k.N" && compact_head_) diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 9e633b99..a3dc3fc5 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -53,9 +53,25 @@ class KRuntime { using AllocFn = std::function; // device memory owned (and freed) by the caller // After the caller made the device. external_io: inputs and output always come from FrameIo (no own output). - bool create(const WgRtConfig& cfg, Device* dev, AllocFn alloc, bool external_io); + // native_direct (backend 3, external_io; the D3D12 executor, interop/executor_d3d12.cpp): the game's colour, depth, + // motion and exposure textures and its output texture are bound by the caller (NativeD3D12Ext::bind_direct_frame): + // the kernels that read inputs or write the output run as their "_direct" D3D12 forms (see direct_names), no dense + // input buffers are allocated, and only the exact default path is taken (network=fused, exact input, tiled input). + // On backend 3 create first checks that /dx12/manifest.json has a port of every kernel the context can + // launch (d3d12_missing): a context with a missing port is refused before the model is loaded, so the D3D12 + // executor refuses it and the HIP executor runs it (executor_select.h). + bool create(const WgRtConfig& cfg, Device* dev, AllocFn alloc, bool external_io, bool native_direct = false); // Enqueues one frame (does not wait). bool frame(const WgRtFrame& fr, const FrameIo* io); + // native_direct: whether frame() would take this frame (nothing enqueued); false with err() saying why. The D3D12 + // executor calls it before it records anything ("frame not taken": the HIP executor then runs the context). A frame + // whose render size is above 0.68 x the output (DLAA-like; no tiled input stage) or whose jitter is outside + // [-0.5, 0.5] is not taken: the legacy input kernels have no direct forms. + bool direct_preflight(const WgRtFrame& fr, const FrameIo* io); + // Kernel names (as launched) a context of this configuration can launch on backend 3, and those of them the + // manifest has no port of (comma list; empty: every kernel is ported). Valid after create's option parsing. + std::vector d3d12_kernels() const; + std::string d3d12_missing() const; // Test path (wg_rt_upload_inputs / wg_rt_upload_inputs_rgba): dense colour float32 of `channels` values per pixel (3: // RGB, the alpha is 1 as for a colour without an alpha channel; 4: RGBA, the alpha as given), depth, motion (render // pixels) into the own input buffers. The output alpha follows the colour's alpha (issue #26). @@ -80,12 +96,19 @@ class KRuntime { bool launch(const char* kernel, unsigned grid, unsigned block, void* arg0, void* arg1 = nullptr, void* arg2 = nullptr, void* arg3 = nullptr, void* arg4 = nullptr, void* arg5 = nullptr, void* arg6 = nullptr, void* arg7 = nullptr); - bool network(int f); + // after(stage name): called after each stage's launch (k_color_at / k_warm launches) + bool network(int f, const std::function& after = nullptr); + bool warm(const char* at); + // the tiled input stage's guard (frame and direct_preflight): sizes, the 0.68 ratio, finite jitter in [-0.5, 0.5] + bool tiled_guard(int rw, int rh, float jx, float jy) const; + bool options_ok_direct(); + int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with WgRtConfig cfg_ = {}; Device* dev_ = nullptr; AllocFn alloc_; bool external_io_ = false; + bool native_direct_ = false; // create's native_direct: the "_direct" kernels on the caller's bound textures std::string err_, build_dir_, model_path_, map_path_, kernels_text_; int W_ = 0, H_ = 0, W16_ = 0, H16_ = 0, Bw_ = 0, Bh_ = 0, max_rw_ = 0, max_rh_ = 0; bool hdr_ = false, depth_inverted_ = false, f16_ = false; // f16_: runtime option network=fused16 (or fused8) @@ -98,6 +121,7 @@ class KRuntime { bool tiled_input_ = false, tiled_last_ = false, tiled_color_last_ = false, input_uniforms_last_ = false; uint64_t tiled_launches_ = 0, tiled_color_launches_ = 0, input_uniform_launches_ = 0; uint64_t legacy_color_launches_ = 0, legacy_base_launches_ = 0; + bool input_uniforms_ = true; // runtime option k_in_uniforms (0: always k_in_tiled, a port-check diagnostic) bool diag_input2_ = false, diag_copyin_ = false; // runtime option k_diag (timing diagnostics) // d0 hands the head to k_out_pixel_c as the compact pixel head (k_stage2*_d0_hp; default). Runtime option // k_head=full: the 40-channel head map and k_out_pixel as before (same output bits; buffer "k.N" needs it) @@ -121,6 +145,11 @@ class KRuntime { int frames_ = 0, parity_ = 0, fidx_ = 0, last_rw_ = 0, last_rh_ = 0, level_last_ = -1, omode_last_ = -1; float jprev_[2] = {0, 0}; int pending_mark_ = -1; + // D3D12 latency options (create: k_color_at, k_warm, k_warm_line, k_warm_groups): the stage after which k_out_color + // runs unordered (empty: before the input stage, HIP's order), the stage after which k_warm runs (empty: off) + std::string color_at_, warm_at_, warm_to_; + unsigned warm_line_ = 128, warm_groups_ = 64; + DevPtr warm_dummy_ = 0; WgRtFrameInfo info_ = {}; }; diff --git a/clean/k/tests/raw_compare.py b/clean/k/tests/raw_compare.py new file mode 100644 index 00000000..4fcbd853 --- /dev/null +++ b/clean/k/tests/raw_compare.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: MIT +"""Raw-byte comparison of a D3D12 result against HIP's, with a distinct sentinel per backend (model K's D3D12 tests). + +Before a frame every output and intermediate buffer is filled with a sentinel byte: 0xA5 on HIP, 0x5A on D3D12. After +it, the two buffers are compared as 4-byte words of raw bit patterns (no float conversion: an FP16 NaN keeps its +payload, -0 stays -0). A word that differs is a failure, except a word that neither side wrote (HIP's sentinel on one +side and D3D12's on the other): those are counted, and a caller that knows the buffer is written in full allows none. +A write that D3D12 dropped leaves its sentinel where HIP stored a value, and a write that only D3D12 made leaves HIP's +sentinel: both fail, which the negative controls in test_k_raw_compare.py show. +""" +import numpy as np + +HIP_SENTINEL = 0xA5 +D3D12_SENTINEL = 0x5A + + +def sentinel_word(byte): + return np.uint32(byte * 0x01010101) + + +def compare_raw(what, hip, d3d12, allow_untouched=0, hip_sentinel=HIP_SENTINEL, d3d12_sentinel=D3D12_SENTINEL): + """hip, d3d12: bytes-like of equal length (a multiple of 4). Raises AssertionError on a differing word or on more + than allow_untouched words written by neither; returns the number of such words.""" + hip, d3d12 = bytes(hip), bytes(d3d12) + if len(hip) != len(d3d12) or len(hip) % 4: + raise AssertionError(f"{what}: sizes {len(hip)} and {len(d3d12)} (need equal, a multiple of 4)") + h, d = np.frombuffer(hip, np.uint32), np.frombuffer(d3d12, np.uint32) + differ = h != d + untouched = differ & (h == sentinel_word(hip_sentinel)) & (d == sentinel_word(d3d12_sentinel)) + bad = differ & ~untouched + if bad.any(): + i = int(np.flatnonzero(bad)[0]) + raise AssertionError(f"{what}: {int(bad.sum())} of {len(h)} words differ; first at word {i}: HIP {h[i]:08x}, " + f"D3D12 {d[i]:08x}") + n = int(untouched.sum()) + if n > allow_untouched: + raise AssertionError(f"{what}: {n} of {len(h)} words written by neither side (allowed {allow_untouched})") + return n diff --git a/clean/k/tests/test_k_direct_motion.py b/clean/k/tests/test_k_direct_motion.py index 4268707a..ba96b212 100644 --- a/clean/k/tests/test_k_direct_motion.py +++ b/clean/k/tests/test_k_direct_motion.py @@ -112,7 +112,9 @@ def test_sources(self): if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): self.skipTest("the D3D12 input and output ports are not in this tree") out_i, out_k = read("dx12", "hlsl", "k_out.hlsli"), read("dx12", "kernels", "k_out.hlsl") - fr, kin = read("dx12", "hlsl", "k_frame.hlsli"), read("dx12", "kernels", "k_in.hlsl") + fr = read("dx12", "hlsl", "k_frame.hlsli") + # the input kernels' shared stage code (k_in_common.hlsli) is included by k_in.hlsl and k_in_legacy.hlsl + kin = read("dx12", "kernels", "k_in.hlsl") + read("dx12", "hlsl", "k_in_common.hlsli") # direct reads: the raw texel (no per-texel scale; not model M's nd_motion_xy) self.assertNotIn("nd_motion_xy(", re.sub(r"//[^\n]*", "", out_i)) for src in (out_i, fr): @@ -146,14 +148,16 @@ def test_full_head_direct_port_and_alpha(self): self.skipTest("the D3D12 input and output ports are not in this tree") out_k = read("dx12", "kernels", "k_out.hlsl") line = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_direct ")] - if line: # the direct forms are declared with the runtime wiring; their declarations are checked once they are + self.assertEqual(len(line), 1) # declared since the native executor runs model K on D3D12 + if line: self.assertEqual(len(line), 1) for need in ("entry=k_out_pixel_c", "rs=direct", "KO_COMPACT=0", "WG_DIRECT_TEXTURES=1"): self.assertIn(need, line[0]) c = [l for l in out_k.splitlines() if l.startswith("// @wg-kernel k_out_pixel_c_direct ")][0] self.assertEqual(c.replace("k_out_pixel_c_direct", "X").replace("KO_COMPACT=1", "C"), line[0].replace("k_out_pixel_direct", "X").replace("KO_COMPACT=0", "C")) - self.assertIn('"k_out_pixel_direct"', read("equiv", "k_stages.py")) + # the runtime names a direct form by the suffix (runtime_k.cpp kname): the head form follows k_head + self.assertIn('kname_buf += "_direct"', read("runtime", "runtime_k.cpp")) # the output alpha is cbar3 (the colour input's alpha, issue #26) in both head forms: no constant alpha self.assertNotIn("KO_OUT_ALPHA0", out_k) self.assertIn("native_direct_output[uint2((uint)x, (uint)y)] = float4(v, (float)A);", out_k) diff --git a/clean/k/tests/test_k_m16_alpha.py b/clean/k/tests/test_k_m16_alpha.py index eaf07e0f..2a47dc6c 100644 --- a/clean/k/tests/test_k_m16_alpha.py +++ b/clean/k/tests/test_k_m16_alpha.py @@ -93,7 +93,8 @@ def test_d3d12_sources(self): if not all(os.path.exists(os.path.join(ROOT, "clean", "k", "dx12", "kernels", f)) for f in ("k_in.hlsl", "k_out.hlsl")): self.skipTest("the D3D12 input and output ports are not in this tree") o, oi = read("k", "dx12", "kernels", "k_out.hlsl"), read("k", "dx12", "hlsl", "k_out.hlsli") - fr, ki = read("k", "dx12", "hlsl", "k_frame.hlsli"), read("k", "dx12", "kernels", "k_in.hlsl") + fr = read("k", "dx12", "hlsl", "k_frame.hlsli") + ki = read("k", "dx12", "kernels", "k_in.hlsl") + read("k", "dx12", "hlsl", "k_in_common.hlsli") self.assertNotIn("f_rmsx * r0", oi) self.assertNotIn("F.rmsx * wg_h2f_lo(r)", fr) self.assertIn("mvx = wg_h2f(wg_ld16(f_motion, i * 2u));", oi) diff --git a/clean/k/tests/test_k_port_check_nan.py b/clean/k/tests/test_k_port_check_nan.py new file mode 100644 index 00000000..3452b395 --- /dev/null +++ b/clean/k/tests/test_k_port_check_nan.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: MIT +"""The port check's NaN rule (clean/k/dx12/k_port_check.py nan_equivalent_differ): the only accepted difference between +the D3D12 port and HIP is the SIGN of a NaN; the exponent, the quiet bit and the payload must be equal, and every other +differing element counts as a difference. CPU only. + +Run: python -m unittest clean.k.tests.test_k_port_check_nan +""" +import unittest + +import numpy as np + +from clean.k.dx12.k_port_check import nan_equivalent_differ as differ + + +def run(got, want, dtype, width): + g, w = np.array(got, dtype).tobytes(), np.array(want, dtype).tobytes() + return differ(g, w, len(g), width) + + +class TestNanRule(unittest.TestCase): + def test_identical(self): + for v in (0x7E00, 0x1234, 0xFE00): + self.assertEqual(run([v], [v], np.uint16, 2), (0, 0)) + + def test_sign_only_fp16(self): + self.assertEqual(run([0x7E00, 0xFE01], [0xFE00, 0x7E01], np.uint16, 2), (0, 2)) + + def test_sign_only_fp32(self): + self.assertEqual(run([0x7FC00000, 0xFFC12345], [0xFFC00000, 0x7FC12345], np.uint32, 4), (0, 2)) + + def test_payload_differs_is_outside(self): + self.assertEqual(run([0x7E01], [0x7E02], np.uint16, 2), (1, 0)) + self.assertEqual(run([0x7E01], [0xFE02], np.uint16, 2), (1, 0)) # the sign and the payload differ + self.assertEqual(run([0x7FC00001], [0x7FC00002], np.uint32, 4), (1, 0)) + + def test_quiet_bit_differs_is_outside(self): + self.assertEqual(run([0x7E00], [0x7C01], np.uint16, 2), (1, 0)) # quiet and signalling NaN + self.assertEqual(run([0x7FC00000], [0x7F800001], np.uint32, 4), (1, 0)) + + def test_nan_against_number_is_outside(self): + self.assertEqual(run([0x7E00], [0x3C00], np.uint16, 2), (1, 0)) + self.assertEqual(run([0x3C00], [0x7E00], np.uint16, 2), (1, 0)) + + def test_negative_zero_and_numbers_are_outside(self): + self.assertEqual(run([0x0000], [0x8000], np.uint16, 2), (1, 0)) # +0 against -0 is not a NaN sign + self.assertEqual(run([0x1234], [0x1235], np.uint16, 2), (1, 0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_raw_compare.py b/clean/k/tests/test_k_raw_compare.py new file mode 100644 index 00000000..e0a74878 --- /dev/null +++ b/clean/k/tests/test_k_raw_compare.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: MIT +"""CPU test of the raw-byte comparison with distinct sentinels (clean/k/tests/raw_compare.py): it must pass equal data +and fail a dropped write, a write only D3D12 made, a changed bit, a NaN payload and a negative zero.""" +import unittest + +import numpy as np + +from clean.k.tests.raw_compare import D3D12_SENTINEL, HIP_SENTINEL, compare_raw, sentinel_word + + +class TestRawCompare(unittest.TestCase): + def data(self): + rng = np.random.default_rng(5) + h = rng.integers(0, 2**32, 64, dtype=np.uint64).astype(np.uint32) + h[(h == sentinel_word(HIP_SENTINEL)) | (h == sentinel_word(D3D12_SENTINEL))] = 1 + return h, h.copy() + + def test_equal(self): + h, d = self.data() + self.assertEqual(compare_raw("equal", h.tobytes(), d.tobytes()), 0) + + def test_dropped_write_fails(self): + """D3D12 left its sentinel where HIP stored a value.""" + h, d = self.data() + d[7] = sentinel_word(D3D12_SENTINEL) + with self.assertRaisesRegex(AssertionError, "differ"): + compare_raw("dropped", h.tobytes(), d.tobytes()) + + def test_write_only_on_d3d12_fails(self): + h, d = self.data() + h[9] = sentinel_word(HIP_SENTINEL) + with self.assertRaisesRegex(AssertionError, "differ"): + compare_raw("extra", h.tobytes(), d.tobytes()) + + def test_one_bit_fails(self): + h, d = self.data() + d[3] ^= np.uint32(1 << 31) + with self.assertRaises(AssertionError): + compare_raw("bit", h.tobytes(), d.tobytes()) + + def test_nan_payload_and_negative_zero_fail(self): + h, d = self.data() + h[1], d[1] = np.uint32(0x7FC00000), np.uint32(0x7FC00001) + with self.assertRaises(AssertionError): + compare_raw("nan", h.tobytes(), d.tobytes()) + h, d = self.data() + h[2], d[2] = np.uint32(0), np.uint32(0x80000000) + with self.assertRaises(AssertionError): + compare_raw("negzero", h.tobytes(), d.tobytes()) + + def test_untouched_words_counted_and_limited(self): + """A word neither side wrote (padding) is counted; a caller that knows the buffer is full allows none.""" + h, d = self.data() + h[5], d[5] = sentinel_word(HIP_SENTINEL), sentinel_word(D3D12_SENTINEL) + self.assertEqual(compare_raw("pad", h.tobytes(), d.tobytes(), allow_untouched=1), 1) + with self.assertRaisesRegex(AssertionError, "neither"): + compare_raw("pad", h.tobytes(), d.tobytes()) + + def test_size_mismatch_fails(self): + h, d = self.data() + with self.assertRaises(AssertionError): + compare_raw("size", h.tobytes(), d[:-1].tobytes()) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py new file mode 100644 index 00000000..1ca88c09 --- /dev/null +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -0,0 +1,292 @@ +# SPDX-License-Identifier: MIT +"""Model K's per-frame runtime on the D3D12 devices against the HIP backend, frame by frame, byte for byte (issue #15). + +The same frames (a synthetic model K, random FP16 parameters; clean/k/tests/test_k_runtime_gpu.py) go through two +runtimes of the same build: HIP (backend 0) and a D3D12 device, and after every frame the game output (the raw bytes of +the RGBA16F rows, or the dense RGB float32) and the K buffers must be identical, compared as raw 4-byte words +(clean/k/tests/raw_compare.py): +- the output buffer and the network's intermediates (the input stage's x0, the skip maps S0..S4, the ping-pong maps A + and B, the output colour taps ctap / ctapT, the head map or compact pixel head, exposure.pair, k.scalars) are filled + with a sentinel before every frame: 0xA5 on HIP, 0x5A on D3D12. A write that one side did not make leaves its + sentinel and fails the comparison; the output region and every intermediate must be written in full (a word neither side wrote fails); +- the first frame too: option k_fill (a test hook of the runtime) fills every buffer a frame writes and both history + copies with the backend's sentinel after allocation, so first-frame and reset-only writes are proven; +- the exposure buffers and the histories k.Hc / k.L / k.Hh (the state a frame carries to the next) are compared raw; +- a buffer that does not exist fails the test (no skipping); +- negative controls: a dropped write in the real output (its first word set back to the D3D12 sentinel) must be caught. +The devices: +- the hybrid device (backend 2: the ports of dx12/manifest.json run on D3D12, any other kernel on HIP, one arena): + every kernel the frame launches must have run on D3D12 (the launch trace), so that an unported kernel cannot hide + behind HIP running on both sides; +- the native device (backend 3, no HIP). +Sequences: a reset frame, steady frames with a pan, a game reset, a render-size change (the history is kept); HDR with a +value, with the game's exposure texel, and with auto-exposure, and LDR; output rows RGBA16F and dense RGB; render size +equal to the output (native AA, DLAA 1.0: the legacy input kernels), a jitter outside [-0.5, 0.5], and input_tiled=0. +Refusals (one line, create or frame): an approximation option on the native device, an input-copy frame (raw_inputs). + +GPU only: WG_K_GPU_BUILD=, ROCM_PATH; run each test under the +machine's GPU lock. +Run: python -m unittest clean.k.tests.test_k_runtime_d3d12 +""" +import os +import unittest + +import numpy as np + +# read once per process, at the first launch of the first runtime +TRACE = os.path.join(os.environ.get("WG_K_GPU_BUILD", "."), "k_d3d12_launch_trace.txt") +os.environ.setdefault("WG_DX12_KERNELS", "all") +os.environ.setdefault("WG_LAUNCH_TRACE", TRACE) + +from clean.k.reference.network import random_network_params # noqa: E402 +from clean.k.tests.raw_compare import D3D12_SENTINEL, HIP_SENTINEL, compare_raw, sentinel_word # noqa: E402 +from clean.k.tests.test_k_runtime_gpu import (KRuntimeExt, gpu_build, inputs, pitch256, # noqa: E402 + write_k_model) + +OUT, MAX_RENDER = (200, 120), (130, 78) +# written in full by every frame: compared raw with a sentinel before the frame (the head buffer is added per frame) +WRITTEN = ["exposure.pair", "k.scalars", "k.x0", "k.S0", "k.S1", "k.S2", "k.S3", "k.S4", "k.A", "k.B", "k.ctap", + "k.ctapT"] +# the state a frame carries to the next (the histories) and the last auto-exposure value: compared raw, not poked +STATE = ["exposure.measured", "k.Hc", "k.L", "k.Hh"] +# kernels a HDR / LDR frame of the default path launches (the D3D12 ports must run every one of them in the hybrid device) +STAGES = ["k_stage2_e0", "k_stage2_e1", "k_stage_e2", "k_stagew2_e3", "k_stagew4_e4", "k_stagew4_m", "k_stagew4_d4", + "k_stagew2_d3", "k_stage_d2", "k_stage2_d1", "k_stage2_d0_hp"] +ALWAYS = ["k_expo_scalars", "k_out_pixel_c"] +TILED = ["kx/k_in_tiled", "kx/k_in_tiled_uniform"] +LEGACY = ["kx/k_in_color", "kx/k_in_base"] # (+ kx/k_in_reset on a reset frame) + + +def trace_since(pos): + if not os.path.exists(TRACE): + return [], pos + with open(TRACE, "rb") as f: + f.seek(pos) + data = f.read() + return data.decode(errors="replace").splitlines(), pos + len(data) + + +def poke(rt, ptr, nbytes, byte): + rt.upload(ptr, np.full(nbytes, byte, np.uint8)) + + +def out_bytes(rt, half): + """The game output's defined bytes as the frame wrote them: RGBA16F rows (8 bytes a pixel, the pitch padding left + out) or dense RGB float32.""" + W, H = rt.W, rt.H + if half: + rows = rt.download(rt.bufs["out"], rt.half_pitch * H, np.uint8).reshape(H, rt.half_pitch) + return np.ascontiguousarray(rows[:, :8 * W]).tobytes() + return rt.download(rt.bufs["out"], W * H * 12, np.uint8).tobytes() + + +class TestKRuntimeD3D12(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.build = gpu_build() + if not os.path.exists(os.path.join(cls.build, "dx12", "manifest.json")): + raise unittest.SkipTest("no dx12/manifest.json in WG_K_GPU_BUILD (build_dx12.py --model-k)") + cls.params = random_network_params(11) + cls.mdir = os.path.join(cls.build, "k_runtime_d3d12_test") + cls.wgcm, cls.map = write_k_model(cls.mdir, cls.params) + + def make(self, backend, hdr, auto, options, out, max_render): + return KRuntimeExt(self.build, self.mdir, self.map, out, max_render, hdr, False, auto, options, backend=backend) + + def sequence(self, backend, hdr, mode, half, frames, options="", seed=0, out=OUT, max_render=MAX_RENDER): + """The frames [(rw, rh, reset, E, pi[, jitter])] through HIP and `backend`; raises on the first difference.""" + auto = mode == 2 + # k_fill: every buffer a frame writes and both history copies start with the backend's own sentinel (HIP 0xA5, + # D3D12 0x5A), so the first frame's and a reset frame's writes are proven too, not only the later frames' + sep = ";" if options else "" + hip = self.make(0, hdr, auto, f"{options}{sep}k_fill={HIP_SENTINEL}", out, max_render) + dx = self.make(backend, hdr, auto, f"{options}{sep}k_fill={D3D12_SENTINEL}", out, max_render) + head = "k.N" if "k_head=full" in options else "k.Np" + pos = os.path.getsize(TRACE) if os.path.exists(TRACE) else 0 + on_dx, on_hip = {}, {} + try: + for i, fr in enumerate(frames): + rw, rh, reset, E, pi = fr[:5] + x = inputs(np.random.default_rng(seed * 100 + i), rw, rh, hdr, reset, jitter=fr[5] if len(fr) > 5 else None) + x["pre_exposure"], x["E_cur"] = pi, float(np.float32(E)) + if i == 2: + x["mv_scale"], x["mv_offset"] = (0.75, 1.25), (0.125, -0.25) + # sentinels before each frame: the output buffer, and from the second frame on the intermediates again + # (the first frame's come from k_fill at create) + for rt, byte in ((hip, HIP_SENTINEL), (dx, D3D12_SENTINEL)): + poke(rt, rt.bufs["out"], rt.bufs_size["out"], byte) + if i: + for name in WRITTEN + [head]: + p, n = rt.buffer_info(name) + poke(rt, p, n, byte) + _, _ = hip.frame(x, mode, half=half) + _, pos = trace_since(pos) + _, _ = dx.frame(x, mode, half=half) + lines, pos = trace_since(pos) + for ln in lines: + parts = ln.split() + if parts: + tgt = on_dx if parts[-1] == "dx12" else on_hip + tgt[parts[0]] = tgt.get(parts[0], 0) + 1 + want, got = out_bytes(hip, half), out_bytes(dx, half) + compare_raw(f"frame {i}: output", want, got) + if i == 0: # negative control: a dropped write in the real output is caught + dropped = bytearray(got) + dropped[:4] = sentinel_word(D3D12_SENTINEL).tobytes() + with self.assertRaises(AssertionError, msg="a dropped output write was not detected"): + compare_raw("control", want, bytes(dropped)) + for name in WRITTEN + STATE + [head]: + # a buffer that does not exist raises RuntimeError here: that is a failure, not a skip + w, g = hip.buffer(name, np.uint8), dx.buffer(name, np.uint8) + # every compared buffer is written in full by a frame: a word neither side wrote fails too + compare_raw(f"frame {i}: {name}", w, g) + finally: + del hip, dx + return on_dx, on_hip + + FRAMES = [(130, 78, True, 1.0, 1.0), (130, 78, False, 1.0, 1.0), (130, 78, False, 1.25, 0.5), + (100, 60, False, 1.0, 1.0), (100, 60, True, 0.75, 1.0)] + + def check_hybrid(self, hdr, mode, half, options="", frames=None, inputs_ran=TILED, **size): + on_dx, on_hip = self.sequence(2, hdr, mode, half, frames or self.FRAMES, options, **size) + want = STAGES + ALWAYS + ["k_out_color"] + (["exposure_auto_diet"] if mode == 2 and hdr else []) + if size.get("out", OUT)[0] >= 3840: # above 24 windows e4 and d4 take their NG = 2 kernels + want = [{"k_stagew4_e4": "k_stagew2_e4", "k_stagew4_d4": "k_stagew2_d4"}.get(k, k) for k in want] + if "k_head=full" in options: # the head map and k_out_pixel instead of the compact pixel head + want = [{"k_stage2_d0_hp": "k_stage2_d0_hidden", "k_out_pixel_c": "k_out_pixel"}.get(k, k) for k in want] + never = [k for k in want if not on_dx.get(k)] + self.assertFalse(never, f"never ran on D3D12 in the hybrid device: {never}; D3D12: {sorted(on_dx)}") + # the input stage: the tiled kernel (the host's frame values by default, else the GPU's own), or the legacy kernels + self.assertTrue(any(on_dx.get(k) for k in inputs_ran), f"none of {inputs_ran} ran on D3D12: {sorted(on_dx)}") + k_on_hip = sorted(k for k in on_hip if k.startswith(("k_", "kx/"))) + self.assertFalse(k_on_hip, f"model K kernels that ran on HIP in the hybrid device: {k_on_hip}") + return on_dx + + def test_hybrid_hdr_value_rows(self): + self.check_hybrid(True, 0, True) + + def test_hybrid_hdr_texel_dense(self): + self.check_hybrid(True, 1, False) + + def test_hybrid_hdr_auto_exposure(self): + self.check_hybrid(True, 2, True) + + def test_hybrid_ldr_rows(self): + self.check_hybrid(False, 0, True) + + def test_hybrid_head_full(self): + self.check_hybrid(True, 0, True, "k_head=full") + + def test_hybrid_input_without_host_uniforms(self): + """k_in_uniforms=0: the tiled input kernel that computes its frame values on the GPU.""" + on_dx, _ = self.sequence(2, True, 0, True, self.FRAMES, "k_in_uniforms=0") + self.assertTrue(on_dx.get("kx/k_in_tiled"), sorted(on_dx)) + self.assertFalse(on_dx.get("kx/k_in_tiled_uniform")) + + # the legacy input kernels: a render size equal to the output (native AA, DLAA 1.0; above 0.68 x the output the tiled + # kernel's guard refuses), a jitter outside [-0.5, 0.5], and input_tiled=0 + DLAA = dict(out=OUT, max_render=OUT) + FRAMES_DLAA = [(200, 120, True, 1.0, 1.0), (200, 120, False, 1.0, 1.0), (200, 120, False, 1.25, 0.5), + (160, 96, False, 1.0, 1.0)] + FRAMES_JITTER = [(130, 78, True, 1.0, 1.0), (130, 78, False, 1.0, 1.0, (0.75, -0.8)), + (130, 78, False, 1.0, 1.0, (-0.9, 0.6)), (130, 78, False, 1.0, 1.0, (0.5, -0.5))] + + def test_hybrid_dlaa_ratio_one(self): + on_dx = self.check_hybrid(True, 0, True, frames=self.FRAMES_DLAA, inputs_ran=LEGACY, **self.DLAA) + self.assertTrue(on_dx.get("kx/k_in_reset"), sorted(on_dx)) + + def test_hybrid_ldr_dlaa_dense(self): + self.check_hybrid(False, 1, False, frames=self.FRAMES_DLAA, inputs_ran=LEGACY, **self.DLAA) + + def test_hybrid_jitter_outside_guard(self): + on_dx = self.check_hybrid(True, 0, True, frames=self.FRAMES_JITTER, inputs_ran=LEGACY) + self.assertTrue(on_dx.get("kx/k_in_tiled_uniform") or on_dx.get("kx/k_in_tiled"), sorted(on_dx)) + + def test_hybrid_input_not_tiled(self): + self.check_hybrid(True, 0, True, "input_tiled=0", inputs_ran=LEGACY) + + # the sizes of a game: 1440p Quality (2560 x 1440 from 1707 x 960) and 4K Performance (3840 x 2160 from 1920 x 1080, + # where e4 / d4 run their NG = 2 kernels); a reset frame, a steady frame with a pan, and a smaller render size + QUALITY = dict(out=(2560, 1440), max_render=(1707, 960)) + FRAMES_Q = [(1707, 960, True, 1.0, 1.0), (1707, 960, False, 1.0, 1.0), (1500, 840, False, 1.0, 1.0)] + + def test_hybrid_1440p_quality(self): + self.check_hybrid(True, 0, True, frames=self.FRAMES_Q, **self.QUALITY) + + def test_hybrid_4k_performance(self): + self.check_hybrid(True, 0, True, frames=[(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)], + out=(3840, 2160), max_render=(1920, 1080)) + + def test_native_1440p_quality(self): + self.sequence(3, True, 0, True, self.FRAMES_Q, **self.QUALITY) + + def test_native_hdr_value_rows(self): + self.sequence(3, True, 0, True, self.FRAMES) + + def test_native_hdr_auto_exposure_dense(self): + self.sequence(3, True, 2, False, self.FRAMES) + + def test_native_ldr_rows(self): + self.sequence(3, False, 0, True, self.FRAMES) + + def test_native_dlaa_ratio_one(self): + self.sequence(3, True, 0, True, self.FRAMES_DLAA, **self.DLAA) + + def test_native_jitter_outside_guard(self): + self.sequence(3, True, 0, True, self.FRAMES_JITTER) + + def test_native_input_not_tiled(self): + self.sequence(3, True, 0, True, self.FRAMES, "input_tiled=0") + + def test_fill_hook_reaches_both_backends(self): + """k_fill: the runtime's own input copy of the depth (never written when the caller passes its buffers) keeps + the backend's sentinel through a frame: HIP 0xA5 and D3D12 0x5A, so the first-frame check is not vacuous.""" + for backend in (2, 3): + with self.subTest(backend=backend): + hip = self.make(0, True, False, f"k_fill={HIP_SENTINEL}", OUT, MAX_RENDER) + dx = self.make(backend, True, False, f"k_fill={D3D12_SENTINEL}", OUT, MAX_RENDER) + try: + x = inputs(np.random.default_rng(3), 130, 78, True, True) + x["pre_exposure"], x["E_cur"] = 1.0, 1.0 + hip.frame(x, 0, half=True) + dx.frame(x, 0, half=True) + want, got = hip.buffer("input.depth", np.uint8), dx.buffer("input.depth", np.uint8) + self.assertTrue((want == HIP_SENTINEL).all() and (got == D3D12_SENTINEL).all()) + with self.assertRaises(AssertionError): # and the comparison tells the two apart + compare_raw("control", want, got) + finally: + del hip, dx + + # refusals: one line, naming the cause; the game keeps its own upscaler (no HIP route behind the device's back) + def test_native_refuses_approximation_options(self): + for options in ("network=fused16", "k_fast=1", "k_in_exact=0"): + with self.subTest(options=options): + with self.assertRaises(RuntimeError) as cm: + self.make(3, True, False, options, OUT, MAX_RENDER) + msg = str(cm.exception) + self.assertNotIn("\n", msg.strip()) + self.assertIn("HIP", msg) + + def test_input_copy_frames_refused_on_d3d12(self): + """raw_inputs (the interop's HIP input copies) has no checked D3D12 port: refused, not run unchecked.""" + for backend in (2, 3): + with self.subTest(backend=backend): + rt = self.make(backend, True, False, "", OUT, MAX_RENDER) + try: + rw, rh = MAX_RENDER + x = inputs(np.random.default_rng(1), rw, rh, True, True) + x["pre_exposure"], x["E_cur"] = 1.0, 1.0 + crow = np.zeros((rh, pitch256(8 * rw) // 2), np.float16) + crow[:, :4 * rw] = x["color"].astype(np.float16).reshape(rh, 4 * rw) + mrow = np.zeros((rh, pitch256(4 * rw) // 2), np.float16) + with self.assertRaises(RuntimeError) as cm: + rt.frame(x, 0, half=True, raw=(crow, mrow, (1.0, 1.0), (0.0, 0.0), 2)) + self.assertIn("raw_inputs", str(cm.exception)) + self.assertNotIn("\n", str(cm.exception).strip()) + finally: + del rt + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/k/tests/test_k_runtime_gpu.py b/clean/k/tests/test_k_runtime_gpu.py index cadb990e..7201e04c 100644 --- a/clean/k/tests/test_k_runtime_gpu.py +++ b/clean/k/tests/test_k_runtime_gpu.py @@ -131,7 +131,7 @@ class KRuntimeExt: """wg_rt_create_external with model=K and the slot buffers of the interop (FORMATS.md "Per-frame runtime").""" def __init__(self, build, model_dir, map_path, out_size, max_render, hdr, depth_inverted, auto_exposure, - options=""): + options="", backend=0): from clean.runner.build import runtime_library_name from clean.runner.pipeline import WgRtConfig, WgRtFrame self.WgRtFrame = WgRtFrame @@ -140,6 +140,7 @@ def __init__(self, build, model_dir, map_path, out_size, max_render, hdr, depth_ os.add_dll_directory(os.path.join(rocm, "bin")) lib = self.lib = ctypes.CDLL(os.path.abspath(os.path.join(build, runtime_library_name()))) lib.wg_rt_create_external.argtypes = [P(WgRtConfig), P(vp), ctypes.c_char_p, ctypes.c_int] + lib.wg_rt_create.argtypes = [P(WgRtConfig), P(vp), ctypes.c_char_p, ctypes.c_int] lib.wg_rt_destroy.argtypes = [vp] lib.wg_rt_error.argtypes = [vp] lib.wg_rt_error.restype = ctypes.c_char_p @@ -160,20 +161,23 @@ def __init__(self, build, model_dir, map_path, out_size, max_render, hdr, depth_ opts = f"model=K;model_k_map={map_path}" + (";" + options if options else "") self._keep = [os.fsencode(build), os.fsencode(rocm), os.fsencode(model_dir), opts.encode()] cfg.build_dir, cfg.rocm_dir, cfg.model_dir, cfg.runtime_options = self._keep - cfg.backend = 0 + cfg.backend = backend # 0 HIP; 2 the hybrid D3D12 device; 3 the native D3D12 device (no HIP) cfg.max_rw, cfg.max_rh, cfg.ow, cfg.oh = self.max_rw, self.max_rh, self.W, self.H cfg.depth_inverted, cfg.hdr, cfg.auto_exposure = int(depth_inverted), int(hdr), int(auto_exposure) cfg.model_format = 2 rt, err = ctypes.c_void_p(), ctypes.create_string_buffer(4096) - if lib.wg_rt_create_external(ctypes.byref(cfg), ctypes.byref(rt), err, len(err)) != 0: + # the native D3D12 device takes caller buffers (wg_rt_alloc, wg_rt_frame_io) from a runtime made with wg_rt_create + create = lib.wg_rt_create if backend == 3 else lib.wg_rt_create_external + if create(ctypes.byref(cfg), ctypes.byref(rt), err, len(err)) != 0: raise RuntimeError("runtime create failed: " + err.value.decode(errors="replace")) self.rt = rt R, O = self.max_rw * self.max_rh, self.W * self.H self.half_pitch = pitch256(8 * self.W) - self.bufs = {k: self.alloc(n) for k, n in { + self.bufs_size = { "color": max(R * 16, pitch256(8 * self.max_rw) * self.max_rh), "depth": R * 4, "motion": max(R * 8, pitch256(8 * self.max_rw) * self.max_rh), - "out": max(O * 12, self.half_pitch * self.H), "expo": 256}.items()} + "out": max(O * 12, self.half_pitch * self.H), "expo": 256} + self.bufs = {k: self.alloc(n) for k, n in self.bufs_size.items()} def check(self, rc, what): if rc != 0: @@ -193,10 +197,15 @@ def download(self, ptr, n, dtype): self.check(self.lib.wg_rt_download(self.rt, ptr, a.ctypes.data, a.nbytes), "download") return a - def buffer(self, name, dtype): + def buffer_info(self, name): + """(device pointer, bytes) of a runtime buffer (wg_rt_buffer); raises RuntimeError when it does not exist.""" p, n = up(), sz() self.check(self.lib.wg_rt_buffer(self.rt, name.encode(), ctypes.byref(p), ctypes.byref(n)), name) - return self.download(p.value, n.value // np.dtype(dtype).itemsize, dtype) + return p.value, n.value + + def buffer(self, name, dtype): + p, n = self.buffer_info(name) + return self.download(p, n // np.dtype(dtype).itemsize, dtype) def time_parts(self, x, mode, half, frames=100, warmup=300): """The inputs already in the slot buffers (frame() ran on x before): `frames` steady frames back to back with diff --git a/clean/k/tests/test_k_wide_ng.py b/clean/k/tests/test_k_wide_ng.py new file mode 100644 index 00000000..a27b5dde --- /dev/null +++ b/clean/k/tests/test_k_wide_ng.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: MIT +"""Model K: the NG = 4 forms of the wide stages e3 and d3 (k_stagew4_e3, k_stagew4_d3) cannot be reached (host checks). + +The runtime launches a wide stage as k_stagew_ with NG from wide_ng (clean/k/runtime/runtime_k.cpp), in one +place for the frame (network()) and one for the list of kernels a context can launch (d3d12_kernels()). For the FP32 +(exact) network, the only one the D3D12 devices run, wide_ng gives e3 and d3 NG = 2 whatever the window count, the +render size, the output size or any option; the FP16 networks (network=fused16 / fused8) take other kernels and +are refused on the native device. So the NG = 4 forms of e3 and d3 are not part of any context's kernel set and are not +declared in the D3D12 build. This test pins that: +- the source of wide_ng: e3 and d3 return 2 before any window-count rule, and no option or size reaches them; +- the only places that build a k_stagew name use wide_ng's result; +- the declared D3D12 kernels (clean/dx12/kernel_sets.json) hold k_stagew2_e3 and k_stagew2_d3 and no NG = 4 form of e3 or + d3, and the Python mirror of the rule (clean/k/runner/k_launch.py wide_ng) agrees for every window count. +If a later change makes the NG = 4 form of e3 or d3 reachable, this test fails until the form is ported and checked. + +Run: python -m unittest clean.k.tests.test_k_wide_ng +""" +import json +import os +import re +import unittest + +from clean.k.runner import k_launch as KL + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +RT = os.path.join(ROOT, "clean", "k", "runtime", "runtime_k.cpp") + + +def source(): + with open(RT, encoding="utf-8") as f: + return f.read() + + +def wide_ng_body(src): + m = re.search(r"int wide_ng\(const std::string& s, bool f16, int nwin\) \{\n(.*?)\n\}\n", src, re.S) + assert m, "wide_ng not found in runtime_k.cpp" + return m.group(1) + + +class TestWideNg(unittest.TestCase): + def test_wide_ng_source(self): + body = wide_ng_body(source()).splitlines() + body = [ln.strip() for ln in body] + # the FP16 networks first: e4, m, d4 only (e3 and d3 take k_stageh_*, no wide kernel) + self.assertEqual(body[0], 'if (f16) return s == "e4" || s == "m" || s == "d4" ? 4 : 0;') + # FP32: only e4 and d4 look at the window count; e3 and d3 are 2 without one + self.assertIn('if (s == "e3" || s == "d3") return 2;', body) + self.assertEqual([ln for ln in body if "nwin" in ln and "int wide_ng" not in ln], + ['if (s == "e4" || s == "d4") return nwin <= 24 ? 4 : 2;']) + self.assertEqual(body[-1], "return 0;") + self.assertEqual(len(body), 5) + + def test_wide_names_come_from_wide_ng(self): + src = source() + # the two sites that build a wide kernel name (the frame and the kernel list) use wide_ng's result only + sites = re.findall(r'"k_stagew" \+ std::to_string\((\w+)\)', src) + self.assertEqual(sorted(sites), ["ng", "ng"]) + calls = re.findall(r"const int ng = wide_ng\(([^;]*)\);", src) + self.assertEqual(len(calls), 2, calls) + self.assertIn("st.name, f16_, p.nwx * p.nwy", calls) + self.assertIn("n, false, k_window_count(lw, lh, o, o)", calls) + self.assertNotIn("k_stagew4_e3", src) + self.assertNotIn("k_stagew4_d3", src) + + def test_python_mirror(self): + for stage in ("e3", "d3"): + self.assertEqual(KL.WIDE_DEFAULT[stage], 2) + for nwin in range(0, 2000): + self.assertEqual(KL.wide_ng(stage, KL.WIDE_DEFAULT[stage], nwin), 2, (stage, nwin)) + self.assertNotIn("e3", KL.WIDE4_SMALL) + self.assertNotIn("d3", KL.WIDE4_SMALL) + + def test_declared_kernel_set(self): + with open(os.path.join(ROOT, "clean", "dx12", "kernel_sets.json"), encoding="utf-8") as f: + names = {n for n, _ in json.load(f)["model_k"]} + self.assertTrue({"k_stagew2_e3", "k_stagew2_d3"} <= names) + self.assertFalse({"k_stagew4_e3", "k_stagew4_d3"} & names) + # the NG = 4 forms the runtime does reach are declared: e4 and d4 at up to 24 windows, and m + self.assertTrue({"k_stagew4_e4", "k_stagew4_d4", "k_stagew4_m"} <= names) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runtime/interop/backend.cpp b/clean/runtime/interop/backend.cpp index 1323abfc..31486b09 100644 --- a/clean/runtime/interop/backend.cpp +++ b/clean/runtime/interop/backend.cpp @@ -616,10 +616,16 @@ class Context { if (ch.executor.empty()) return fail(WG_ERROR_PARAMETER, "unknown executor " + o.executor); if (!ch.why.empty()) say(ch.warn ? WG_LOG_WARN : WG_LOG_INFO, ch.why); o.executor = ch.executor; // model K's default mode below applies with the HIP executor + if (ch.sparkle_off) o.sparkle_guard = 0.0f; // model K: the guard is HIP only; K stays on D3D12 dx12_default_ = ch.by_default && ch.executor == "dx12"; const RefusedFramePolicy pol = refused_frame_policy(o.hip_switch); // clean/fe-wrapped-list hip_switch_ = pol.switch_to_hip; if (!pol.note.empty()) say(WG_LOG_WARN, pol.note); + if (q.model_k) { // model K never moves to HIP, whatever hip_switch says + hip_switch_ = false; + const std::string kn = k_hip_switch_note(o.hip_switch); + if (!kn.empty()) say(WG_LOG_INFO, kn); + } } k_default_ = model_k_default(o); const bool split = services_ && services_->struct_size >= sizeof(WgFrontEndServices) && services_->split_command_list; @@ -662,6 +668,8 @@ class Context { no_slots_ = true; // its kernels read the game's textures: no slot buffers (fences and a 128 KB flag arena only) if (!exec_->prepare(&err) || !exec_->prepare_device(device_, &err) || !make_shared_objects(false, "arena", &err) || !exec_->init(device_, set_, &err)) { + // model K: no HIP route in a game; the one error line below is the whole report, the game keeps its own upscaler + if (model_k_) return fail(WG_ERROR_UNSUPPORTED, dx12_refused_line_k(dx12_default_, err)); say(WG_LOG_WARN, dx12_refused_line(dx12_default_, err)); exec_.reset(); release_set(); @@ -1129,17 +1137,23 @@ class Context { phase(5); // a frame the executor did not take (nothing of it recorded, e.g. native AA): with the default executor the HIP // executor takes over the context; with executor=dx12 the frame gets the bilinear fallback below - const bool refused = !ok && err.rfind("frame not taken", 0) == 0; - if (refused && dx12_default_ && !hip_switch_) { // clean/fe-wrapped-list: no move to HIP; nothing recorded + const bool refused = !ok && frame_refused(err); + // a refused frame (executor_select.h refused_frame_action): model K, and model M with hip_switch=off, answer + // WG_ERROR_UNSUPPORTED and the front end takes the frame (AMD FSR, else its stand-in); the context stays usable + const RefusedAction refused_action = refused_frame_action(model_k_, dx12_default_, hip_switch_); + if (refused && refused_action == RefusedAction::kNoSwitch) { // clean/fe-wrapped-list: no move to HIP; nothing recorded if (!no_switch_said_) { no_switch_said_ = true; - say(WG_LOG_WARN, no_switch_line(v, err)); + say(WG_LOG_WARN, model_k_ ? k_refusal_line(v, err) : no_switch_line(v, err)); } return WG_ERROR_UNSUPPORTED; // one frame's answer: the context stays usable (failed_ not set) } - if (refused && dx12_default_ && switch_to_hip(v, err)) return *moved = true, WG_OK; + if (refused && refused_action == RefusedAction::kSwitchToHip && switch_to_hip(v, err)) return *moved = true, WG_OK; if (!refused) dx12_last_ = v; note_path(&input_path_, "input: read directly by the network's D3D12 kernels (executor dx12, no input pass)"); + if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && model_k_) + note_path(&motion_path_, "motion: output resolution, the texel under each render pixel's centre picked by the " + "executor's D3D12 pass (as the input pass) and read by model K's D3D12 kernels"); if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && !model_k_) note_path(&motion_path_, motion_hi_use_ ? (Converter::motion_fp16(f) @@ -1474,6 +1488,10 @@ class Context { // render-size motion (point); without the create flag the runtime gets no such option and nothing changes if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && !model_k_) n_rtopts_ += (n_rtopts_.empty() ? "" : ";") + std::string("motion_outres=") + o.motion_outres; + // model K on the D3D12 executor: the executor's D3D12 pass picks the motion texel under each render pixel's centre + // as the input pass does (executor_d3d12.cpp motion_outres=point); model K's runtime ignores the key + if ((flags_ & WG_CREATE_MOTION_OUTPUT_RES) && model_k_ && o.executor == "dx12") + n_rtopts_ += (n_rtopts_.empty() ? "" : ";") + std::string("motion_outres=point"); // Pre-exposure: the runtime's default, pre_exposure=scale (color / pi into the network, result * pi). The rule of // spec 10 draft 4 (pre_exposure=original) stays an option: with auto-exposure and a pre-exposure far from 1 it // leaves the network input scaled by pi (Hogwarts Legacy, pi = 0.00077: twice the frame-to-frame change on diff --git a/clean/runtime/interop/executor_d3d12.cpp b/clean/runtime/interop/executor_d3d12.cpp index fdc90515..e28d8a41 100644 --- a/clean/runtime/interop/executor_d3d12.cpp +++ b/clean/runtime/interop/executor_d3d12.cpp @@ -21,6 +21,16 @@ // motionhi pass does (FrameIo.motion_hi_direct; the texture's size and FP16-ness in b1 word 15); point = a D3D12 pass // (Converter::record_motion_point, the input pass's own texel choice) writes the texel under each render pixel's centre // into an RG32F texture of ours, which the network then reads as its render-size motion texture. +// Model K (runtime option model=K; clean/k/runtime/runtime_k.h, native_direct): the same frame recording with K's +// "_direct" kernels. The bytes reproduced are those the HIP executor feeds K in game: the input pass's dense f32 +// colour (the texel's rgb and, for the output stage's colour taps, its alpha (the output alpha follows it: issue #26), +// NOT divided by the pre-exposure: K applies kappa = e1 E_cur / pi itself), f32 depth and f32 motion in render pixels, +// the exposure texel; K's output stage writes the output texture with the bytes of the HIP path (RGBA16F rows + copy, +// or the output pass's typed store). Motion at output resolution: the point route above (the backend names +// motion_outres=point for model K: the input pass's own texel choice). The runtime refuses at create a K context whose +// kernels lack a D3D12 port or whose options leave the exact default path, and per frame ("frame not taken") a render +// size above 0.68 x the output or a jitter outside [-0.5, 0.5] (only the tiled input stage has direct forms). Timing +// marks: exposure, input, network, output (KRuntime::kTimingLabels). #include #include #include @@ -377,9 +387,18 @@ class D3d12Executor : public Executor { return true; } std::string timing_info() const override { + if (rt_ && std::string(wgrt::runtime_model(rt_)) == "K") + return "D3D12 timestamps in the game's command list (model K): [0] start of the first kernel (exposure_auto_diet " + "if auto-exposure, else k_expo_scalars) -> [1] start of k_out_color (input stage) -> [2] start of the " + "first network stage (e0) -> [3] start of k_out_pixel_c (output stage) -> [4] after the last kernel"; return "D3D12 timestamps in the game's command list: [0] start of the network -> [1] start of the first Swin " "kernel -> [2] start of dec0 -> [3] start of post -> [4] after the last kernel"; } + std::vector timing_labels() const override { + if (!rt_) return Executor::timing_labels(); + const char* const* l = wgrt::runtime_timing_labels(rt_); // model K: exposure, input, network, output + return {l[0], l[1], l[2], l[3]}; + } bool timing(uint64_t value, NetTiming* t) override { if (!timing_ || tagged_[value % kRing] != value) return false; wgrt::Device* d = wgrt::runtime_device(rt_); diff --git a/clean/runtime/interop/executor_select.h b/clean/runtime/interop/executor_select.h index 616ad06f..5dc379bd 100644 --- a/clean/runtime/interop/executor_select.h +++ b/clean/runtime/interop/executor_select.h @@ -3,18 +3,28 @@ // tests/test_executor_select.cpp checks it on any host. WrongGPU's own code. // // Option `executor` (wronggpu.ini runtime_options, an [fsr] line, WG_RUNTIME_OPTIONS): -// - not named (or `auto`): Model M on an RDNA 4 adapter (../rdna4.h) runs on the D3D12 executor (executor_d3d12.cpp), -// everything else on the HIP executor, with one log line saying why (other GPU, model K, the sparkle guard or the -// frame capture, which run on HIP only). The D3D12 executor can still refuse the context at create (its -// driver-channel probe, the network's create, a context its kernels do not take): then the HIP executor runs it, -// again with one line (backend.cpp). A frame it does not take later (e.g. above the checked render size; native AA +// - not named (or `auto`): Model M and model K on an RDNA 4 adapter (../rdna4.h) run on the D3D12 executor +// (executor_d3d12.cpp), everything else on the HIP executor, with one log line saying why (other GPU, the sparkle +// guard or the frame capture, which run on HIP only). The D3D12 executor can still refuse the context at create +// (its driver-channel probe, the network's create, a context its kernels do not take): then the HIP executor runs +// it, again with one line (backend.cpp; model K: no HIP route, the context is not created and the game keeps its own +// upscaler, dx12_refused_line_k). Model K is taken only when every kernel the context can launch has a +// D3D12 port in the build's dx12/manifest.json and its options select the exact default path (KRuntime::create on +// backend 3 checks both before it loads the model; clean/k/runtime/runtime_k.h); else the refusal line names what +// is missing. A frame it does not take later (e.g. above the checked render size; native AA // and every ratio between 1.0 and 1.5 run on it since Brief 15) goes back to the front end // (AMD FSR, else its stand-in; hip_switch=off, the default since the owner's decision of 2026-11-04); with // hip_switch=on the context moves to the HIP executor for good (backend.cpp switch_to_hip; one line). -// - `hip`: the HIP executor, always. +// - `hip`: the HIP executor, always: a diagnostic, not the in-game runtime (owner 2026-11-14: every model runs on the +// D3D12 runtime in a game); one warning line says so. // - `dx12` (or `d3d12`): the D3D12 executor on any adapter (the device refuses unsupported ones and the HIP executor -// takes the context); model K, the sparkle guard and the frame capture keep the HIP executor (warnings, as before); -// frames it does not take get the bilinear fallback (no switch). +// takes the context; model K as above); the sparkle guard and the frame capture keep the HIP executor (warnings, as +// before) except for model K; frames it does not take get the bilinear fallback (no switch). +// - Model K (REVIEW-215b, oversight 2026-11-14): never moved to HIP by the sparkle guard (HIP-only: it is turned off +// for K, one line, K runs on D3D12), never by hip_switch (ignored for K, one line if set), never by a refused frame +// (the frame goes to the front end's own upscaler, AMD FSR, else its stand-in, the same path as model M's refused +// frames; a create-level refusal ends the context with one line and the game keeps its own upscaler). The frame capture and executor=hip stay opt-in diagnostics with a loud +// line. Auto on an adapter that is not an RDNA 4 Radeon keeps its rule for every model, logged (owner question). // - `d3d12-test`: the offline test executor. #pragma once @@ -27,7 +37,7 @@ namespace wgi { struct ExecutorRequest { std::string executor = "auto"; // the option's value ("auto" when not named) - bool model_k = false; // the runtime options select model K + bool model_k = false; // the runtime options select model K (D3D12 executor: ports checked at its create) bool adapter_known = false; // the game device's adapter could be read (DXGI) uint32_t vendor_id = 0; bool software = false; // DXGI_ADAPTER_FLAG_SOFTWARE @@ -41,25 +51,49 @@ struct ExecutorChoice { bool by_default = false; // chosen by the default rule (option not named): frames the D3D12 executor does not take // move the context to the HIP executor bool warn = false; // log `why` as a warning (else information) + bool sparkle_off = false; // model K: the sparkle guard (HIP only) is turned off; the context stays on D3D12 std::string why; // the one log line about the choice; empty: nothing to log here }; +// The one line when model K meets the sparkle guard: the guard is HIP-only, so it is off for K. +inline std::string k_sparkle_off_line() { + return "model K: the sparkle guard runs on HIP only and is turned off for model K; model K runs on the D3D12 executor"; +} + inline ExecutorChoice choose_executor(const ExecutorRequest& r) { ExecutorChoice c; const std::string& e = r.executor; - if (e == "hip" || e == "d3d12-test") { + if (e == "d3d12-test") { + c.executor = e; + return c; + } + if (e == "hip") { // a diagnostic for every model: loud, never the default c.executor = e; + c.warn = true; + c.why = "diagnostic: executor=hip selects the HIP executor; it is not the in-game runtime (the default and " + "executor=dx12 run on the D3D12 executor)"; return c; } if (e == "dx12" || e == "d3d12") { // forced (the messages of the option before it had a default) - const char* why = r.model_k ? nullptr - : r.sparkle_guard ? "the sparkle guard runs on HIP only" - : r.capture ? "the frame capture runs on HIP only" - : nullptr; - c.executor = r.model_k || why ? "hip" : "dx12"; + if (r.model_k && r.capture) { + c.executor = "hip"; + c.warn = true; + c.why = "diagnostic: frame capture forces the HIP executor (model K runs on the D3D12 executor without it)"; + return c; + } + if (r.model_k && r.sparkle_guard) { + c.executor = "dx12"; + c.sparkle_off = true; + c.warn = true; + c.why = k_sparkle_off_line(); + return c; + } + const char* why = r.sparkle_guard ? "the sparkle guard runs on HIP only" + : r.capture ? "the frame capture runs on HIP only" + : nullptr; + c.executor = why ? "hip" : "dx12"; c.warn = c.executor == "hip"; - if (r.model_k) c.why = "executor=dx12: model K has no D3D12 port; the HIP executor runs it"; - else if (why) c.why = std::string("executor=dx12 not available for this context: ") + why + "; the HIP executor runs it"; + if (why) c.why = std::string("executor=dx12 not available for this context: ") + why + "; the HIP executor runs it"; return c; } if (e != "auto" && !e.empty()) return c; // unknown value @@ -68,11 +102,20 @@ inline ExecutorChoice choose_executor(const ExecutorRequest& r) { : std::string("the adapter (not identified)"); const char* why = nullptr; std::string text; - if (r.model_k) { - why = "model K has no D3D12 port"; - } else if (!r.adapter_known || !wgrt::rdna4_adapter(r.vendor_id, r.software, r.adapter)) { + if (!r.adapter_known || !wgrt::rdna4_adapter(r.vendor_id, r.software, r.adapter)) { text = gpu + " is not an RDNA 4 Radeon RX 9070 / RX 9060, where the D3D12 executor runs"; why = text.c_str(); + } else if (r.model_k && r.capture) { // an opt-in diagnostic, never on by default + c.executor = "hip"; + c.warn = true; + c.why = "diagnostic: frame capture forces the HIP executor (model K runs on the D3D12 executor without it)"; + return c; + } else if (r.model_k && r.sparkle_guard) { // the guard never moves model K to HIP: it is turned off + c.executor = "dx12"; + c.sparkle_off = true; + c.warn = true; + c.why = k_sparkle_off_line(); + return c; } else if (r.sparkle_guard) { why = "the sparkle guard runs on HIP only"; } else if (r.capture) { @@ -115,6 +158,37 @@ inline RefusedFramePolicy refused_frame_policy(const std::string& option) { return p; } +// Model K ignores hip_switch (it never runs on HIP in a game): the option's line, empty when the option is not set. +inline std::string k_hip_switch_note(const std::string& option) { + if (option.empty()) return ""; + return "model K ignores hip_switch=" + option + + ": a frame the D3D12 executor does not take goes to the front end's own upscaler, never to HIP"; +} + +// What a context does with a frame the D3D12 executor did not take (nothing of it recorded). +// kNoSwitch the frame is answered WG_ERROR_UNSUPPORTED, the context stays usable: the front end takes it from there +// (AMD's FSR when it exists, else its stand-in; the same path for every model); +// kSwitchToHip the default executor with hip_switch=on moves the context to HIP (never for model K); +// kBilinear executor=dx12 (forced): the frame gets the bilinear fallback (never for model K). +// Model K has no state of its own here: every refused K frame is answered WG_ERROR_UNSUPPORTED, exactly as model M's +// refused frames are with hip_switch=off, and the front end does the rest (REVIEW-215c, oversight 2026-11-15). +enum class RefusedAction { kNoSwitch, kSwitchToHip, kBilinear }; + +inline RefusedAction refused_frame_action(bool model_k, bool by_default, bool hip_switch) { + if (model_k) return RefusedAction::kNoSwitch; + if (by_default) return hip_switch ? RefusedAction::kSwitchToHip : RefusedAction::kNoSwitch; + return RefusedAction::kBilinear; +} + +// A frame the executor did not take (nothing recorded) carries this prefix in its error (executor_d3d12.cpp, convert.cpp: +// no output texture, a render size outside the configured maximum, an output texture without unordered access, ...). +inline bool frame_refused(const std::string& err) { return err.rfind("frame not taken", 0) == 0; } + +inline std::string k_refusal_line(uint64_t frame, const std::string& why) { + return "frame " + std::to_string(frame) + ": model K: the D3D12 executor does not take this frame (" + why + + "); the frame goes to the front end's own upscaler (AMD FSR, else its stand-in), never to the HIP executor"; +} + // The one warning line of a context with hip_switch=off whose D3D12 executor does not take a frame. inline std::string no_switch_line(uint64_t frame, const std::string& why) { return "frame " + std::to_string(frame) + ": executor dx12 (default on RDNA 4) does not take this frame (" + why + @@ -125,6 +199,14 @@ inline std::string no_switch_line(uint64_t frame, const std::string& why) { // After choose_executor gave dx12: the D3D12 executor did not take the context at create (`err`: the driver-channel // probe, the network's create on the D3D12 device, a context its kernels do not take). The HIP executor runs it; this // is the one warning line about it. +// Model K has no HIP route in a game (owner 2026-11-14: models run on the D3D12 runtime, HIP is a test reference and an +// opt-in diagnostic): the context is not created, the one line says why, and the game keeps its own upscaler. +inline std::string dx12_refused_line_k(bool by_default, const std::string& err) { + return std::string("model K: ") + (by_default ? "executor dx12 (default on RDNA 4)" : "executor=dx12") + + " could not take this context (" + err + "); model K has no HIP route in a game (executor=hip selects it as a " + "diagnostic): the game keeps its own upscaler"; +} + inline std::string dx12_refused_line(bool by_default, const std::string& err) { return std::string(by_default ? "executor dx12 (default on RDNA 4)" : "executor=dx12") + " could not take this context (" + err + "); the HIP executor runs it"; diff --git a/clean/runtime/interop/tests/test_executor_select.cpp b/clean/runtime/interop/tests/test_executor_select.cpp index 002c5e5d..2f5a9c0f 100644 --- a/clean/runtime/interop/tests/test_executor_select.cpp +++ b/clean/runtime/interop/tests/test_executor_select.cpp @@ -73,13 +73,27 @@ int main() { c = choose_executor(r); expect("default: unknown adapter -> hip", c.executor == "hip" && has(c.why, "not identified")); } - // default with what has no D3D12 port -> hip, one line with the reason + // default with model K on RDNA 4 -> dx12 by the default rule (the D3D12 executor's create checks K's ports and + // refuses the context when one is missing: dx12_refused_line, then the HIP executor); on other GPUs -> hip { ExecutorRequest r = gpu("AMD Radeon RX 9070 XT"); r.model_k = true; ExecutorChoice c = choose_executor(r); - expect("default: model K -> hip", c.executor == "hip" && has(c.why, "model K")); + expect("default: model K on RDNA 4 -> dx12 (by default)", c.executor == "dx12" && c.by_default && c.why.empty()); + r = gpu("AMD Radeon RX 7900 XTX"); + r.model_k = true; + c = choose_executor(r); + expect("default: model K on RDNA 3 -> hip", c.executor == "hip" && has(c.why, "not an RDNA 4")); r = gpu("AMD Radeon RX 9070 XT"); + r.model_k = true; + r.capture = true; + c = choose_executor(r); + expect("default: model K + frame capture -> hip", c.executor == "hip" && has(c.why, "frame capture")); + } + // default with what has no D3D12 port -> hip, one line with the reason + { + ExecutorRequest r = gpu("AMD Radeon RX 9070 XT"); + ExecutorChoice c; r.sparkle_guard = true; c = choose_executor(r); expect("default: sparkle guard -> hip", c.executor == "hip" && has(c.why, "sparkle guard")); @@ -93,7 +107,12 @@ int main() { ExecutorRequest r = gpu("AMD Radeon RX 9070 XT"); r.executor = "hip"; ExecutorChoice c = choose_executor(r); - expect("executor=hip on RDNA 4 -> hip", c.executor == "hip" && !c.by_default && c.why.empty()); + expect("executor=hip on RDNA 4 -> hip, as a loud diagnostic", + c.executor == "hip" && !c.by_default && c.warn && has(c.why, "diagnostic: executor=hip")); + r.model_k = true; + c = choose_executor(r); + expect("executor=hip + model K -> hip, the same diagnostic line (never a game route)", + c.executor == "hip" && !c.by_default && c.warn && has(c.why, "not the in-game runtime")); } // forced: executor=dx12 (or d3d12) -> dx12 on any adapter (the device refuses unsupported ones), not by default { @@ -110,8 +129,17 @@ int main() { r.executor = "dx12"; r.model_k = true; c = choose_executor(r); - expect("executor=dx12 + model K -> hip with the earlier warning", - c.executor == "hip" && c.warn && c.why == "executor=dx12: model K has no D3D12 port; the HIP executor runs it"); + expect("executor=dx12 + model K -> dx12 (forced; ports checked at the executor's create)", + c.executor == "dx12" && !c.by_default && c.why.empty()); + { + const std::string e = "D3D12 runtime: model K: the D3D12 build has no port of k_out_color_direct"; + expect("model K port missing (default): the refusal line names it", + has(wgi::dx12_refused_line_k(true, e), "no port of k_out_color_direct") && + has(wgi::dx12_refused_line_k(true, e), "model K has no HIP route") && + has(wgi::dx12_refused_line_k(true, e), "the game keeps its own upscaler") && + !has(wgi::dx12_refused_line_k(true, e), "the HIP executor runs it") && + wgi::dx12_refused_line_k(true, e).find('\n') == std::string::npos); + } r.model_k = false; r.sparkle_guard = true; c = choose_executor(r); @@ -150,6 +178,91 @@ int main() { has(l, "frame 2: executor dx12 (default on RDNA 4) does not take this frame") && has(l, "native direct frame") && has(l, "does not move to the HIP executor") && has(l, "AMD FSR, else its stand-in")); } + // REVIEW-215b P1-a: model K's executor selection + { + using wgi::RefusedAction; + ExecutorRequest r = gpu("AMD Radeon RX 9070 XT"); + r.model_k = true; + ExecutorChoice c = choose_executor(r); + expect("K auto on RDNA 4 -> dx12 (by default), nothing to log", c.executor == "dx12" && c.by_default && c.why.empty()); + r.sparkle_guard = true; + c = choose_executor(r); + expect("K auto + sparkle guard -> dx12 with the guard turned off, one line", + c.executor == "dx12" && c.by_default && c.sparkle_off && c.warn && has(c.why, "turned off for model K") && + has(c.why, "runs on the D3D12 executor")); + r.executor = "dx12"; + c = choose_executor(r); + expect("K executor=dx12 + sparkle guard -> dx12, guard off", c.executor == "dx12" && c.sparkle_off && c.warn); + r = gpu("AMD Radeon RX 9070 XT"); + r.model_k = true; + r.sparkle_guard = false; + r.capture = true; + c = choose_executor(r); + expect("K auto + capture -> hip with the loud diagnostic line", + c.executor == "hip" && c.warn && c.why == "diagnostic: frame capture forces the HIP executor (model K runs on the " + "D3D12 executor without it)"); + r.executor = "dx12"; + c = choose_executor(r); + expect("K executor=dx12 + capture -> hip, the same diagnostic line", + c.executor == "hip" && c.warn && has(c.why, "diagnostic: frame capture forces the HIP executor")); + r.capture = false; + r.executor = "auto"; + r.sparkle_guard = true; + r.model_k = false; + c = choose_executor(r); + expect("model M + sparkle guard -> hip as before (the guard rule is K's exception only)", + c.executor == "hip" && !c.sparkle_off && has(c.why, "sparkle guard")); + r = gpu("AMD Radeon RX 7900 XTX"); + r.model_k = true; + c = choose_executor(r); + expect("K auto on a non-RDNA 4 adapter: today's rule, logged (owner question)", + c.executor == "hip" && has(c.why, "not an RDNA 4")); + r = wgi::ExecutorRequest(); + r.model_k = true; + c = choose_executor(r); + expect("K auto on an unidentified adapter: today's rule, logged", c.executor == "hip" && has(c.why, "not identified")); + + // hip_switch: ignored for K, one line when it is set + expect("K: no line when hip_switch is not set", wgi::k_hip_switch_note("").empty()); + expect("K: hip_switch=on is ignored with one line", + has(wgi::k_hip_switch_note("on"), "model K ignores hip_switch=on") && + has(wgi::k_hip_switch_note("on"), "never to HIP")); + + // REVIEW-215b P1-b: a frame the D3D12 executor did not take. The three paths of the review and the messages the + // executor gives them: no output texture, a render size outside the configured maximum, an unsupported frame (an + // output texture without unordered access); each with hip_switch off and on, by default and forced. + const char* const refusals[] = { + "frame not taken (nothing recorded): no output texture", + "frame not taken (nothing recorded): model K direct frame: render size outside the configured maximum", + "frame not taken: output texture has no unordered access", + }; + for (const char* why : refusals) { + const std::string w = why; + expect(("refused frame recognised: " + w).c_str(), wgi::frame_refused(w)); + for (bool by_default : {true, false}) { + for (bool hip_switch : {false, true}) { + const std::string tag = w.substr(0, 40) + (by_default ? " default" : " forced") + (hip_switch ? " switch=on" : " switch=off"); + // no counter, no state: every refused K frame is answered the same way (WG_ERROR_UNSUPPORTED), whatever came before + expect(("K refused frame -> WG_ERROR_UNSUPPORTED, the front end's upscaler: " + tag).c_str(), + wgi::refused_frame_action(true, by_default, hip_switch) == RefusedAction::kNoSwitch); + } + } + } + expect("a frame that is not a refusal is not a 'frame not taken'", + !wgi::frame_refused("input refused: texture format") && !wgi::frame_refused("model K: kernel failed")); + // the other models keep their rules + expect("M default + hip_switch=on -> switch to HIP (the 0.6.0 option)", + wgi::refused_frame_action(false, true, true) == RefusedAction::kSwitchToHip); + expect("M default + hip_switch=off -> front end's upscaler", + wgi::refused_frame_action(false, true, false) == RefusedAction::kNoSwitch); + expect("M executor=dx12 -> bilinear fallback", wgi::refused_frame_action(false, false, true) == RefusedAction::kBilinear); + const std::string rl = wgi::k_refusal_line(7, "frame not taken (nothing recorded): no output texture"); + expect("the refusal line names the frame, the reason, the front end's upscaler and 'never to the HIP executor'", + has(rl, "frame 7: model K") && has(rl, "no output texture") && has(rl, "AMD FSR, else its stand-in") && + has(rl, "never to the HIP executor") && rl.find('\n') == std::string::npos); + expect("K and M (hip_switch=off) take the same path: the same action", + wgi::refused_frame_action(true, true, true) == wgi::refused_frame_action(false, true, false)); + } // test executor and unknown values { ExecutorRequest r = gpu("AMD Radeon RX 9070 XT"); diff --git a/clean/runtime/model_select.h b/clean/runtime/model_select.h index b789516d..91c71c97 100644 --- a/clean/runtime/model_select.h +++ b/clean/runtime/model_select.h @@ -26,7 +26,7 @@ struct ModelInfo { constexpr ModelInfo kModelTable[] = { {ModelId::M, "M", "preset-M.v1.map.json", "", true}, - {ModelId::K, "K", "preset-K.v1.map.json", "K", false}, + {ModelId::K, "K", "preset-K.v1.map.json", "K", true}, {ModelId::L, "L", "preset-L.v1.map.json", "L", false}, }; constexpr size_t kModelCount = sizeof kModelTable / sizeof kModelTable[0]; diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 572718e4..87b4d2d0 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -884,12 +884,18 @@ struct Runtime { dev = make_emu_device(c.emu_library ? c.emu_library : build_dir + "/wg_emu.so", &e); } else if (c.backend == 2) { dev = make_dx12_device(c.rocm_dir ? c.rocm_dir : "", build_dir, &e); + } else if (c.backend == 3) { +#if defined(_WIN32) + dev = make_d3d12_device(build_dir, &e); +#else + return fail("native D3D12 requires Windows"); +#endif } else { dev = make_hip_device(c.rocm_dir ? c.rocm_dir : "", &e); } if (!dev) return fail(e); k.reset(new KRuntime()); - if (!k->create(c, dev.get(), [this](DevPtr* p, size_t n) { return alloc(p, n); }, external_io)) + if (!k->create(c, dev.get(), [this](DevPtr* p, size_t n) { return alloc(p, n); }, external_io, native_direct_io)) return fail(k->err()); blobs = k->blobs(); model_source = "model.wgcm"; @@ -1263,6 +1269,9 @@ struct Runtime { bool direct_preflight(const WgRtFrame& fr, const FrameIo* io) { if ((io && io->native_direct) != native_direct_io) return fail("FrameIo.native_direct does not match the runtime's I/O mode"); +#ifndef WG_NO_MODEL_K + if (k) return !native_direct_io || k->direct_preflight(fr, io) || fail(k->err()); // model K: its own rules +#endif return !native_direct_io || validate_native_direct_frame(fr, io); } diff --git a/clean/tests/test_model_select.cpp b/clean/tests/test_model_select.cpp index e92dbf88..1227f0c2 100644 --- a/clean/tests/test_model_select.cpp +++ b/clean/tests/test_model_select.cpp @@ -48,10 +48,10 @@ int main() { EXPECT(std::string(m.name) == "M" && std::string(m.map_file) == "preset-M.v1.map.json" && std::string(m.model_subdir).empty() && m.native_d3d12); EXPECT(std::string(k.name) == "K" && std::string(k.map_file) == "preset-K.v1.map.json" && - std::string(k.model_subdir) == "K" && !k.native_d3d12); + std::string(k.model_subdir) == "K" && k.native_d3d12); EXPECT(std::string(l.name) == "L" && std::string(l.map_file) == "preset-L.v1.map.json" && std::string(l.model_subdir) == "L" && !l.native_d3d12); - EXPECT(wgrt::native_d3d12_models() == "Model M"); // "native D3D12 supports Model M only", as before + EXPECT(wgrt::native_d3d12_models() == "Models M, K"); // "native D3D12 supports Models M, K only" (Model L: native_ok in runtime.cpp) ModelId id = ModelId::K; EXPECT(wgrt::model_from_name("m", &id) && id == ModelId::M); EXPECT(wgrt::model_from_name("L", &id) && id == ModelId::L); diff --git a/tools/ci/run_python_tests.py b/tools/ci/run_python_tests.py index bbed5d27..4532397a 100644 --- a/tools/ci/run_python_tests.py +++ b/tools/ci/run_python_tests.py @@ -30,7 +30,7 @@ 'clean/k/tests/test_k_hidden_guard.py', 'clean/k/tests/test_k_input_uniforms.py', 'clean/k/tests/test_k_interop_timing.py', 'clean/k/tests/test_k_norm_r16.py', 'clean/k/tests/test_k_output_arithmetic.py', 'clean/k/tests/test_k_output_selector.py', - 'clean/k/tests/test_k_runtime_gpu.py', 'clean/k/tests/test_k_tiled_input.py', + 'clean/k/tests/test_k_runtime_d3d12.py', 'clean/k/tests/test_k_runtime_gpu.py', 'clean/k/tests/test_k_tiled_input.py', 'clean/rr_f/proxy/tests/test_capture_replay.py', 'clean/rr_f/runtime/tests/test_model_pack.py', } RUN_ELSEWHERE = {'clean/wgshared/tests/test_dummy_extension.py', 'clean/wgshared/tests/test_gen_registry.py'}