From 530aa7c7c739334281432e1d327c04a38e1f3751 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Tue, 17 Nov 2026 00:36:29 -0500 Subject: [PATCH 1/2] M input encoder: opt-in E1b (runtime option e1b=on, default off) With e1b=on, Model M on the native D3D12 executor runs the encoder forms e1b/input_encoder[_direct] (encoder.hlsl, ENC_E1 == 2). Each wave issues the loads of its four history taps first. Then, per tap, one wave-wide test checks whether every lane's near / far tap position equals its centre tap position; where it does, that tap's values are copied from the centre tap instead of being computed again. The output is byte for byte the default encoder's. The gain depends on camera motion: still and evenly panning views save the most, strong parallax can be slightly slower. - runtime: runtime_options key e1b=on|off (also WG_RUNTIME_OPTIONS), default off; the kernel line in the log shows e1b=on, or e1b=off (...) when the model, executor or manifest cannot run it (the default encoder then runs). manifest_kernels() moves out of the Model L guard so that builds without Model L can check the manifest. - kernels: e1b/input_encoder and e1b/input_encoder_direct in the model M set (VGPR limit 128). ENC_E1 == 1 (the same test without the early loads) stays in the source as the reference form and is not declared. - test harness: WG_TEST_MOTION_PATTERN = ramp (default) | zero | pan | parallax for the timing legs. - FORMATS.md: the option. Option off: every default kernel unchanged. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/FORMATS.md | 8 ++ clean/dx12/kernel_sets.json | 8 ++ clean/dx12/kernels/encoder.hlsl | 91 +++++++++++++++++++ clean/dx12/vgpr_limits.json | 4 +- .../interop/tests/test_interop_d3d12.cpp | 10 +- clean/runtime/runtime.cpp | 50 ++++++---- 6 files changed, 153 insertions(+), 18 deletions(-) diff --git a/clean/FORMATS.md b/clean/FORMATS.md index 75d6af0d..bd28998a 100644 --- a/clean/FORMATS.md +++ b/clean/FORMATS.md @@ -1312,6 +1312,14 @@ with FP16 arithmetic: each weight rounded to FP16 once, `a ← h(fma(ŵ_k, t_k, FP16 rounding per step), then the clamp to the range of the five taps; the same in both passes. **analytic**: the Lanczos set of draft 3 with FP32 sums, the constants `kLz0 … kLz4` of `post.co` / `post_w64.co`, unchanged. +Runtime option `e1b` (`runtime_options` / `WG_RUNTIME_OPTIONS`; model M on the native D3D12 executor; default `off`): +`e1b=on` runs the input encoder forms `e1b/input_encoder[_direct]` (kernels/encoder.hlsl, `ENC_E1 == 2`). Each wave +issues the loads of its four history taps first, then tests per tap whether every lane's near / far tap position equals +its centre tap position. Where it does, that tap's values are copied from the centre tap instead of being computed +again. The output is byte for byte the default encoder's. The gain depends on camera motion: still and evenly panning +views save the most; strong parallax gains nothing and can be slightly slower. The log's kernel line shows `e1b=on`, or `e1b=off (...)` when the model, +executor or manifest cannot run it (the default encoder then runs). `e1b=off` or no key: the default encoder. + Runtime option `output_filter` (`runtime_options` / `WG_RUNTIME_OPTIONS`): | Value | Filter | diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 07fe9ce0..5bf742c2 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -187,6 +187,14 @@ [ "w64/post_tf_p12_diet_vsit_tex", "post_w64__orig.co" + ], + [ + "e1b/input_encoder", + "-" + ], + [ + "e1b/input_encoder_direct", + "-" ] ], "model_l": [ diff --git a/clean/dx12/kernels/encoder.hlsl b/clean/dx12/kernels/encoder.hlsl index a5e0fd18..a1226f0e 100644 --- a/clean/dx12/kernels/encoder.hlsl +++ b/clean/dx12/kernels/encoder.hlsl @@ -17,6 +17,12 @@ // and only the store is conditional: the same bytes. // @wg-kernel input_encoder co=encoder.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 // @wg-kernel input_encoder_direct co=encoder.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1 +// E1b (runtime option e1b=on, opt-in, default off): the near / far history taps copied from the centre tap when the +// whole wave's tap positions match (pixel_values, ENC_E1 == 2: all four taps' loads issued first, then one wave-wide +// test per tap); byte for byte the default encoder. ENC_E1 == 1 is the same test without the early loads (kept as the +// reference form; not declared). D3D12-only forms. +// @wg-kernel e1b/input_encoder co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=ENC_E1=2 +// @wg-kernel e1b/input_encoder_direct co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,ENC_E1=2 // // Motion at output resolution (WG_ENC_MO; HIP: input_encoder_mo of kernels/encoder_mo.hip, code object encoder_mo.co // under the tag "emo"; encoder_motion_outres.h): no depth neighbourhood; per pixel (x, y) one point read of the raw motion @@ -45,6 +51,9 @@ // @wg-kernel lev/input_encoder_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 // @wg-kernel lev/input_encoder_direct_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 #include "wg_platform.hlsli" +#ifndef ENC_E1 +#define ENC_E1 0 +#endif #ifndef WG_ENC_VALUES_OUT #define WG_ENC_VALUES_OUT 0 #endif @@ -163,6 +172,52 @@ float3 hist_values(int mode, Uni U, float px, float py, bool valid, float3 gc) return o; } +#if ENC_E1 == 2 +// E1b: hist_values split into its fetch (footprint + the two history loads) and its arithmetic, text for text, so that +// all four taps' loads are issued before the wave-uniform skip decisions (today's code overlaps them across taps). +struct HTap { uint2 t0, t1; float fx, fy; }; +HTap hist_fetch(Uni U, float px, float py) +{ + Foot ft = enc_footprint(px * U.shx - 0.5f, py * U.shy - 0.5f, P_hw, P_hh, 4u); + HTap h; + h.t0 = wg_ld64(hist, ft.off0); + h.t1 = wg_ld64(hist, ft.off1); + h.fx = ft.fx; + h.fy = ft.fy; + return h; +} +float3 hist_math(int mode, Uni U, HTap h, bool valid, float3 gc) +{ + uint2 t0 = h.t0, t1 = h.t1; + float kS = mode == kHistSame ? kS_SAME : kS_GEN; + float wy1 = h.fy * kS, wy0 = kS - wy1; + float ifx = 1.0f - h.fx; + float w00 = ifx * wy0, w10 = h.fx * wy0, w01 = ifx * wy1, w11 = h.fx * wy1; + float3 o; + [unroll] for (uint c = 0; c < 3; ++c) { + float a0 = (float)((t0.x >> (10u * c)) & 1023u), b0 = (float)((t0.y >> (10u * c)) & 1023u); + float a1 = (float)((t1.x >> (10u * c)) & 1023u), b1 = (float)((t1.y >> (10u * c)) & 1023u); + float x = mad(w00, a0, mad(w10, b0, 0.0f)); + x = mad(w01, a1, mad(w11, b1, x)); + float g; + if (mode == kHistSame) { + g = wg_med3f(x, k1em4, 3.0f); + } else { + float a; + if (mode == kHistGeneral) { + float pm1 = wg_exp2(wg_med3f(x, kL2_1em4, kL2_3)) - 1.0f; + a = mad(U.rr, pm1, 1.0f); + } else { + a = mad(x, U.ce, 1.0f); + } + g = kG * wg_log2(wg_fmax(a, k1em4)); + } + o[c] = valid ? g : gc[c]; + } + return o; +} +#endif + // The 20 values of the pixel at render column x, row y (pixel_values of encoder.hip). void pixel_values(int mode, Uni U, int x, int y, float4 c, float2 mn, float2 mf, float2 mc, out float v[20] #if defined(WG_DIRECT_TEXTURES) @@ -204,10 +259,46 @@ void pixel_values(int mode, Uni U, int x, int y, float4 c, float2 mn, float2 mf, [unroll] for (uint i2 = 15; i2 < 20; ++i2) v[i2] = 0.0f; return; } +#if ENC_E1 == 2 + // E1b (runtime option e1b=on): as E1, with all four taps' loads issued first (hist_fetch), then the skip decisions, then the + // arithmetic of the taps that are not copies (hist_math). Same operations per tap, same bytes. + { + const HTap f0 = hist_fetch(U, qx[0], qy[0]), f1 = hist_fetch(U, qx[1], qy[1]); + const HTap f2 = hist_fetch(U, qx[2], qy[2]), f3 = hist_fetch(U, qx[3], qy[3]); + const bool s0 = WaveActiveAllTrue(qx[0] == qx[1] && qy[0] == qy[1]); + const bool s2 = WaveActiveAllTrue(qx[2] == qx[1] && qy[2] == qy[1]); + const float3 h1 = hist_math(mode, U, f1, valid[1], gc); + float3 h0 = h1, h2 = h1; + if (!s0) h0 = hist_math(mode, U, f0, valid[0], gc); + if (!s2) h2 = hist_math(mode, U, f2, valid[2], gc); + const float3 h3 = hist_math(mode, U, f3, valid[3], gc); + v[3] = h0.x; v[4] = h0.y; v[5] = h0.z; + v[6] = h1.x; v[7] = h1.y; v[8] = h1.z; + v[9] = h2.x; v[10] = h2.y; v[11] = h2.z; + v[12] = h3.x; v[13] = h3.y; v[14] = h3.z; + } +#elif ENC_E1 + // E1 (reference form, not declared): the near (0) / far (2) tap is the centre tap (1) whenever its position is the centre's: same + // qx, qy -> same footprint, same valid flag, same values. When that holds for every lane of the wave (a ballot), + // the copy replaces the evaluation; otherwise the tap is computed as in the default. NaN positions compare unequal + // and are computed; +0 / -0 positions compare equal and give the same footprint. Same bytes in every case. + { + const float3 h1 = hist_values(mode, U, qx[1], qy[1], valid[1], gc); + float3 h0 = h1, h2 = h1; + if (!WaveActiveAllTrue(qx[0] == qx[1] && qy[0] == qy[1])) h0 = hist_values(mode, U, qx[0], qy[0], valid[0], gc); + if (!WaveActiveAllTrue(qx[2] == qx[1] && qy[2] == qy[1])) h2 = hist_values(mode, U, qx[2], qy[2], valid[2], gc); + const float3 h3 = hist_values(mode, U, qx[3], qy[3], valid[3], gc); + v[3] = h0.x; v[4] = h0.y; v[5] = h0.z; + v[6] = h1.x; v[7] = h1.y; v[8] = h1.z; + v[9] = h2.x; v[10] = h2.y; v[11] = h2.z; + v[12] = h3.x; v[13] = h3.y; v[14] = h3.z; + } +#else [unroll] for (uint k2 = 0; k2 < 4; ++k2) { float3 h = hist_values(mode, U, qx[k2], qy[k2], valid[k2], gc); v[3 + 3 * k2] = h.x; v[4 + 3 * k2] = h.y; v[5 + 3 * k2] = h.z; } +#endif // Motion disagreement (value 15) { diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index 492da741..f7a96cd9 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -40,5 +40,7 @@ "bneck/swin_bneck6_mb_nc": 192, "bneck/swin_bneck6_mbo_nc": 192, "d1p12/swin_block_c64_add_dec0_nc": 192, - "d1p12/swin_block_c64_add_dec0_xd_nc": 192 + "d1p12/swin_block_c64_add_dec0_xd_nc": 192, + "e1b/input_encoder": 128, + "e1b/input_encoder_direct": 128 } diff --git a/clean/runtime/interop/tests/test_interop_d3d12.cpp b/clean/runtime/interop/tests/test_interop_d3d12.cpp index 9d39919a..c4d12cb8 100644 --- a/clean/runtime/interop/tests/test_interop_d3d12.cpp +++ b/clean/runtime/interop/tests/test_interop_d3d12.cpp @@ -433,9 +433,17 @@ struct Scene { g.upload(depth, tw, th, 4, d.data(), depthState); std::vector m((size_t)mw * mh * 4); motionF.assign((size_t)mw * mh * 2, 0.0f); + // WG_TEST_MOTION_PATTERN (input encoder E1b timing): ramp (default: the field below), zero (a static scene), pan (one + // constant vector), parallax (the ramp plus a depth term: every neighbour differs, and more so near the camera) + const char* mpat = std::getenv("WG_TEST_MOTION_PATTERN"); + const int mp = !mpat ? 0 : !std::strcmp(mpat, "zero") ? 1 : !std::strcmp(mpat, "pan") ? 2 : !std::strcmp(mpat, "parallax") ? 3 : 0; for (UINT y = 0; y < mh; ++y) for (UINT x = 0; x < mw; ++x) { - const float a = q16(x / 128.0f - 0.25f), b = q16(0.5f - y / 64.0f); + const float dd = (x < tw && y < th) ? d[(size_t)y * tw + x] : 0.0f; + const float a = mp == 1 ? 0.0f : mp == 2 ? q16(0.75f) : mp == 3 ? q16(x / 128.0f - 0.25f + 48.0f * dd) + : q16(x / 128.0f - 0.25f); + const float b = mp == 1 ? 0.0f : mp == 2 ? q16(-0.375f) : mp == 3 ? q16(0.5f - y / 64.0f - 24.0f * dd) + : q16(0.5f - y / 64.0f); uint16_t* p = &m[4 * ((size_t)y * mw + x)]; p[0] = to_half(a), p[1] = to_half(b), p[2] = to_half(7.0f), p[3] = 0; motionF[2 * ((size_t)y * mw + x)] = a; diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 95c6e460..236de817 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -519,6 +519,11 @@ struct Runtime { // swin_resident=N (runtime option / WG_SWIN_RESIDENT): the plain C = 64 block launches (enc1; dec1 when not fused) // run the weight-resident persistent kernels swin_block_c64[_add]_r[_nc] with N workgroups of 512 lanes; 0 = off int swin_resident = 0; + // e1b=on (runtime option; default off; Model M on native D3D12): the input encoder's near / far history taps copied + // from the centre tap when the whole wave's tap positions match (e1b/input_encoder[_direct], kernels/encoder.hlsl + // ENC_E1 == 2): byte for byte the default encoder; faster when the camera is still or pans evenly. Without the two + // kernels in the manifest the default encoder runs (one log line). + bool e1b_opt = false, e1b_built = false; bool post_diet = true; // option post_diet (default 1; 0 = the previous kernels): exposure_pair_diet + the _diet post kernels (kernels/post.hip); bit-identical // On top of post_diet, the true-fusion kernel post_tf_*_diet_ (kernels/post.hip WG_TF_KERNEL_DIET_X; all three // off = the plain post_tf_*_diet), each bit-identical and on by default: @@ -1438,6 +1443,10 @@ struct Runtime { if (k == "dec_rdiv") dec_rdiv = !(v == "0" || v == "off" || v == "false"); if (k == "dec_bdiv") dec_bdiv = !(v == "0" || v == "off" || v == "false"); if (k == "swin_resident") swin_resident = std::max(0, std::atoi(v.c_str())); + if (k == "e1b") { + if (v != "on" && v != "off") return fail("e1b must be on or off"); + e1b_opt = v == "on"; + } if (k == "post_tf_cpairs") post_tf_cpairs = std::max(2, std::atoi(v.c_str()) & ~1); // even if (k == "post_tf4") { if (v != "auto" && v != "force") return fail("post_tf4 must be auto or force"); @@ -1754,6 +1763,13 @@ struct Runtime { kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll || fz_le) kernels_text += " d3d12_fusion=" + fz_text; + if (e1b_opt) { // e1b=on: Model M on native D3D12 with both kernels in the manifest, else the default encoder + const std::vector have = manifest_kernels(); + e1b_built = d3d12_native && !model_l && + std::find(have.begin(), have.end(), "e1b/input_encoder") != have.end() && + std::find(have.begin(), have.end(), "e1b/input_encoder_direct") != have.end(); + kernels_text += std::string(" e1b=") + (e1b_built ? "on" : "off (needs Model M on native D3D12 and its kernels)"); + } kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -1947,6 +1963,22 @@ struct Runtime { return true; } + // The kernel names of /dx12/manifest.json (none if it is missing or unreadable). + std::vector manifest_kernels() const { + std::string text, error; + wgc::Json manifest; + std::vector have; + if (read_text(build_dir + "/dx12/manifest.json", &text) && wgc::parse_json(text, manifest, error)) { + const auto* kernels = manifest.find("kernels"); + if (kernels && kernels->is_array()) + for (const auto& k : kernels->items) { + const auto* name = k.find("name"); + if (name && name->is_string()) have.push_back(name->text); + } + } + return have; + } + #ifdef WG_MODEL_L // ---- Model L (clean/l/FORMATS.md: maps, blobs, kernels, launch order) ---- // Window offsets (ox, oy) of Model L's two width-32 blocks for frame parity f: the default table's window_offsets @@ -2089,21 +2121,6 @@ struct Runtime { // its encoder (the direct form on the direct-resource route), linears, clamp-free blocks (the only D3D12 forms), // projections and pack12 coefficient store, and dec1's own block with its expand (Model M runs dec1 inside its fused // decoder there; Model L's decoder reads dec1's map). Model M's other kernels are its native frame's. - // The kernel names of /dx12/manifest.json (none if it is missing or unreadable). - std::vector manifest_kernels() const { - std::string text, error; - wgc::Json manifest; - std::vector have; - if (read_text(build_dir + "/dx12/manifest.json", &text) && wgc::parse_json(text, manifest, error)) { - const auto* kernels = manifest.find("kernels"); - if (kernels && kernels->is_array()) - for (const auto& k : kernels->items) { - const auto* name = k.find("name"); - if (name && name->is_string()) have.push_back(name->text); - } - } - return have; - } std::string l_native_missing() const { if (l_q7) // the FP16-form blocks (clean/dx12/kernels/l_q7.hlsl, l_q7w.hlsl) in place of the l_swin / Model M blocks @@ -2647,7 +2664,8 @@ struct Runtime { {&cnet, &mhi, &reduced, &latent[cur], &mnear[cur], &feat0, &mnear[nxt], &mfar, &mcenter, &expo, &w, &p, &emop})) return false; - } else if (!launch(native_direct_io ? "input_encoder_direct" : "input_encoder", + } else if (!launch(e1b_built ? (native_direct_io ? "e1b/input_encoder_direct" : "e1b/input_encoder") + : (native_direct_io ? "input_encoder_direct" : "input_encoder"), cdiv(ntok, 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &feat0, &mnear[nxt], &mfar, &mcenter, &expo, &w, &p})) From 1cfbd590c7831da1fa2019d48d46a33de2a2354a Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Tue, 17 Nov 2026 01:55:32 -0500 Subject: [PATCH 2/2] E1b: resolve at create, keep it out of the default warm-up, never fail create - device (native D3D12): the default warm-up skips the opt-in forms (optional_kernel: e1b/*), so a missing or failing E1b shader cannot fail a default create, and a default create compiles exactly what it did before. - runtime: the e1b key is read once at create, before Model K's create returns (e1b_option.h e1b_scan), and resolved once (e1b_resolve): E1b runs only for Model M on the native D3D12 executor without motion at output resolution, and only when the route's own form (direct or buffer) prepares. Otherwise the default encoder runs and the reason is kept: Model L, Model K, not the native D3D12 executor, motion at output resolution, not in the manifest, or could not be prepared. The kernel report only reads it (no side effects); the launch follows the create-time route from the first frame. - an invalid value (e.g. e1b=1 or a typo) no longer fails create: E1b stays off and the log gets one line, "e1b: unknown value '1', using off". - tests: clean/runtime/tests/test_e1b_option.py (CPU): the value parse, the options scan and the report item (test_e1b_option.cpp, MSVC), and source checks of the wiring (read and resolved at create, side-effect-free report, no opt-in form in the default warm-up). - test harness: WG_TEST_RESET_AT=[,...] for mid-stream resets in the byte legs. - FORMATS.md: the create-time choice, the reasons, the fallback, and the byte-identity scope. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/FORMATS.md | 15 ++- clean/runtime/device_d3d12.cpp | 6 +- clean/runtime/e1b_option.h | 33 ++++++ .../interop/tests/test_interop_d3d12.cpp | 11 ++ clean/runtime/runtime.cpp | 67 ++++++++---- clean/runtime/tests/test_e1b_option.cpp | 58 ++++++++++ clean/runtime/tests/test_e1b_option.py | 102 ++++++++++++++++++ 7 files changed, 270 insertions(+), 22 deletions(-) create mode 100644 clean/runtime/e1b_option.h create mode 100644 clean/runtime/tests/test_e1b_option.cpp create mode 100644 clean/runtime/tests/test_e1b_option.py diff --git a/clean/FORMATS.md b/clean/FORMATS.md index bd28998a..d092c901 100644 --- a/clean/FORMATS.md +++ b/clean/FORMATS.md @@ -1316,9 +1316,18 @@ Runtime option `e1b` (`runtime_options` / `WG_RUNTIME_OPTIONS`; model M on the n `e1b=on` runs the input encoder forms `e1b/input_encoder[_direct]` (kernels/encoder.hlsl, `ENC_E1 == 2`). Each wave issues the loads of its four history taps first, then tests per tap whether every lane's near / far tap position equals its centre tap position. Where it does, that tap's values are copied from the centre tap instead of being computed -again. The output is byte for byte the default encoder's. The gain depends on camera motion: still and evenly panning -views save the most; strong parallax gains nothing and can be slightly slower. The log's kernel line shows `e1b=on`, or `e1b=off (...)` when the model, -executor or manifest cannot run it (the default encoder then runs). `e1b=off` or no key: the default encoder. +again. On the validated driver the output is byte for byte the default encoder's (the two forms compile to the same +floating-point arithmetic instructions). The gain depends on camera motion: still and evenly panning views save the +most; strong parallax gains nothing and can be slightly slower. + +The choice is made once at create (runtime.cpp `e1b_resolve`). E1b runs only for model M on the native D3D12 executor +without motion at output resolution, and only when its form for the context's route (direct or buffer) prepares: +shader file or built-in copy, digest, pipeline. The default warm-up never compiles the E1b forms (device_d3d12.cpp +`optional_kernel`), so a missing or failing E1b shader cannot fail a create. The log's kernel line shows `e1b=on`, or +`e1b=off ()`: Model L, Model K, not the native D3D12 executor, motion at output resolution, not in the manifest, +or could not be prepared. The default encoder then runs. `e1b=off` or no key: the default encoder, and no `e1b` item. +Any other value (a typo) does not fail create: E1b stays off and the log gets one line, +`e1b: unknown value '', using off` (clean/runtime/e1b_option.h). Runtime option `output_filter` (`runtime_options` / `WG_RUNTIME_OPTIONS`): diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index 31bc79d2..d9632065 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -502,11 +502,15 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { return v; } bool external_markers() const override { return marker_map_ != nullptr; } + // Opt-in kernel forms (selected by a runtime option, e.g. e1b=on): never compiled with the default set. The runtime + // prepares the one form it selects at create (prepare()), and falls back to the default kernel if that fails, so a + // missing or failing optional shader cannot fail a default create. + static bool optional_kernel(const std::string& name) { return name.compare(0, 4, "e1b/") == 0; } bool warm_up(std::string* error) override { if (external_) return extension_result(fail("native D3D12 warm_up: recording into the caller's list"), error); for (auto& entry : kernels_) { NativeKernel& k = entry.second; - if (k.pso || !variant_matches(k) || (k.rs != "buffer" && k.rs != "direct")) continue; + if (k.pso || optional_kernel(k.name) || !variant_matches(k) || (k.rs != "buffer" && k.rs != "direct")) continue; if (!create_pso(k)) return extension_result(false, error); } return extension_result(true, error); diff --git a/clean/runtime/e1b_option.h b/clean/runtime/e1b_option.h new file mode 100644 index 00000000..c196830e --- /dev/null +++ b/clean/runtime/e1b_option.h @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: MIT +// The runtime option e1b (runtime.cpp e1b_resolve; clean/FORMATS.md "Runtime option e1b"). "on" asks for E1b, "off" +// turns it off. Any other value is most likely a typo in a player's ini, so it never fails create: E1b stays off and +// *note gets one line for the log. A valid value clears *note, so the last e1b key decides. +#pragma once +#include + +inline bool e1b_option_on(const std::string& value, std::string* note) { + if (note) note->clear(); + if (value == "on") return true; + if (value != "off" && note) *note = "e1b: unknown value '" + value + "', using off"; + return false; +} + +// The e1b key of a runtime options string ("key=value" items separated by ';' or ','; other keys are ignored). +// No e1b key: off, no note. +inline void e1b_scan(const std::string& options, bool* on, std::string* note) { + *on = false; + note->clear(); + for (size_t a = 0; a <= options.size();) { + size_t b = options.find_first_of(";,", a); + if (b == std::string::npos) b = options.size(); + const std::string kv = options.substr(a, b - a); + a = b + 1; + const size_t eq = kv.find('='); + if (eq != std::string::npos && kv.compare(0, eq, "e1b") == 0 && eq == 3) *on = e1b_option_on(kv.substr(eq + 1), note); + } +} + +// The kernel report's e1b item (only when the option asked for E1b): the route chosen at create, never a side effect. +inline std::string e1b_report(bool on, const std::string& reason) { + return on ? std::string(" e1b=on") : " e1b=off (" + reason + ")"; +} diff --git a/clean/runtime/interop/tests/test_interop_d3d12.cpp b/clean/runtime/interop/tests/test_interop_d3d12.cpp index c4d12cb8..e4dc0b52 100644 --- a/clean/runtime/interop/tests/test_interop_d3d12.cpp +++ b/clean/runtime/interop/tests/test_interop_d3d12.cpp @@ -2192,6 +2192,17 @@ bool case_soak_pipe(Gpu& g, const char* dll, const std::string& options, int fra f.jitter[0] = jitter[(t - 1) % 4][0]; f.jitter[1] = jitter[(t - 1) % 4][1]; } + // WG_TEST_RESET_AT=[,...] (this test program only): a history reset on those frames as well + // (mid-stream resets for the byte legs; the timing runs set none) + if (const char* ra = std::getenv("WG_TEST_RESET_AT")) { + for (const char* p = ra; *p;) { + char* e = nullptr; + const long v = std::strtol(p, &e, 10); + if (e == p) break; + if (v == t) f.reset = 1; + p = *e ? e + 1 : e; + } + } f.command_list = g.list; const WgResult rr = c.be.b->dispatch(c.u, &f); if (rr != WG_OK) { std::printf(" soak-pipe frame %d: dispatch %d%c", t, (int)rr, 10); CHECK(rr == WG_OK); return false; } diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 236de817..57485ccb 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -23,6 +23,7 @@ #include "m_q7_pack.h" #endif #include "device.h" +#include "e1b_option.h" #include "embedded.h" #include "fpconv.h" #include "model.h" @@ -519,11 +520,15 @@ struct Runtime { // swin_resident=N (runtime option / WG_SWIN_RESIDENT): the plain C = 64 block launches (enc1; dec1 when not fused) // run the weight-resident persistent kernels swin_block_c64[_add]_r[_nc] with N workgroups of 512 lanes; 0 = off int swin_resident = 0; - // e1b=on (runtime option; default off; Model M on native D3D12): the input encoder's near / far history taps copied - // from the centre tap when the whole wave's tap positions match (e1b/input_encoder[_direct], kernels/encoder.hlsl - // ENC_E1 == 2): byte for byte the default encoder; faster when the camera is still or pans evenly. Without the two - // kernels in the manifest the default encoder runs (one log line). - bool e1b_opt = false, e1b_built = false; + // e1b=on (runtime option, e1b_option.h; default off): Model M's input encoder forms e1b/input_encoder[_direct] + // (kernels/encoder.hlsl ENC_E1 == 2): the near / far history taps copied from the centre tap when the whole wave's tap + // positions match; byte for byte the default encoder on the validated driver; faster when the camera is still or pans + // evenly. Resolved once at create (e1b_resolve): e1b_on = the route's E1b kernel prepared, else e1b_reason says why + // the default encoder runs. Never fails create; kernels() only reports it. + bool e1b_opt = false, e1b_on = false; + std::string e1b_note; // e1b with an unknown value: one line for the log at create + std::string e1b_reason; // why e1b=on runs the default encoder (kernels() report) + std::string k_kernels_text; // model K's kernel report with the e1b item bool post_diet = true; // option post_diet (default 1; 0 = the previous kernels): exposure_pair_diet + the _diet post kernels (kernels/post.hip); bit-identical // On top of post_diet, the true-fusion kernel post_tf_*_diet_ (kernels/post.hip WG_TF_KERNEL_DIET_X; all three // off = the plain post_tf_*_diet), each bit-identical and on by default: @@ -612,6 +617,10 @@ struct Runtime { d3d12_native = c.backend == 3; if (c.backend == 3 && external && !native_direct_io) return fail("native D3D12 external I/O requires the private direct-resource route"); + { // e1b (e1b_option.h): read before Model K's create returns, so every model validates and reports it + const char* env = std::getenv("WG_RUNTIME_OPTIONS"); + e1b_scan(std::string(c.runtime_options ? c.runtime_options : "") + ";" + (env ? env : ""), &e1b_opt, &e1b_note); + } #ifndef WG_NO_MODEL_K if (model_id == ModelId::K) return create_k(c, std::move(given)); #else @@ -1160,6 +1169,7 @@ struct Runtime { // the fused dec1 + dec0 kernel takes dec1's block, so dec1's expand stays a separate launch if (dec1_dec0) fuse_stages.erase(std::remove(fuse_stages.begin(), fuse_stages.end(), std::string("dec1")), fuse_stages.end()); + e1b_resolve(); if (native_direct_io && !validate_native_direct_config()) return false; for (auto& kv : blobs) { if (kv.first == kFilterStage) continue; // the output-filter weights are not read from device memory @@ -1200,6 +1210,8 @@ struct Runtime { return fail(k->err()); blobs = k->blobs(); notes_text = k->notes(); // one line when the softmax stages take their round-to-odd ports (no silent switch) + if (!e1b_note.empty()) notes_text += std::string(notes_text.empty() ? "" : "\n") + e1b_note; + if (e1b_opt) e1b_reason = "Model K"; // E1b is a Model M encoder form model_source = "model.wgcm"; return true; } @@ -1415,6 +1427,31 @@ struct Runtime { // Runtime options: graph=1 (HIP graph per frame), stream_priority=high (highest stream priority). Keys of the // configuration string first, then WG_RUNTIME_OPTIONS (its keys win, as in the interop). Unknown keys are ignored. + // e1b, resolved once at create (after every option and the device exist). E1b runs only for Model M on the native + // D3D12 executor without motion at output resolution, and only when the route's own E1b kernel prepares (shader on + // disk or built in, digest, pipeline). Otherwise the default encoder runs and e1b_reason says why. Never fails + // create: an optional kernel that cannot be prepared is a fallback, not an error. + void e1b_resolve() { + e1b_on = false; + e1b_reason.clear(); + if (!e1b_note.empty()) notes_text += std::string(notes_text.empty() ? "" : "\n") + e1b_note; + if (!e1b_opt) return; + const std::string k = native_direct_io ? "e1b/input_encoder_direct" : "e1b/input_encoder"; + const std::vector have = manifest_kernels(); + if (model_l) + e1b_reason = "Model L"; + else if (!d3d12_native) + e1b_reason = "not the native D3D12 executor"; + else if (motion_outres == 1) + e1b_reason = "motion at output resolution runs its own encoder"; + else if (std::find(have.begin(), have.end(), k) == have.end()) + e1b_reason = k + " is not in the manifest"; + else if (!dev->prepare(k)) + e1b_reason = k + " could not be prepared: " + dev->err(); + else + e1b_on = true; + } + bool runtime_options(const WgRtConfig& c) { const char* env = std::getenv("WG_RUNTIME_OPTIONS"); options_text = std::string(c.runtime_options ? c.runtime_options : "") + ";" + (env ? env : ""); @@ -1443,10 +1480,6 @@ struct Runtime { if (k == "dec_rdiv") dec_rdiv = !(v == "0" || v == "off" || v == "false"); if (k == "dec_bdiv") dec_bdiv = !(v == "0" || v == "off" || v == "false"); if (k == "swin_resident") swin_resident = std::max(0, std::atoi(v.c_str())); - if (k == "e1b") { - if (v != "on" && v != "off") return fail("e1b must be on or off"); - e1b_opt = v == "on"; - } if (k == "post_tf_cpairs") post_tf_cpairs = std::max(2, std::atoi(v.c_str()) & ~1); // even if (k == "post_tf4") { if (v != "auto" && v != "force") return fail("post_tf4 must be auto or force"); @@ -1715,7 +1748,11 @@ struct Runtime { const char* kernels() { #ifndef WG_NO_MODEL_K - if (k) return k->kernels(); + if (k) { // model K: its own report, plus the e1b item when the option asked for E1b (never used by K) + if (!e1b_opt) return k->kernels(); + k_kernels_text = std::string(k->kernels()) + e1b_report(false, e1b_reason); + return k_kernels_text.c_str(); + } #endif kernels_text = "swin=" + swin_co + " enc0=" + enc0_co + " dec0=" + dec0_name + " exposure=" + exposure_variant + " post=" + post_kernel + " filter=" + (post_fused ? std::string("fused") : filter_kernel) + @@ -1763,13 +1800,7 @@ struct Runtime { kernels_text += std::string(" dec_bdiv=") + (dec_div.find('b') == std::string::npos ? "1" : "0"); kernels_text += std::string(" dec1_dec0=") + (dec1_dec0 ? "fused" : "separate"); if (fz_pm || fz_xe || fz_mb || fz_rv || fz_xo || fz_xd || fz_lc || fz_ll || fz_le) kernels_text += " d3d12_fusion=" + fz_text; - if (e1b_opt) { // e1b=on: Model M on native D3D12 with both kernels in the manifest, else the default encoder - const std::vector have = manifest_kernels(); - e1b_built = d3d12_native && !model_l && - std::find(have.begin(), have.end(), "e1b/input_encoder") != have.end() && - std::find(have.begin(), have.end(), "e1b/input_encoder_direct") != have.end(); - kernels_text += std::string(" e1b=") + (e1b_built ? "on" : "off (needs Model M on native D3D12 and its kernels)"); - } + if (e1b_opt) kernels_text += e1b_report(e1b_on, e1b_reason); kernels_text += " swin_mid=" + (mid_variant.empty() ? std::string("none") : mid_variant); kernels_text += std::string(" post_pack=") + (p12 ? "p12" : "none"); kernels_text += std::string(" pre_exposure=") + (pi_original ? "original" : "scale"); @@ -2664,7 +2695,7 @@ struct Runtime { {&cnet, &mhi, &reduced, &latent[cur], &mnear[cur], &feat0, &mnear[nxt], &mfar, &mcenter, &expo, &w, &p, &emop})) return false; - } else if (!launch(e1b_built ? (native_direct_io ? "e1b/input_encoder_direct" : "e1b/input_encoder") + } else if (!launch(e1b_on ? (native_direct_io ? "e1b/input_encoder_direct" : "e1b/input_encoder") : (native_direct_io ? "input_encoder_direct" : "input_encoder"), cdiv(ntok, 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &feat0, &mnear[nxt], &mfar, &mcenter, diff --git a/clean/runtime/tests/test_e1b_option.cpp b/clean/runtime/tests/test_e1b_option.cpp new file mode 100644 index 00000000..3c90648f --- /dev/null +++ b/clean/runtime/tests/test_e1b_option.cpp @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: MIT +// CPU test of the e1b runtime option (clean/runtime/e1b_option.h): the value parse (on / off; every other value falls +// back to off with one log line, never a failed create), the scan of a whole options string, and the report item. +// Built and run by test_e1b_option.py. +#include +#include + +#include "../e1b_option.h" + +static int failures = 0; + +static void expect(bool ok, const std::string& what) { + if (!ok) { + std::printf("FAIL %s\n", what.c_str()); + ++failures; + } +} + +static void value(const char* v, bool want_on, const std::string& want_note) { + std::string note = "stale"; + const bool on = e1b_option_on(v, ¬e); + expect(on == want_on && note == want_note, + std::string("e1b=") + v + ": on " + (on ? "1" : "0") + ", note '" + note + "'"); +} + +static void scan(const char* opts, bool want_on, const std::string& want_note) { + bool on = !want_on; + std::string note = "stale"; + e1b_scan(opts, &on, ¬e); + expect(on == want_on && note == want_note, + std::string("scan '") + opts + "': on " + (on ? "1" : "0") + ", note '" + note + "'"); +} + +int main() { + value("on", true, ""); + value("off", false, ""); + value("1", false, "e1b: unknown value '1', using off"); + value("ON", false, "e1b: unknown value 'ON', using off"); + value("yes", false, "e1b: unknown value 'yes', using off"); + value("", false, "e1b: unknown value '', using off"); + value("onn", false, "e1b: unknown value 'onn', using off"); + expect(!e1b_option_on("off", nullptr) && e1b_option_on("on", nullptr), "no note pointer"); + + scan("", false, ""); + scan("executor=dx12;model=M", false, ""); + scan("executor=dx12;model=M;e1b=on", true, ""); + scan("e1b=on,model=M", true, ""); + scan("model=M;e1b=1", false, "e1b: unknown value '1', using off"); + scan("e1b=1;e1b=on", true, ""); // the last e1b key decides + scan("e1b=on;e1b=typo", false, "e1b: unknown value 'typo', using off"); + scan("e1bx=on;xe1b=on;e1b", false, ""); // other keys and a key without a value are ignored + scan("e1b=on;", true, ""); // the interop's "configuration;environment" join + + expect(e1b_report(true, "") == " e1b=on", "report on"); + expect(e1b_report(false, "Model K") == " e1b=off (Model K)", "report off"); + std::printf("e1b option: %d failure(s)\n", failures); + return failures ? 1 : 0; +} diff --git a/clean/runtime/tests/test_e1b_option.py b/clean/runtime/tests/test_e1b_option.py new file mode 100644 index 00000000..ee9e944a --- /dev/null +++ b/clean/runtime/tests/test_e1b_option.py @@ -0,0 +1,102 @@ +# SPDX-License-Identifier: MIT +"""The runtime option e1b. An invalid value must fall back to off with one log line, never fail create (a typo in a +player's ini must not lose the upscaler). clean/runtime/tests/test_e1b_option.cpp is built with MSVC cl (on PATH, e.g. a +developer prompt or CI, or through vcvars64.bat; skipped without one) and checks the value parse, the options scan and +the report item. A source check confirms the wiring: the option is read and resolved at create, the kernel report has +no side effects, and the default warm-up never compiles an opt-in form. + +Run: python -m unittest clean.runtime.tests.test_e1b_option +""" +import os +import re +import shutil +import subprocess +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +RT = os.path.join(ROOT, "clean", "runtime") + + +def _vcvars64(): + """vcvars64.bat of the newest Visual Studio (vswhere), or None.""" + vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), + "Microsoft Visual Studio", "Installer", "vswhere.exe") + if not os.path.exists(vswhere): + return None + r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True, + text=True) + vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat") + return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None + + +def _read(name): + with open(os.path.join(RT, name), encoding="utf-8") as f: + return f.read() + + +def _block(src, head): + """The text from `head` to the end of that member function (the next line that is just two spaces and a brace).""" + start = src.find(head) + if start < 0: + return "" + end = src.find("\n }\n", start) + return src[start:end if end >= 0 else len(src)] + + +class E1bOption(unittest.TestCase): + def test_parse_scan_report(self): + if os.name != "nt": + raise unittest.SkipTest("built with MSVC cl: Windows only") + vcvars = None if shutil.which("cl") else _vcvars64() + if not shutil.which("cl") and not vcvars: + raise unittest.SkipTest("no MSVC cl (not on PATH, no Visual Studio found)") + work = tempfile.mkdtemp(prefix="wg_e1b_") + exe = os.path.join(work, "test_e1b_option.exe") + src = os.path.join(RT, "tests", "test_e1b_option.cpp") + bat = os.path.join(work, "build.bat") + with open(bat, "w") as f: + f.write("@echo off\r\n") + if vcvars: + f.write('call "%s" >nul || exit /b 1\r\n' % vcvars) + f.write('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /W4 /WX "%s" /Fo"%s\\\\" /Fe"%s" || exit /b 1\r\n' + % (src, work, exe)) + r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, "build failed:\n" + r.stdout + r.stderr) + r = subprocess.run([exe], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + self.assertIn("e1b option: 0 failure(s)", r.stdout) + shutil.rmtree(work, ignore_errors=True) + + def test_runtime_wiring(self): + src = _read("runtime.cpp") + self.assertIn('#include "e1b_option.h"', src) + self.assertNotIn("e1b must be", src, "an invalid e1b value must not fail create") + # read before Model K's create returns, so every model validates and reports it + scan, k_create = src.find("e1b_scan("), src.find("return create_k(c, std::move(given));") + self.assertTrue(0 <= scan < k_create, "e1b_scan must come before Model K's create") + # resolved at create: create() calls e1b_resolve(), which prepares the route's kernel and alone sets e1b_on + create = src[src.find(" bool create(const WgRtConfig& c"):src.find(" bool create_k(")] + self.assertIn("e1b_resolve();", create) + resolve = _block(src, " void e1b_resolve() {") + self.assertIn("dev->prepare(k)", resolve) + assign = re.compile(r"^\s+e1b_on = [^;]*;", re.M) # statements (not the declaration or a comment) + self.assertEqual(len(assign.findall(src)), 2, "e1b_on is set only in e1b_resolve") + self.assertEqual(len(assign.findall(resolve)), 2) + # the report is side-effect free: kernels() reads the route resolved at create and changes nothing + body = _block(src, " const char* kernels() {") + self.assertIn("e1b_report(e1b_on, e1b_reason)", body) + for forbidden in ("e1b_on =", "e1b_reason =", "e1b_opt =", "prepare(", "manifest_kernels()"): + self.assertNotIn(forbidden, body, "kernels() must not contain " + forbidden) + # the launch follows the create-time route + self.assertIn('launch(e1b_on ? (native_direct_io ? "e1b/input_encoder_direct" : "e1b/input_encoder")', src) + + def test_default_warm_up_skips_opt_in_forms(self): + dev = _read("device_d3d12.cpp") + warm = _block(dev, " bool warm_up(std::string* error) override {") + self.assertIn("optional_kernel(k.name)", warm, "the default warm-up must skip the opt-in forms") + self.assertIn('name.compare(0, 4, "e1b/") == 0', dev) + + +if __name__ == "__main__": + unittest.main()