From a39f019e98128dcf35ff85e57c3ab03237f80cf3 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 16:04:10 -0500 Subject: [PATCH 1/6] Model K: pack tiled input transfer with exact FP16 breakpoints Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/dx12/hlsl/k_in_common.hlsli | 24 ++++ clean/k/dx12/kernels/k_in.hlsl | 32 +++-- clean/k/runtime/k_s_breakpoints.h | 76 ++++++++++++ clean/k/runtime/runtime_k.cpp | 10 +- clean/k/runtime/runtime_k.h | 1 + clean/k/tests/ki_s_breakpoints_cpu.cpp | 14 +++ clean/k/tests/prove_ki_s_breakpoints.py | 156 ++++++++++++++++++++++++ 7 files changed, 303 insertions(+), 10 deletions(-) create mode 100644 clean/k/runtime/k_s_breakpoints.h create mode 100644 clean/k/tests/ki_s_breakpoints_cpu.cpp create mode 100644 clean/k/tests/prove_ki_s_breakpoints.py diff --git a/clean/k/dx12/hlsl/k_in_common.hlsli b/clean/k/dx12/hlsl/k_in_common.hlsli index df5ff083..c62cdb79 100644 --- a/clean/k/dx12/hlsl/k_in_common.hlsli +++ b/clean/k/dx12/hlsl/k_in_common.hlsli @@ -56,6 +56,22 @@ float ki_S(float z) const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu)); } +#if defined(KI_TILED_PACKED_S) +uint ki_tiled_S_bits(float z, uint table) +{ +#if KI_HWMATH + return kh_round_bits(ki_S(z)); +#else + const float zp = kf_min(kf_max(z, 0.0f), 1.0f); + if (!(zp > kf_bits(0x3B4D2E1Cu))) return kh_round_bits(kf_bits(0x414EB852u) * zp); + const uint u = asuint(zp); + // Every reachable 4,096-code bucket is flat or makes one +1 transition + // (clean.k.tests.prove_ki_s_breakpoints). The cut is 4,096 for flat buckets. + const uint e = wg_ld32(table, ((u - 0x3B4D2000u) >> 12u) * 4u); + return (e & 0xffffu) + ((u & 0xfffu) >= (e >> 16u) ? 1u : 0u); +#endif +} +#endif // z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart // (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it) float ki_T(float z) @@ -132,3 +148,11 @@ void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3) wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); } + +#if defined(KI_TILED_PACKED_S) +void ki_store_tiled_x0(uint x0, int id, int k, uint c0, uint c1, float c2, float c3) +{ + wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u, + uint2(c0 | (c1 << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16))); +} +#endif diff --git a/clean/k/dx12/kernels/k_in.hlsl b/clean/k/dx12/kernels/k_in.hlsl index a40c4c11..0497c08d 100644 --- a/clean/k/dx12/kernels/k_in.hlsl +++ b/clean/k/dx12/kernels/k_in.hlsl @@ -27,7 +27,7 @@ // motion from the game's textures t0 / t1 / t2, k_frame.hlsli kio_texel / kio_motion / // kio_depth; WG_DIRECT_TEXTURES); the uniform form has no direct form (the native executor has // no host frame values) -// The KInParams pointers cpre / cpre2 are null in the tiled launches (runtime_k.cpp), as in HIP unused here. +// The exact D3D12 tiled launches use KInParams.cpre for the packed S transfer table. // @wg-kernel kx/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1 // @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 // @wg-kernel kx/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1 @@ -65,8 +65,21 @@ WG_ARGS_END float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } +#if !KI_HWMATH +#define KI_TILED_PACKED_S 1 +#endif #include "k_in_common.hlsli" +#if KI_HWMATH +#define KI_TILED_CHANNEL float +#define KI_TILED_S(z, table) ki_S(z) +#define KI_TILED_STORE ki_store_x0 +#else +#define KI_TILED_CHANNEL uint +#define KI_TILED_S(z, table) ki_tiled_S_bits(z, table) +#define KI_TILED_STORE ki_store_tiled_x0 +#endif + struct KiTiledTap { float4 cv; // Y, Co, Cg, phi(Y) float om; // HDR weight max(1 / (Y + 1), 1e-4); 1 for LDR (A = fma(1, v, A) is HIP's A + v, one rounding of the sum) @@ -83,12 +96,12 @@ KiTiledTap ki_tiled_tap_from_raw(KFrameIn F, float4 c) return r; } KiTiledTap ki_tiled_tap(KFrameIn F, int rx, int ry) { return ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); } -float ki_tiled_ch0(KFrameIn F, int rx, int ry) +KI_TILED_CHANNEL ki_tiled_ch0(KFrameIn F, int rx, int ry, uint table) { const float3 rgb = ki_rgb(F, kio_color(F, rx, ry)); const bool hdr = (F.flags & KIO_HDR) != 0; const float Yc = kio_luma_fma(rgb.x, rgb.y, rgb.z); - return ki_S(ki_T(F.lam * ki_L(Yc, hdr))); + return KI_TILED_S(ki_T(F.lam * ki_L(Yc, hdr)), table); } // HIP ki_tiled_transpose: lane bit 0 / bit 1 swaps through ds_bpermute (wg_xor = WaveReadLaneAt(lane ^ d)) float4 ki_tiled_transpose(float4 v, int lane) @@ -162,7 +175,8 @@ WG_KERNEL(k_in_tiled) const int id = by * F.Bw + bx; const bool hdr = (F.flags & KIO_HDR) != 0; const float c1e4 = kf_bits(0x38D1B717u); - float ch0 = 0.0f, ch1 = 0.0f, ch3 = 0.0f; + KI_TILED_CHANNEL ch0 = 0, ch1 = 0; + float ch3 = 0.0f; float4 hid = 0.0f; if ((F.flags & KIO_RESET) != 0) { if (active) { @@ -170,8 +184,8 @@ WG_KERNEL(k_in_tiled) const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); - const float h = ki_tiled_ch0(F, rx, ry); - ki_store_x0(P.x0, id, k, h, h, 0.0f, 0.0f); + const KI_TILED_CHANNEL h = ki_tiled_ch0(F, rx, ry, P.cpre); + KI_TILED_STORE(P.x0, id, k, h, h, 0.0f, 0.0f); } return; } @@ -247,11 +261,11 @@ WG_KERNEL(k_in_tiled) const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1); const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f)); const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f)); - ch0 = ki_tiled_ch0(F, rx, ry); + ch0 = ki_tiled_ch0(F, rx, ry, P.cpre); const float4 hs = kio_bilinear4(P.Hh, F.Bw, F.Bh, own_nx, own_ny); hid = valid ? hs : 0.0f; // channel 1 is channel 0's function of Yh, and Yh = Yc without a valid history: then it equals channel 0 - ch1 = valid ? ki_S(ki_T(F.lam * ki_L(own_hx, hdr))) : ch0; + ch1 = valid ? KI_TILED_S(ki_T(F.lam * ki_L(own_hx, hdr)), P.cpre) : ch0; const float d3 = valid ? D : 0.0f; ch3 = kf_finite(d3) ? d3 : 0.0f; } @@ -259,5 +273,5 @@ WG_KERNEL(k_in_tiled) if (!active) return; float hsum = (hs.x + hs.w) + (hs.y + hs.z); hsum = hsum * 0.25f; - ki_store_x0(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3); + KI_TILED_STORE(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3); } diff --git a/clean/k/runtime/k_s_breakpoints.h b/clean/k/runtime/k_s_breakpoints.h new file mode 100644 index 00000000..3ba702f7 --- /dev/null +++ b/clean/k/runtime/k_s_breakpoints.h @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include "../../runtime/fpconv.h" + +#include +#include +#include + +#if defined(_MSC_VER) && !defined(__clang__) +#pragma fp_contract(off) +#else +#pragma STDC FP_CONTRACT OFF +#endif + +namespace wgrt::ki_s_breakpoints { + +inline constexpr uint32_t kFirst = 0x3b4d2e1du, kLast = 0x3f800000u; +inline constexpr uint32_t kBucket = 4096, kBase = kFirst & ~(kBucket - 1); +using Table = std::array; + +inline float log32(float x) { + constexpr double kLn2 = 0.6931471805599453; + int exponent; + double m = std::frexp(static_cast(x), &exponent); + if (m < 0.7071067811865476) { + m *= 2.0; + --exponent; + } + const double t = (m - 1.0) / (m + 1.0); + const double t2 = t * t; + double p = 1.0 / 23.0; + for (int n = 21; n >= 1; n -= 2) p = p * t2 + 1.0 / n; + return static_cast(exponent + (2.0 / kLn2) * t * p); +} + +inline float exp32(float x) { + constexpr double kLn2 = 0.6931471805599453; + // The reduced exponent is integral. Either side of a tie is valid here. + const int n = static_cast(std::floor(static_cast(x) + 0.5)); + const double t = (static_cast(x) - n) * kLn2; + double p = 1.0; + for (int k = 14; k >= 1; --k) p = 1.0 + (t / k) * p; + return static_cast(std::ldexp(p, n)); +} + +inline uint16_t half_code(uint32_t word) { + const float y = log32(bits_f32(word)) * bits_f32(0x3ed55555u); + return f32_to_f16(std::fma(exp32(y), bits_f32(0x3f870a3du), bits_f32(0xbd6147aeu))); +} + +inline Table make_table() { + Table table; + for (size_t i = 0; i < table.size(); ++i) { + const uint32_t base = kBase + static_cast(i) * kBucket; + uint32_t lo = std::max(base, kFirst), hi = std::min(base + kBucket - 1, kLast); + const uint16_t first = half_code(lo); + uint32_t cut = kBucket; + if (first != half_code(hi)) { + // The exhaustive CPU proof verifies that each bucket is flat or + // increments by one once. Find the first code that increments. + while (lo < hi) { + const uint32_t mid = lo + (hi - lo) / 2; + if (half_code(mid) == first) + lo = mid + 1; + else + hi = mid; + } + cut = lo - base; + } + table[i] = first | (cut << 16); + } + return table; +} + +} // namespace wgrt::ki_s_breakpoints diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index b0d2ab7b..6fe2c24e 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -4,6 +4,7 @@ #include "runtime_k.h" #include "k_input_uniforms.h" #include "k_mix_probe.h" +#include "k_s_breakpoints.h" #include "k_smx_selfcheck.h" #include @@ -696,6 +697,12 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); + if ((c.backend == 2 || c.backend == 3) && tiled_input_ && in_exact_ && !approx_a1_) { + static const auto breakpoints = ki_s_breakpoints::make_table(); + if (!alloc(&s_breakpoints_, sizeof breakpoints) || + !dev_->upload(s_breakpoints_, breakpoints.data(), sizeof breakpoints)) + return dfail(); + } if (!dev_->sync()) return dfail(); // the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or // corrupt shader fails create here, not at the first frame @@ -1347,7 +1354,8 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { auto input_stage = [&]() { if (use_tiled) { KInParams local = ip; - local.cpre = local.cpre2 = 0; + local.cpre = s_breakpoints_; + local.cpre2 = 0; if (use_tiled_color) { KInColorParams fused = {local, ctap_, ctapT_}; if (use_uniforms) { diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index a1d3540a..57af19d2 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -167,6 +167,7 @@ class KRuntime { std::map net_bytes_; DevPtr Hc_[2] = {0, 0}, L_[2] = {0, 0}, Hh_[2] = {0, 0}; DevPtr color_ = 0, depth_ = 0, motion_ = 0, out_ = 0, cpre_ = 0, cpre2_ = 0, ctap_ = 0, ctapT_ = 0; + DevPtr s_breakpoints_ = 0; DevPtr exp2tab_ = 0, tables_ = 0, expo_ = 0, expo_partials_ = 0, expo_computed_ = 0, expo_sync_ = 0, xs_ = 0; DevPtr kexpo_ = 0; // k_expo_scalars state: E_cur, E_prev, E_aux of the last frame, E_cur kept for the next // frame state diff --git a/clean/k/tests/ki_s_breakpoints_cpu.cpp b/clean/k/tests/ki_s_breakpoints_cpu.cpp new file mode 100644 index 00000000..0f4f2399 --- /dev/null +++ b/clean/k/tests/ki_s_breakpoints_cpu.cpp @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: MIT +// Dump the production table for the exhaustive Python oracle. +#include "../runtime/k_s_breakpoints.h" +#include + +int main(int argc, char** argv) { + if (argc != 2) return 2; + auto* out = std::fopen(argv[1], "wb"); + if (!out) return 2; + const auto table = wgrt::ki_s_breakpoints::make_table(); + const size_t count = std::fwrite(table.data(), sizeof table[0], table.size(), out); + const int rc = std::fclose(out); + return count == table.size() && rc == 0 ? 0 : 2; +} diff --git a/clean/k/tests/prove_ki_s_breakpoints.py b/clean/k/tests/prove_ki_s_breakpoints.py new file mode 100644 index 00000000..a5a4ab32 --- /dev/null +++ b/clean/k/tests/prove_ki_s_breakpoints.py @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: MIT +"""Exhaustively check the production CPU table for the tiled input's packed FP16 transfer. + +Run as a module from the repository root with the path written by ki_s_breakpoints_cpu.cpp. +""" + +import argparse +from decimal import Decimal, localcontext +from pathlib import Path + +import numpy as np + +from clean.k.reference.fp import fma32 + +FIRST = 0x3B4D2E1D +LAST = 0x3F800000 +SHIFT = 12 +BUCKET = 1 << SHIFT +BASE = FIRST & ~(BUCKET - 1) +INVALID = 0x80000000 + + +def f32(word): + return np.uint32(word).view(np.float32) + + +def round_with_bounds(v, err, x, kind): + """Resolve FP32 midpoints with Decimal only when the fast interval straddles one.""" + mid = v.astype(np.float32) + lo = (v - err).astype(np.float32) + hi = (v + err).astype(np.float32) + uncertain = np.flatnonzero(lo != hi) + if len(uncertain): + with localcontext() as ctx: + ctx.prec = 80 + ln2 = Decimal(2).ln() + for i in uncertain: + d = Decimal.from_float(float(x[i])) + if kind == "log": + exact = d.ln() / ln2 + else: + exact = (d * ln2).exp() + # Pick the FP32 neighbour at the smaller Decimal distance. The + # wider Decimal error bound must separate the two distances. + choices = [mid[i], np.nextafter(mid[i], np.float32(-np.inf)), + np.nextafter(mid[i], np.float32(np.inf))] + closest = sorted((abs(exact - Decimal.from_float(float(c))), c) + for c in choices) + assert closest[1][0] - closest[0][0] > Decimal("1e-74") + mid[i] = closest[0][1] + return mid + + +def log32(x): + # m in [sqrt(0.5), sqrt(2)], so |t| < 0.172. The omitted terms of + # 2*atanh(t) are bounded by 2*0.172**25 / (25*(1-0.172**2)). + # Binary64 unit roundoff is 2**-53. Near m=1 the subtraction is exact + # (Sterbenz), and a 60*2**-53*|t| bound covers the subsequent operations. + # The exponent and final addition contribute at most 2**-49 here. + m, e = np.frexp(x.astype(np.float64)) + low = m < 0.7071067811865476 + m = np.where(low, 2.0 * m, m) + e = e - low + t = (m - 1.0) / (m + 1.0) + t2 = t * t + poly = 1.0 / 23.0 + for n in range(21, 0, -2): + poly = poly * t2 + 1.0 / n + v = e + (2.0 / 0.6931471805599453) * t * poly + err = 2.0**-46 + return round_with_bounds(v, err, x, "log") + + +def exp32(x): + # |t| <= ln(2)/2. The omitted exponential series tail is below + # 1.42*(ln(2)/2)**15/15!. Horner's method uses 28 binary64 rounded + # operations, with absolute intermediates below 1.5. Scaling is exact. + y = x.astype(np.float64) + n = np.rint(y).astype(np.int32) + t = (y - n) * 0.6931471805599453 + v = 1.0 + for k in range(14, 0, -1): + v = 1.0 + (t / k) * v + return round_with_bounds(np.ldexp(v, n), 2.0**-46, x, "exp") + + +def reference(zp): + y = log32(zp) * f32(0x3ED55555) + return fma32(exp32(y), f32(0x3F870A3D), f32(0xBD6147AE)).astype(np.float16).view(np.uint16) + + +def decode(entry, low): + return (entry & 0xffff) + (low >= ((entry >> 16) & 0x1fff)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("production_table", type=Path) + args = parser.parse_args() + entries = [] + first_cut = None + n_codes = 0 + for start in range(BASE, LAST + 1, BUCKET * 128): + stop = min(start + BUCKET * 128, (LAST | (BUCKET - 1)) + 1) + words = np.arange(start, stop, dtype=np.uint32).reshape(-1, BUCKET) + reachable = (words >= FIRST) & (words <= LAST) + want = reference(np.clip(words, FIRST, LAST).ravel().view(np.float32)).reshape(words.shape) + for row, ramp, used in zip(words, want, reachable): + change = np.flatnonzero(ramp != ramp[0]) + initial = int(ramp[0]) + cut = int(change[0]) if len(change) else BUCKET + entry = initial | (cut << 16) + low = row & (BUCKET - 1) + if not np.array_equal(decode(entry, low)[used], ramp[used]): + entry = INVALID + else: + if first_cut is None and cut < BUCKET and used[cut]: + mutant = initial | ((cut + 1) << 16) + assert decode(mutant, low)[cut] != ramp[cut] + first_cut = int(row[cut]) + entries.append(entry) + n_codes += int(reachable.sum()) + assert n_codes == LAST - FIRST + 1 + assert first_cut is not None + assert INVALID not in entries, "shader requires all buckets to be encodable" + # All half inputs include both signed zeros, denormals, infinities and NaNs. + # The shader retains the original clamp and linear branch verbatim. + with np.errstate(invalid="ignore"): + half = np.arange(65536, dtype=np.uint16).view(np.float16).astype(np.float32) + clamped = np.fmin(np.fmax(half, np.float32(0)), np.float32(1)) + nonlinear = clamped > f32(FIRST - 1) + hwords = clamped[nonlinear].view(np.uint32) + enc = np.asarray(entries, dtype=np.uint32)[(hwords - BASE) >> SHIFT] + got = np.empty(65536, dtype=np.uint16) + want = np.empty_like(got) + got[nonlinear] = decode(enc, hwords & (BUCKET - 1)) + want[nonlinear] = reference(clamped[nonlinear]) + linear = clamped[~nonlinear] + linear32 = linear * f32(0x414EB852) + # wg_f2h uses a biased FP32 addition for FP16 subnormals. + words = linear32.view(np.uint32) + mag = words & 0x7fffffff + normal = (mag + np.uint32(0xc8000fff) + ((mag >> 13) & 1)) >> 13 + subnormal = (mag.view(np.float32) + np.float32(0.5)).view(np.uint32) - 0x3f000000 + got[~nonlinear] = (words >> 16 & 0x8000) | np.where(mag < 0x38800000, subnormal, normal) + want[~nonlinear] = linear32.astype(np.float16).view(np.uint16) + assert np.array_equal(got, want) + stored = np.fromfile(args.production_table, dtype=" Date: Wed, 18 Nov 2026 17:27:02 -0500 Subject: [PATCH 2/6] Model K: verify the S table and retain the exact tiled fallback Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernel_sets.json | 12 +++++++ clean/k/dx12/kernels/k_in.hlsl | 16 ++++++---- clean/k/runtime/k_s_breakpoints.h | 11 +++++++ clean/k/runtime/runtime_k.cpp | 44 +++++++++++++++++++------- clean/k/runtime/runtime_k.h | 1 + clean/k/tests/ki_s_breakpoints_cpu.cpp | 6 +++- 6 files changed, 72 insertions(+), 18 deletions(-) diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index c15b4c92..b13ec118 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -639,6 +639,18 @@ [ "kx/k_in_tiled_uniform__a1", "k_io_exact.co" + ], + [ + "kxs/k_in_tiled", + "k_io_exact.co" + ], + [ + "kxs/k_in_tiled_direct", + "k_io_exact.co" + ], + [ + "kxs/k_in_tiled_uniform", + "k_io_exact.co" ] ] } diff --git a/clean/k/dx12/kernels/k_in.hlsl b/clean/k/dx12/kernels/k_in.hlsl index 0497c08d..9519f347 100644 --- a/clean/k/dx12/kernels/k_in.hlsl +++ b/clean/k/dx12/kernels/k_in.hlsl @@ -31,6 +31,10 @@ // @wg-kernel kx/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1 // @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 // @wg-kernel kx/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1 +// The kxs tag is the pre-breakpoint exact S fallback. Its HIP forms are identical to those under kx. +// @wg-kernel kxs/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1,KI_EXACT_S=1 +// @wg-kernel kxs/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 defines=KI_EXACT_S=1 +// @wg-kernel kxs/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1,KI_EXACT_S=1 // The __a1 forms: the opt-in close approximation a1 (runtime option k_approx=a1, never the default): log2 / 2^x / // divisions on the transcendental unit (hlsl/k_in_common.hlsli KI_HWMATH); not byte-identical to HIP. // @wg-kernel kx/k_in_tiled_uniform__a1 co=k_io_exact.co entry=k_in_tiled_uniform block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1,KI_HWMATH=1 @@ -65,19 +69,19 @@ WG_ARGS_END float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); } -#if !KI_HWMATH +#if !KI_HWMATH && !defined(KI_EXACT_S) #define KI_TILED_PACKED_S 1 #endif #include "k_in_common.hlsli" -#if KI_HWMATH -#define KI_TILED_CHANNEL float -#define KI_TILED_S(z, table) ki_S(z) -#define KI_TILED_STORE ki_store_x0 -#else +#if defined(KI_TILED_PACKED_S) #define KI_TILED_CHANNEL uint #define KI_TILED_S(z, table) ki_tiled_S_bits(z, table) #define KI_TILED_STORE ki_store_tiled_x0 +#else +#define KI_TILED_CHANNEL float +#define KI_TILED_S(z, table) ki_S(z) +#define KI_TILED_STORE ki_store_x0 #endif struct KiTiledTap { diff --git a/clean/k/runtime/k_s_breakpoints.h b/clean/k/runtime/k_s_breakpoints.h index 3ba702f7..c82f49bb 100644 --- a/clean/k/runtime/k_s_breakpoints.h +++ b/clean/k/runtime/k_s_breakpoints.h @@ -19,6 +19,17 @@ inline constexpr uint32_t kFirst = 0x3b4d2e1du, kLast = 0x3f800000u; inline constexpr uint32_t kBucket = 4096, kBase = kFirst & ~(kBucket - 1); using Table = std::array; +// FNV-1a of the little-endian table verified by prove_ki_s_breakpoints.py. +inline constexpr uint64_t kProvenDigest = 0xfe032345814078feULL; + +inline uint64_t table_digest(const Table& table) { + uint64_t hash = 0xcbf29ce484222325ULL; + for (uint32_t entry : table) + for (unsigned shift = 0; shift < 32; shift += 8) + hash = (hash ^ ((entry >> shift) & 0xffu)) * 0x100000001b3ULL; + return hash; +} + inline float log32(float x) { constexpr double kLn2 = 0.6931471805599453; int exponent; diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 6fe2c24e..fa31f057 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -591,6 +591,25 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e #else if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only"); #endif + const std::string s_test = option(c, "k_s_breakpoints_test"); + bool force_s_mismatch = false; +#ifdef WG_DEV_DIAGNOSTICS + if (!s_test.empty() && s_test != "mismatch") return fail("model K: k_s_breakpoints_test must be mismatch"); + force_s_mismatch = s_test == "mismatch"; +#else + if (!s_test.empty()) return fail("model K: k_s_breakpoints_test is a test hook of developer builds only"); +#endif + const ki_s_breakpoints::Table* s_table = nullptr; + if ((c.backend == 2 || c.backend == 3) && tiled_input_ && in_exact_ && !approx_a1_) { + static const auto table = ki_s_breakpoints::make_table(); + static const auto digest = ki_s_breakpoints::table_digest(table); + s_breakpoints_fallback_ = force_s_mismatch || digest != ki_s_breakpoints::kProvenDigest; + if (s_breakpoints_fallback_) + notes_ += (notes_.empty() ? "" : "\n") + + std::string("model K: S breakpoint digest mismatch; using the pre-breakpoint exact tiled S path"); + else + s_table = &table; + } // The native D3D12 device (backend 3: the D3D12 executor, or k_bench.py's own-buffer timing) runs the exact default // path only, and only when every kernel this context can launch has a port in /dx12/manifest.json; both are // checked here, before the model is loaded, so a refused context costs nothing more (for model K the context ends and @@ -634,6 +653,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e // k_in_exact=1: the same input kernels with exact arithmetic, addressed as "kx/" if (in_exact_ && !dev_->load_module_tagged(join(build_dir_, "k_io_exact.co"), "kx")) return fail("model K: " + dev_->err()); + // The second tag selects the pre-breakpoint shader ports and resolves to the same exact kernels on HIP. + if (s_breakpoints_fallback_ && !dev_->load_module_tagged(join(build_dir_, "k_io_exact.co"), "kxs")) + return fail("model K: the pre-breakpoint exact tiled S path is unavailable: " + dev_->err()); // buffers for (auto& kv : blobs_) { DevPtr p; @@ -697,10 +719,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e std::vector tab((size_t)8 * kFilterEntries * kFilterValues); for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); - if ((c.backend == 2 || c.backend == 3) && tiled_input_ && in_exact_ && !approx_a1_) { - static const auto breakpoints = ki_s_breakpoints::make_table(); - if (!alloc(&s_breakpoints_, sizeof breakpoints) || - !dev_->upload(s_breakpoints_, breakpoints.data(), sizeof breakpoints)) + if (s_table) { + if (!alloc(&s_breakpoints_, sizeof *s_table) || + !dev_->upload(s_breakpoints_, s_table->data(), sizeof *s_table)) return dfail(); } if (!dev_->sync()) return dfail(); @@ -1327,7 +1348,8 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { auto kname = [&](const char* n) -> const char* { const bool io_kernel = std::strncmp(n, "k_in_", 5) == 0 || std::strncmp(n, "k_out_", 6) == 0; if (in_exact_ && std::strncmp(n, "k_in_", 5) == 0) { // input-stage kernels from k_io_exact.co - kname_buf = std::string("kx/") + n; + const bool fallback = s_breakpoints_fallback_ && std::strncmp(n, "k_in_tiled", 10) == 0; + kname_buf = std::string(fallback ? "kxs/" : "kx/") + n; } else if (fast_) { kname_buf = std::string(n) + "_f"; } else { @@ -1514,11 +1536,11 @@ std::vector KRuntime::d3d12_kernels() const { v.push_back("k_expo_scalars"); if (native_direct_) v.push_back("k_expo_scalars_direct"); // exposure from the game's texture (mode 1) v.push_back("k_out_color" + d); - if (!native_direct_) v.push_back("kx/k_in_tiled_uniform"); // (no direct form: the direct frame uses k_in_tiled_direct) - v.push_back("kx/k_in_tiled" + d); // input_uniforms_safe refusing the host uniforms - if (approx_a1_) { // k_approx=a1 (opt-in) - if (!native_direct_) v.push_back("kx/k_in_tiled_uniform__a1"); - v.push_back("kx/k_in_tiled" + d + "__a1"); + if (tiled_input_) { + const std::string tag = s_breakpoints_fallback_ ? "kxs/" : "kx/"; + const std::string suffix = approx_a1_ ? "__a1" : ""; + if (!native_direct_ && input_uniforms_) v.push_back(tag + "k_in_tiled_uniform" + suffix); + v.push_back(tag + "k_in_tiled" + d + suffix); // input_uniforms_safe refusing the host uniforms } if (approx_a2_) { // k_approx=a2 (opt-in) v.push_back("k_out_color" + d + "__a2"); @@ -1576,7 +1598,7 @@ std::string KRuntime::d3d12_missing() const { for (const std::string& k : d3d12_kernels()) { auto it = co.find(k); // the code object the port stands for must be one this runtime loads (else the device refuses it at launch) - const bool tagged = k.rfind("kx/", 0) == 0; + const bool tagged = k.rfind("kx/", 0) == 0 || k.rfind("kxs/", 0) == 0; const bool co_ok = it != co.end() && (tagged ? it->second == "k_io_exact.co" : (it->second == "k_block.co" || it->second == "k_block_wide.co" || diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index 57af19d2..7fa1dff6 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -147,6 +147,7 @@ class KRuntime { std::string f8why_; // stages that keep their FP16 kernel, and why std::vector f8stages_; // runtime option fp8_stages (default e0, e1, d1, d0) bool tiled_input_ = false, tiled_last_ = false, tiled_color_last_ = false, input_uniforms_last_ = false; + bool s_breakpoints_fallback_ = false; uint64_t tiled_launches_ = 0, tiled_color_launches_ = 0, input_uniform_launches_ = 0; uint64_t legacy_color_launches_ = 0, legacy_base_launches_ = 0; bool input_uniforms_ = true; // runtime option k_in_uniforms (0: always k_in_tiled, a port-check diagnostic) diff --git a/clean/k/tests/ki_s_breakpoints_cpu.cpp b/clean/k/tests/ki_s_breakpoints_cpu.cpp index 0f4f2399..bae58b2f 100644 --- a/clean/k/tests/ki_s_breakpoints_cpu.cpp +++ b/clean/k/tests/ki_s_breakpoints_cpu.cpp @@ -5,9 +5,13 @@ int main(int argc, char** argv) { if (argc != 2) return 2; + const auto table = wgrt::ki_s_breakpoints::make_table(); + if (wgrt::ki_s_breakpoints::table_digest(table) != wgrt::ki_s_breakpoints::kProvenDigest) { + std::fputs("Model K S breakpoints do not match the proven digest\n", stderr); + return 1; + } auto* out = std::fopen(argv[1], "wb"); if (!out) return 2; - const auto table = wgrt::ki_s_breakpoints::make_table(); const size_t count = std::fwrite(table.data(), sizeof table[0], table.size(), out); const int rc = std::fclose(out); return count == table.size() && rc == 0 ? 0 : 2; From 51fa5f52b756105bee7d4095fa1e4e09406e1b91 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:27:25 -0500 Subject: [PATCH 3/6] Model K: run the S breakpoint proof in unittest Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/tests/test_k_s_breakpoints.py | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 clean/k/tests/test_k_s_breakpoints.py diff --git a/clean/k/tests/test_k_s_breakpoints.py b/clean/k/tests/test_k_s_breakpoints.py new file mode 100644 index 00000000..d5da9e7a --- /dev/null +++ b/clean/k/tests/test_k_s_breakpoints.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: MIT +"""Prove the production S transfer table with a CPU-only dump.""" +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest + + +class TestSBreakpoints(unittest.TestCase): + def test_production_table(self): + with tempfile.TemporaryDirectory(prefix="wg_k_s_breakpoints_") as tmp: + supplied = os.environ.get("WG_K_S_BREAKPOINTS_DUMP") + if supplied: + dump = Path(supplied).resolve() + if not dump.is_file(): + self.skipTest("WG_K_S_BREAKPOINTS_DUMP does not name a table dump") + else: + cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++") + if not cxx or not shutil.which(cxx): + self.skipTest("no host C++ compiler or WG_K_S_BREAKPOINTS_DUMP") + probe = Path(tmp) / ("ki_s_breakpoints_cpu.exe" if sys.platform == "win32" else "ki_s_breakpoints_cpu") + source = Path(__file__).with_name("ki_s_breakpoints_cpu.cpp") + dump = Path(tmp) / "table.bin" + self.run_checked(cxx, "-std=c++17", "-O2", "-ffp-contract=off", source, "-o", probe) + self.run_checked(probe, dump) + repo = Path(__file__).resolve().parents[3] + self.run_checked(sys.executable, "-m", "clean.k.tests.prove_ki_s_breakpoints", dump, cwd=repo) + + def run_checked(self, *args, cwd=None): + run = subprocess.run(args, cwd=cwd, capture_output=True, text=True) + self.assertEqual(run.returncode, 0, run.stdout + run.stderr) From e52309d6c24f64bdce1075c3a1069fe777f130dd Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:27:41 -0500 Subject: [PATCH 4/6] Model K: scope breakpoint generator FP contraction controls Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/runtime/k_s_breakpoints.h | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/clean/k/runtime/k_s_breakpoints.h b/clean/k/runtime/k_s_breakpoints.h index c82f49bb..11b7bee9 100644 --- a/clean/k/runtime/k_s_breakpoints.h +++ b/clean/k/runtime/k_s_breakpoints.h @@ -7,12 +7,6 @@ #include #include -#if defined(_MSC_VER) && !defined(__clang__) -#pragma fp_contract(off) -#else -#pragma STDC FP_CONTRACT OFF -#endif - namespace wgrt::ki_s_breakpoints { inline constexpr uint32_t kFirst = 0x3b4d2e1du, kLast = 0x3f800000u; @@ -30,7 +24,18 @@ inline uint64_t table_digest(const Table& table) { return hash; } +#if defined(_MSC_VER) && !defined(__clang__) +#pragma float_control(precise, on, push) +#pragma fp_contract(off) +#elif defined(__GNUC__) && !defined(__clang__) +#pragma GCC push_options +#pragma GCC optimize ("fp-contract=off") +#endif + inline float log32(float x) { +#if defined(__clang__) +#pragma STDC FP_CONTRACT OFF +#endif constexpr double kLn2 = 0.6931471805599453; int exponent; double m = std::frexp(static_cast(x), &exponent); @@ -46,6 +51,9 @@ inline float log32(float x) { } inline float exp32(float x) { +#if defined(__clang__) +#pragma STDC FP_CONTRACT OFF +#endif constexpr double kLn2 = 0.6931471805599453; // The reduced exponent is integral. Either side of a tie is valid here. const int n = static_cast(std::floor(static_cast(x) + 0.5)); @@ -55,6 +63,12 @@ inline float exp32(float x) { return static_cast(std::ldexp(p, n)); } +#if defined(_MSC_VER) && !defined(__clang__) +#pragma float_control(pop) +#elif defined(__GNUC__) && !defined(__clang__) +#pragma GCC pop_options +#endif + inline uint16_t half_code(uint32_t word) { const float y = log32(bits_f32(word)) * bits_f32(0x3ed55555u); return f32_to_f16(std::fma(exp32(y), bits_f32(0x3f870a3du), bits_f32(0xbd6147aeu))); From f4a56d3e28dc70b61e5c95dd18369fec36dda83d Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:27:51 -0500 Subject: [PATCH 5/6] Model K: refuse tiled frames missing the S breakpoint buffer Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/runtime/runtime_k.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index fa31f057..59c76a57 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -1375,6 +1375,8 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) { const bool use_uniforms = use_tiled && !native_direct_ && input_uniforms_ && input_uniforms_safe(fin, &input_host); auto input_stage = [&]() { if (use_tiled) { + if (in_exact_ && !approx_a1_ && cfg_.backend >= 2 && !s_breakpoints_fallback_ && !s_breakpoints_) + return fail("model K: exact tiled input would get a null S breakpoint table (arena offset 0); the frame is refused"); KInParams local = ip; local.cpre = s_breakpoints_; local.cpre2 = 0; From ed95243825a8798173fd90613e9b68a42cb7569e Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:27:58 -0500 Subject: [PATCH 6/6] Model K: remove the dead approximate S branch in the packed helper Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/k/dx12/hlsl/k_in_common.hlsli | 4 ---- 1 file changed, 4 deletions(-) diff --git a/clean/k/dx12/hlsl/k_in_common.hlsli b/clean/k/dx12/hlsl/k_in_common.hlsli index c62cdb79..13857c7a 100644 --- a/clean/k/dx12/hlsl/k_in_common.hlsli +++ b/clean/k/dx12/hlsl/k_in_common.hlsli @@ -59,9 +59,6 @@ float ki_S(float z) #if defined(KI_TILED_PACKED_S) uint ki_tiled_S_bits(float z, uint table) { -#if KI_HWMATH - return kh_round_bits(ki_S(z)); -#else const float zp = kf_min(kf_max(z, 0.0f), 1.0f); if (!(zp > kf_bits(0x3B4D2E1Cu))) return kh_round_bits(kf_bits(0x414EB852u) * zp); const uint u = asuint(zp); @@ -69,7 +66,6 @@ uint ki_tiled_S_bits(float z, uint table) // (clean.k.tests.prove_ki_s_breakpoints). The cut is 4,096 for flat buckets. const uint e = wg_ld32(table, ((u - 0x3B4D2000u) >> 12u) * 4u); return (e & 0xffffu) + ((u & 0xfffu) >= (e >> 16u) ? 1u : 0u); -#endif } #endif // z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart