Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -639,6 +639,18 @@
[
"kx/k_in_tiled_uniform__a1",
"k_io_exact.co"
],
[
"kxs/k_in_tiled",
"k_io_exact.co"
],
[
"kxs/k_in_tiled_direct",
"k_io_exact.co"
],
[
"kxs/k_in_tiled_uniform",
"k_io_exact.co"
]
]
}
20 changes: 20 additions & 0 deletions clean/k/dx12/hlsl/k_in_common.hlsli
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,18 @@ float ki_S(float z)
const float p = ki_exp2(ki_log2(zp) * kf_bits(0x3ED55555u)); // log2 and 2^x each rounded once
return kf_fma(p, kf_bits(0x3F870A3Du), kf_bits(0xBD6147AEu));
}
#if defined(KI_TILED_PACKED_S)
uint ki_tiled_S_bits(float z, uint table)
{
const float zp = kf_min(kf_max(z, 0.0f), 1.0f);
if (!(zp > kf_bits(0x3B4D2E1Cu))) return kh_round_bits(kf_bits(0x414EB852u) * zp);
const uint u = asuint(zp);
// Every reachable 4,096-code bucket is flat or makes one +1 transition
// (clean.k.tests.prove_ki_s_breakpoints). The cut is 4,096 for flat buckets.
const uint e = wg_ld32(table, ((u - 0x3B4D2000u) >> 12u) * 4u);
return (e & 0xffffu) + ((u & 0xfffu) >= (e >> 16u) ? 1u : 0u);
}
#endif
// z is often a product (lambda L) that HIP rounds before z + 1. No kf_opq: the gfx1201 ISA keeps the product apart
// (G1's precaution link, dropped in speed round 1 after the ISA check: same v_mul / v_fma counts without it)
float ki_T(float z)
Expand Down Expand Up @@ -132,3 +144,11 @@ void ki_store_x0(uint x0, int id, int k, float c0, float c1, float c2, float c3)
wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u,
uint2(kh_round_bits(c0) | (kh_round_bits(c1) << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16)));
}

#if defined(KI_TILED_PACKED_S)
void ki_store_tiled_x0(uint x0, int id, int k, uint c0, uint c1, float c2, float c3)
{
wg_st64(x0, ((uint)id * 16u + 4u * (uint)k) * 2u,
uint2(c0 | (c1 << 16), kh_round_bits(c2) | (kh_round_bits(c3) << 16)));
}
#endif
36 changes: 27 additions & 9 deletions clean/k/dx12/kernels/k_in.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,14 @@
// motion from the game's textures t0 / t1 / t2, k_frame.hlsli kio_texel / kio_motion /
// kio_depth; WG_DIRECT_TEXTURES); the uniform form has no direct form (the native executor has
// no host frame values)
// The KInParams pointers cpre / cpre2 are null in the tiled launches (runtime_k.cpp), as in HIP unused here.
// The exact D3D12 tiled launches use KInParams.cpre for the packed S transfer table.
// @wg-kernel kx/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1
// @wg-kernel kx/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5
// @wg-kernel kx/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1
// The kxs tag is the pre-breakpoint exact S fallback. Its HIP forms are identical to those under kx.
// @wg-kernel kxs/k_in_tiled_uniform co=k_io_exact.co block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1,KI_EXACT_S=1
// @wg-kernel kxs/k_in_tiled co=k_io_exact.co block=256 wave=64 args=s200:p4.b128.p5 defines=KI_EXACT_S=1
// @wg-kernel kxs/k_in_tiled_direct co=k_io_exact.co entry=k_in_tiled block=256 wave=64 args=s200:p4.b128.p5 rs=direct defines=WG_DIRECT_TEXTURES=1,KI_EXACT_S=1
// The __a1 forms: the opt-in close approximation a1 (runtime option k_approx=a1, never the default): log2 / 2^x /
// divisions on the transcendental unit (hlsl/k_in_common.hlsli KI_HWMATH); not byte-identical to HIP.
// @wg-kernel kx/k_in_tiled_uniform__a1 co=k_io_exact.co entry=k_in_tiled_uniform block=256 wave=64 args=s248:p4.b128.p5.b48 defines=K_UNIFORM=1,KI_HWMATH=1
Expand Down Expand Up @@ -65,8 +69,21 @@ WG_ARGS_END

float3 kf_opq3(float3 x) { return asfloat(asuint(x) ^ kf_opq_z_); }

#if !KI_HWMATH && !defined(KI_EXACT_S)
#define KI_TILED_PACKED_S 1
#endif
#include "k_in_common.hlsli"

#if defined(KI_TILED_PACKED_S)
#define KI_TILED_CHANNEL uint
#define KI_TILED_S(z, table) ki_tiled_S_bits(z, table)
#define KI_TILED_STORE ki_store_tiled_x0
#else
#define KI_TILED_CHANNEL float
#define KI_TILED_S(z, table) ki_S(z)
#define KI_TILED_STORE ki_store_x0
#endif

struct KiTiledTap {
float4 cv; // Y, Co, Cg, phi(Y)
float om; // HDR weight max(1 / (Y + 1), 1e-4); 1 for LDR (A = fma(1, v, A) is HIP's A + v, one rounding of the sum)
Expand All @@ -83,12 +100,12 @@ KiTiledTap ki_tiled_tap_from_raw(KFrameIn F, float4 c)
return r;
}
KiTiledTap ki_tiled_tap(KFrameIn F, int rx, int ry) { return ki_tiled_tap_from_raw(F, kio_color(F, rx, ry)); }
float ki_tiled_ch0(KFrameIn F, int rx, int ry)
KI_TILED_CHANNEL ki_tiled_ch0(KFrameIn F, int rx, int ry, uint table)
{
const float3 rgb = ki_rgb(F, kio_color(F, rx, ry));
const bool hdr = (F.flags & KIO_HDR) != 0;
const float Yc = kio_luma_fma(rgb.x, rgb.y, rgb.z);
return ki_S(ki_T(F.lam * ki_L(Yc, hdr)));
return KI_TILED_S(ki_T(F.lam * ki_L(Yc, hdr)), table);
}
// HIP ki_tiled_transpose: lane bit 0 / bit 1 swaps through ds_bpermute (wg_xor = WaveReadLaneAt(lane ^ d))
float4 ki_tiled_transpose(float4 v, int lane)
Expand Down Expand Up @@ -162,16 +179,17 @@ WG_KERNEL(k_in_tiled)
const int id = by * F.Bw + bx;
const bool hdr = (F.flags & KIO_HDR) != 0;
const float c1e4 = kf_bits(0x38D1B717u);
float ch0 = 0.0f, ch1 = 0.0f, ch3 = 0.0f;
KI_TILED_CHANNEL ch0 = 0, ch1 = 0;
float ch3 = 0.0f;
float4 hid = 0.0f;
if ((F.flags & KIO_RESET) != 0) {
if (active) {
const int u = 2 * bx + (k & 1), v = 2 * by + (k >> 1);
const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1);
const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f));
const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f));
const float h = ki_tiled_ch0(F, rx, ry);
ki_store_x0(P.x0, id, k, h, h, 0.0f, 0.0f);
const KI_TILED_CHANNEL h = ki_tiled_ch0(F, rx, ry, P.cpre);
KI_TILED_STORE(P.x0, id, k, h, h, 0.0f, 0.0f);
}
return;
}
Expand Down Expand Up @@ -247,17 +265,17 @@ WG_KERNEL(k_in_tiled)
const int ofx = 2 * u + 1 - (v & 1), ofy = 2 * v + (u & 1);
const int rx = (int)kf_R(F.jx + kf_fma((float)ofx + 0.5f, hk.a_x, -0.5f));
const int ry = (int)kf_R(F.jy + kf_fma((float)ofy + 0.5f, hk.a_y, -0.5f));
ch0 = ki_tiled_ch0(F, rx, ry);
ch0 = ki_tiled_ch0(F, rx, ry, P.cpre);
const float4 hs = kio_bilinear4(P.Hh, F.Bw, F.Bh, own_nx, own_ny);
hid = valid ? hs : 0.0f;
// channel 1 is channel 0's function of Yh, and Yh = Yc without a valid history: then it equals channel 0
ch1 = valid ? ki_S(ki_T(F.lam * ki_L(own_hx, hdr))) : ch0;
ch1 = valid ? KI_TILED_S(ki_T(F.lam * ki_L(own_hx, hdr)), P.cpre) : ch0;
const float d3 = valid ? D : 0.0f;
ch3 = kf_finite(d3) ? d3 : 0.0f;
}
const float4 hs = ki_tiled_transpose(hid, lane);
if (!active) return;
float hsum = (hs.x + hs.w) + (hs.y + hs.z);
hsum = hsum * 0.25f;
ki_store_x0(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3);
KI_TILED_STORE(P.x0, id, k, ch0, ch1, kf_finite(hsum) ? hsum : 0.0f, ch3);
}
101 changes: 101 additions & 0 deletions clean/k/runtime/k_s_breakpoints.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
// SPDX-License-Identifier: MIT
#pragma once

#include "../../runtime/fpconv.h"

#include <algorithm>
#include <array>
#include <cmath>

namespace wgrt::ki_s_breakpoints {

inline constexpr uint32_t kFirst = 0x3b4d2e1du, kLast = 0x3f800000u;
inline constexpr uint32_t kBucket = 4096, kBase = kFirst & ~(kBucket - 1);
using Table = std::array<uint32_t, (kLast - kBase) / kBucket + 1>;

// FNV-1a of the little-endian table verified by prove_ki_s_breakpoints.py.
inline constexpr uint64_t kProvenDigest = 0xfe032345814078feULL;

inline uint64_t table_digest(const Table& table) {
uint64_t hash = 0xcbf29ce484222325ULL;
for (uint32_t entry : table)
for (unsigned shift = 0; shift < 32; shift += 8)
hash = (hash ^ ((entry >> shift) & 0xffu)) * 0x100000001b3ULL;
return hash;
}

#if defined(_MSC_VER) && !defined(__clang__)
#pragma float_control(precise, on, push)
#pragma fp_contract(off)
#elif defined(__GNUC__) && !defined(__clang__)
#pragma GCC push_options
#pragma GCC optimize ("fp-contract=off")
#endif

inline float log32(float x) {
#if defined(__clang__)
#pragma STDC FP_CONTRACT OFF
#endif
constexpr double kLn2 = 0.6931471805599453;
int exponent;
double m = std::frexp(static_cast<double>(x), &exponent);
if (m < 0.7071067811865476) {
m *= 2.0;
--exponent;
}
const double t = (m - 1.0) / (m + 1.0);
const double t2 = t * t;
double p = 1.0 / 23.0;
for (int n = 21; n >= 1; n -= 2) p = p * t2 + 1.0 / n;
return static_cast<float>(exponent + (2.0 / kLn2) * t * p);
}

inline float exp32(float x) {
#if defined(__clang__)
#pragma STDC FP_CONTRACT OFF
#endif
constexpr double kLn2 = 0.6931471805599453;
// The reduced exponent is integral. Either side of a tie is valid here.
const int n = static_cast<int>(std::floor(static_cast<double>(x) + 0.5));
const double t = (static_cast<double>(x) - n) * kLn2;
double p = 1.0;
for (int k = 14; k >= 1; --k) p = 1.0 + (t / k) * p;
return static_cast<float>(std::ldexp(p, n));
}

#if defined(_MSC_VER) && !defined(__clang__)
#pragma float_control(pop)
#elif defined(__GNUC__) && !defined(__clang__)
#pragma GCC pop_options
#endif

inline uint16_t half_code(uint32_t word) {
const float y = log32(bits_f32(word)) * bits_f32(0x3ed55555u);
return f32_to_f16(std::fma(exp32(y), bits_f32(0x3f870a3du), bits_f32(0xbd6147aeu)));
}

inline Table make_table() {
Table table;
for (size_t i = 0; i < table.size(); ++i) {
const uint32_t base = kBase + static_cast<uint32_t>(i) * kBucket;
uint32_t lo = std::max(base, kFirst), hi = std::min(base + kBucket - 1, kLast);
const uint16_t first = half_code(lo);
uint32_t cut = kBucket;
if (first != half_code(hi)) {
// The exhaustive CPU proof verifies that each bucket is flat or
// increments by one once. Find the first code that increments.
while (lo < hi) {
const uint32_t mid = lo + (hi - lo) / 2;
if (half_code(mid) == first)
lo = mid + 1;
else
hi = mid;
}
cut = lo - base;
}
table[i] = first | (cut << 16);
}
return table;
}

} // namespace wgrt::ki_s_breakpoints
48 changes: 40 additions & 8 deletions clean/k/runtime/runtime_k.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "runtime_k.h"
#include "k_input_uniforms.h"
#include "k_mix_probe.h"
#include "k_s_breakpoints.h"
#include "k_smx_selfcheck.h"

#include <algorithm>
Expand Down Expand Up @@ -590,6 +591,25 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e
#else
if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only");
#endif
const std::string s_test = option(c, "k_s_breakpoints_test");
bool force_s_mismatch = false;
#ifdef WG_DEV_DIAGNOSTICS
if (!s_test.empty() && s_test != "mismatch") return fail("model K: k_s_breakpoints_test must be mismatch");
force_s_mismatch = s_test == "mismatch";
#else
if (!s_test.empty()) return fail("model K: k_s_breakpoints_test is a test hook of developer builds only");
#endif
const ki_s_breakpoints::Table* s_table = nullptr;
if ((c.backend == 2 || c.backend == 3) && tiled_input_ && in_exact_ && !approx_a1_) {
static const auto table = ki_s_breakpoints::make_table();
static const auto digest = ki_s_breakpoints::table_digest(table);
s_breakpoints_fallback_ = force_s_mismatch || digest != ki_s_breakpoints::kProvenDigest;
if (s_breakpoints_fallback_)
notes_ += (notes_.empty() ? "" : "\n") +
std::string("model K: S breakpoint digest mismatch; using the pre-breakpoint exact tiled S path");
else
s_table = &table;
}
// The native D3D12 device (backend 3: the D3D12 executor, or k_bench.py's own-buffer timing) runs the exact default
// path only, and only when every kernel this context can launch has a port in <build>/dx12/manifest.json; both are
// checked here, before the model is loaded, so a refused context costs nothing more (for model K the context ends and
Expand Down Expand Up @@ -633,6 +653,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e
// k_in_exact=1: the same input kernels with exact arithmetic, addressed as "kx/<name>"
if (in_exact_ && !dev_->load_module_tagged(join(build_dir_, "k_io_exact.co"), "kx"))
return fail("model K: " + dev_->err());
// The second tag selects the pre-breakpoint shader ports and resolves to the same exact kernels on HIP.
if (s_breakpoints_fallback_ && !dev_->load_module_tagged(join(build_dir_, "k_io_exact.co"), "kxs"))
return fail("model K: the pre-breakpoint exact tiled S path is unavailable: " + dev_->err());
// buffers
for (auto& kv : blobs_) {
DevPtr p;
Expand Down Expand Up @@ -696,6 +719,11 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e
std::vector<uint16_t> tab((size_t)8 * kFilterEntries * kFilterValues);
for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues);
if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail();
if (s_table) {
if (!alloc(&s_breakpoints_, sizeof *s_table) ||
!dev_->upload(s_breakpoints_, s_table->data(), sizeof *s_table))
return dfail();
}
if (!dev_->sync()) return dfail();
// the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or
// corrupt shader fails create here, not at the first frame
Expand Down Expand Up @@ -1320,7 +1348,8 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) {
auto kname = [&](const char* n) -> const char* {
const bool io_kernel = std::strncmp(n, "k_in_", 5) == 0 || std::strncmp(n, "k_out_", 6) == 0;
if (in_exact_ && std::strncmp(n, "k_in_", 5) == 0) { // input-stage kernels from k_io_exact.co
kname_buf = std::string("kx/") + n;
const bool fallback = s_breakpoints_fallback_ && std::strncmp(n, "k_in_tiled", 10) == 0;
kname_buf = std::string(fallback ? "kxs/" : "kx/") + n;
} else if (fast_) {
kname_buf = std::string(n) + "_f";
} else {
Expand All @@ -1346,8 +1375,11 @@ bool KRuntime::frame(const WgRtFrame& fr, const FrameIo* io) {
const bool use_uniforms = use_tiled && !native_direct_ && input_uniforms_ && input_uniforms_safe(fin, &input_host);
auto input_stage = [&]() {
if (use_tiled) {
if (in_exact_ && !approx_a1_ && cfg_.backend >= 2 && !s_breakpoints_fallback_ && !s_breakpoints_)
return fail("model K: exact tiled input would get a null S breakpoint table (arena offset 0); the frame is refused");
KInParams local = ip;
local.cpre = local.cpre2 = 0;
local.cpre = s_breakpoints_;
local.cpre2 = 0;
if (use_tiled_color) {
KInColorParams fused = {local, ctap_, ctapT_};
if (use_uniforms) {
Expand Down Expand Up @@ -1506,11 +1538,11 @@ std::vector<std::string> KRuntime::d3d12_kernels() const {
v.push_back("k_expo_scalars");
if (native_direct_) v.push_back("k_expo_scalars_direct"); // exposure from the game's texture (mode 1)
v.push_back("k_out_color" + d);
if (!native_direct_) v.push_back("kx/k_in_tiled_uniform"); // (no direct form: the direct frame uses k_in_tiled_direct)
v.push_back("kx/k_in_tiled" + d); // input_uniforms_safe refusing the host uniforms
if (approx_a1_) { // k_approx=a1 (opt-in)
if (!native_direct_) v.push_back("kx/k_in_tiled_uniform__a1");
v.push_back("kx/k_in_tiled" + d + "__a1");
if (tiled_input_) {
const std::string tag = s_breakpoints_fallback_ ? "kxs/" : "kx/";
const std::string suffix = approx_a1_ ? "__a1" : "";
if (!native_direct_ && input_uniforms_) v.push_back(tag + "k_in_tiled_uniform" + suffix);
v.push_back(tag + "k_in_tiled" + d + suffix); // input_uniforms_safe refusing the host uniforms
}
if (approx_a2_) { // k_approx=a2 (opt-in)
v.push_back("k_out_color" + d + "__a2");
Expand Down Expand Up @@ -1568,7 +1600,7 @@ std::string KRuntime::d3d12_missing() const {
for (const std::string& k : d3d12_kernels()) {
auto it = co.find(k);
// the code object the port stands for must be one this runtime loads (else the device refuses it at launch)
const bool tagged = k.rfind("kx/", 0) == 0;
const bool tagged = k.rfind("kx/", 0) == 0 || k.rfind("kxs/", 0) == 0;
const bool co_ok = it != co.end() &&
(tagged ? it->second == "k_io_exact.co"
: (it->second == "k_block.co" || it->second == "k_block_wide.co" ||
Expand Down
2 changes: 2 additions & 0 deletions clean/k/runtime/runtime_k.h
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,7 @@ class KRuntime {
std::string f8why_; // stages that keep their FP16 kernel, and why
std::vector<std::string> f8stages_; // runtime option fp8_stages (default e0, e1, d1, d0)
bool tiled_input_ = false, tiled_last_ = false, tiled_color_last_ = false, input_uniforms_last_ = false;
bool s_breakpoints_fallback_ = false;
uint64_t tiled_launches_ = 0, tiled_color_launches_ = 0, input_uniform_launches_ = 0;
uint64_t legacy_color_launches_ = 0, legacy_base_launches_ = 0;
bool input_uniforms_ = true; // runtime option k_in_uniforms (0: always k_in_tiled, a port-check diagnostic)
Expand All @@ -167,6 +168,7 @@ class KRuntime {
std::map<std::string, size_t> net_bytes_;
DevPtr Hc_[2] = {0, 0}, L_[2] = {0, 0}, Hh_[2] = {0, 0};
DevPtr color_ = 0, depth_ = 0, motion_ = 0, out_ = 0, cpre_ = 0, cpre2_ = 0, ctap_ = 0, ctapT_ = 0;
DevPtr s_breakpoints_ = 0;
DevPtr exp2tab_ = 0, tables_ = 0, expo_ = 0, expo_partials_ = 0, expo_computed_ = 0, expo_sync_ = 0, xs_ = 0;
DevPtr kexpo_ = 0; // k_expo_scalars state: E_cur, E_prev, E_aux of the last frame, E_cur kept for the next
// frame state
Expand Down
18 changes: 18 additions & 0 deletions clean/k/tests/ki_s_breakpoints_cpu.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
// SPDX-License-Identifier: MIT
// Dump the production table for the exhaustive Python oracle.
#include "../runtime/k_s_breakpoints.h"
#include <cstdio>

int main(int argc, char** argv) {
if (argc != 2) return 2;
const auto table = wgrt::ki_s_breakpoints::make_table();
if (wgrt::ki_s_breakpoints::table_digest(table) != wgrt::ki_s_breakpoints::kProvenDigest) {
std::fputs("Model K S breakpoints do not match the proven digest\n", stderr);
return 1;
}
auto* out = std::fopen(argv[1], "wb");
if (!out) return 2;
const size_t count = std::fwrite(table.data(), sizeof table[0], table.size(), out);
const int rc = std::fclose(out);
return count == table.size() && rc == 0 ? 0 : 2;
}
Loading
Loading