From ab13291fd8507960c12d21185463cfea1d763300 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:33:52 -0500 Subject: [PATCH 01/14] K D3D12: phase-split stage m (k_split, deep-split probe) Stage m runs as four launches on the native D3D12 device instead of one k_stagew4_m launch with one 512-lane workgroup per window (6 at 1080p): k_split_m_attn per (window, head), k_split_m_wo per (window, output tile), k_split_m_w1 per (window, two hidden tiles), k_split_m_w2 per (window, output tile), joined by dispatch boundaries. Each accumulator is the same WMMA chain (operands, order, initial value) as k_stagew4_m and every FP16 value handed between launches is the one k_stagew4_m keeps, so the output bytes are unchanged. Operand loads of each chain are fenced (opaque-zero XOR) so the driver issues them together instead of one latency per WMMA. KRuntime::launch redirects k_stagew4_m[__rodd] when k_split=m (default on backend 3; k_split=off restores the one launch), so the create-time self-check runs the split form for its m case against the unchanged HIP reference hashes. Self-check kernel sources and regions are untouched (kSourceDigest unchanged). isa_checks: one v_fma_mix per weight in k_split_m_attn, none in the __rodd twin. --- clean/dx12/isa_checks.json | 4 +- clean/dx12/kernel_sets.json | 20 ++ clean/k/dx12/README.md | 12 + clean/k/dx12/kernels/k_block_split.hlsl | 325 ++++++++++++++++++++++++ clean/k/runtime/runtime_k.cpp | 56 +++- clean/k/runtime/runtime_k.h | 11 + 6 files changed, 426 insertions(+), 2 deletions(-) create mode 100644 clean/k/dx12/kernels/k_block_split.hlsl diff --git a/clean/dx12/isa_checks.json b/clean/dx12/isa_checks.json index c9c4a79e..1d8cf87b 100644 --- a/clean/dx12/isa_checks.json +++ b/clean/dx12/isa_checks.json @@ -1,5 +1,5 @@ { - "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", + "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2; the phase-split m attention k_split_m_attn: one head per workgroup = 1). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", "k_stage_e2": {"v_fma_mix(lo|hi)_f16": 64}, "k_stage_d2": {"v_fma_mix(lo|hi)_f16": 64}, "k_stage2_d1": {"v_fma_mix(lo|hi)_f16": 128}, @@ -7,6 +7,7 @@ "k_stagew4_e4": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew2_e4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew4_m": {"v_fma_mix(lo|hi)_f16": 64}, + "k_split_m_attn": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew4_d4": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew2_d4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew2_d3": {"v_fma_mix(lo|hi)_f16": 64}, @@ -18,6 +19,7 @@ "k_stagew4_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_m__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_attn__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_d3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 07fe9ce0..c15b4c92 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -464,6 +464,26 @@ "k_out_pixel_direct", "k_io.co" ], + [ + "k_split_m_attn", + "k_block_wide.co" + ], + [ + "k_split_m_attn__rodd", + "k_block_wide.co" + ], + [ + "k_split_m_w1", + "k_block_wide.co" + ], + [ + "k_split_m_w2", + "k_block_wide.co" + ], + [ + "k_split_m_wo", + "k_block_wide.co" + ], [ "k_stage2_d0_hidden", "k_block.co" diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index a02f8a4f..e556207d 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -113,6 +113,18 @@ identical. Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero and HIP keeps them. `k_expo_scalars` only meets one with a pre-exposure above about 8.5e37, so the check's inputs keep clear of them. +## Phase-split stage m (runtime option `k_split`, native D3D12 device) + +`kernels/k_block_split.hlsl` runs stage m (`k_stagew4_m`: 6 to 24 windows, one 512-lane workgroup each) as four launches +spread over many more workgroups: `k_split_m_attn` (one per window and head: normalization, q / k / v, softmax, z; +`__rodd` twin), `k_split_m_wo` (one per window and output tile: Wo, x1, a), `k_split_m_w1` (one per window and two hidden +tiles: W1, activation) and `k_split_m_w2` (one per window and output tile: W2, x2, the block output), joined by the +dispatch boundaries. Every accumulator is the same WMMA chain (operands, order, initial value) as in `k_stagew4_m` and every +FP16 value handed between the launches is the one `k_stagew4_m` keeps at that point, so the output bytes are the same. +Default on the native D3D12 device (`k_split=m`; `k_split=off`: the one launch; other devices: the one launch). The runtime +redirects `k_stagew4_m[__rodd]` in `KRuntime::launch`, so the create-time self-check runs the split form for its m case +against the unchanged HIP reference hashes (kernels line `split=m`). + ## Close approximations (runtime option `k_approx`, opt-in) Every port above is exact, and nothing set is the exact path. `k_approx` selects close approximations of two stages on diff --git a/clean/k/dx12/kernels/k_block_split.hlsl b/clean/k/dx12/kernels/k_block_split.hlsl new file mode 100644 index 00000000..b7ef48d0 --- /dev/null +++ b/clean/k/dx12/kernels/k_block_split.hlsl @@ -0,0 +1,325 @@ +// SPDX-License-Identifier: MIT +// Phase-split form of model K's wide stage m (deep-split lever, probe: stage m only). WrongGPU's own code. D3D12 only: +// the same arithmetic as k_stagew4_m (kernels/k_block_wide.hlsl, hlsl/k_wide.hlsli; HIP: k_block_wide.co), whose one +// workgroup per window runs every phase of the block (normalization, q / k / v, attention, Wo, MLP) in sequence on 16 +// waves. At 6 .. 24 windows that leaves most CUs idle and the window's dependent chain of weight loads and WMMAs sets the +// time. Here the window's phases run as four launches, each spread over many more workgroups, joined by the dispatch +// boundaries (the device's UAV barrier between launches): +// k_split_m_attn one workgroup (4 waves) per (window, head): normalization, q / k / v of the head, scores, softmax, +// z of the head -> Z. Every head's K / V is computed once (no redundant K / V). +// k_split_m_wo one workgroup per (window, output tile): y = Wo z + bo over the z tiles in order, x1 = f16(x + y), +// a = f16(x1 g2) -> X1, A. +// k_split_m_w1 one workgroup per (window, KS_HB hidden tiles): h = act(f16(W1 a + b1)) -> H. +// k_split_m_w2 one workgroup per (window, output tile): x2 = f16(x1 + f16(W2 h + b2)) over the hidden tiles in +// order, stored for the real positions of the block output map. +// Wave wv of every workgroup holds the window's tokens 16 wv .. 16 wv + 15 (lane: token 16 wv + lane % 16, channel half +// lane / 16), as in k_stagew4_m. Exactness: every accumulator is the same chain of v_wmma_f32_16x16x16_f16 with the same +// A / B operands in the same order from the same initial value as in k_stagew4_m (bias first; q / k / v over kt; scores +// bias then ct; z over st; Wo over the z tiles 0 .. DT - 1; W1 over kt; W2 over the hidden tiles 0 .. HT - 1), and every +// FP16 value handed between the phases (z, x1, a, h) is the FP16 value k_stagew4_m keeps in LDS or registers at that +// point (the driver's RNE matrix conversion, then the same FP16 operations): the output bytes are k_stagew4_m's. The +// helpers (normalization operations, softmax polynomial, weights k_smx.hlsli, activation, mirror) are k_wide.hlsli's +// and k_smx.hlsli's, unchanged. The create-time self-check runs this form for the m case when the runtime takes it +// (KRuntime::launch redirects k_stagew4_m), against the same HIP reference hashes (k_smx_selfcheck.h). +// Scratch (runtime_k.cpp split_*_, FP16, per window w and token t): Z [w][64][DT 16], X1 / A [w][64][C], H [w][64][HT 16]. +// Arguments: KBlkParams by value (s184:p19.b32, the stage's own), then the four scratch pointers. +// @wg-kernel k_split_m_attn co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_attn defines=KS_PHASE=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_attn__rodd co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_attn defines=K_SMX=1,KS_PHASE=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_wo co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_wo defines=KS_PHASE=2,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_w1 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w1 defines=KS_PHASE=3,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_w2 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w2 defines=KS_PHASE=4,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint P_x, P_skip, P_y, P_merged, P_head, P_g1, P_g2, P_wqkv, P_mix, P_wo, P_bo, P_w1, P_b1, P_w2, P_b2, P_wa, P_ba, + P_wz, P_bz; // pointers (arena offsets; 0 = null) + int P_w, P_h, P_ox, P_oy, P_nwx, P_nwy, P_cw, P_ch; + uint P_sz, P_sx1, P_sa, P_sh; // the split scratch: Z, X1, A, H +WG_ARGS_END + +#include "k_wide.hlsli" +#include "k_smx.hlsli" + +#if KW_IN != 0 || KW_CNEXT != 0 +#error "k_block_split.hlsl: map input, no merge (stage m) only" +#endif + +#ifndef KS_RD +#define KS_RD 20 // weight ring depth of the q / k / v chains (k_split_m_attn) +#endif +#ifndef KS_B2 +#define KS_B2 20 // W2 products per fenced operand batch (k_split_m_w2) +#endif +#ifndef KS_HB +#define KS_HB 2 // hidden tiles per workgroup (k_split_m_w1) +#endif + +// Operand fence: the D3D12 driver's scheduler moves each global load of a short, low-pressure chain down to its +// product (one memory latency per product, measured in the RGA ISA of the first build). A phase therefore loads a +// product group's operands, XORs one word of each into f and XORs KS_FENCE(f) into its first operand (the bias of the +// accumulator, or the first fragment): kw_zero() is 0 at +// run time (grids are below 2^31) but unknown to the compiler, so every load is issued before the group's first product +// and waited once. No bit of any operand changes. +#define KS_FENCE(f) ((f) & kw_zero()) + +static const int SA = 2 * KT * 3; // q / k / v products of one head +static const int NHG = HT / KS_HB; // hidden-tile groups of a window + +// The lane's window geometry (as kw_stage): token t, the mirrored member it reads, whether it is a real position. +struct KsLane { + uint lane, lane8; + int wv, n16, hh, t, gx, gy; + bool real; + uint mtok; +}; +KsLane ks_lane(int win) +{ + KsLane L; + const uint tid = wg_lane_id(); + L.lane = tid & 31u; + L.lane8 = L.lane * 8u; + L.wv = wg_uniform((int)(tid >> 5)); + L.n16 = (int)(L.lane & 15u); + L.hh = (int)(L.lane >> 4); + L.t = 16 * L.wv + L.n16; + const int wa = win % P_nwx, wb = win / P_nwx; + L.gx = 8 * wa - P_ox + (L.t & 7); + L.gy = 8 * wb - P_oy + (L.t >> 3); + L.real = L.gx >= 0 && L.gx < P_w && L.gy >= 0 && L.gy < P_h; + L.mtok = (uint)(kw_mirror(L.gy, P_h) * P_w + kw_mirror(L.gx, P_w)); // mirrored member (spec 41) + return L; +} + +// scratch element offsets (halfs) of token t of window win, channel half hh +uint ks_zrow(int win, int t, int hh) { return (uint)((win * 64 + t) * DT * 16 + 8 * hh); } +uint ks_crow(int win, int t, int hh) { return (uint)((win * 64 + t) * KW_C + 8 * hh); } +uint ks_hrow(int win, int t, int hh) { return (uint)((win * 64 + t) * HT * 16 + 8 * hh); } + +#if KS_PHASE == 1 +// ---- k_split_m_attn: one (window, head) ------------------------------------------------------------------------------ +// LDS: K [64 tokens][4 chunks] and VT [32 channels][8 chunks] of the head, 16-byte chunks xor-swizzled (k_wide.hlsli's +// NG = 4 layout of one group); 8 KB. +groupshared uint4 ks_lds[512]; +uint ks_kidx(int row, int c) { return (uint)(row * 4 + (c ^ (row & 3))) & 511u; } +uint ks_vidx(int row, int c) { return (uint)(256 + row * 8 + (c ^ (row & 7))) & 511u; } + +static uint ksA_qkv; +// product s = (ct KT + kt) 3 + {q, k, v} of the head's q / k / v chains (k_block_wide.hlsl's stream order of a round) +uint4 ks_wq(int s) { return wg_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); } +#define KS_TAKE(s) rg[(s) % KS_RD] +#define KS_NEXT(s) { if ((s) + KS_RD < SA) rg[(s) % KS_RD] = ks_wq((s) + KS_RD); } + +WG_KERNEL(ks_attn) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KW_HEADS, hd = bid % KW_HEADS; + const KsLane L = ks_lane(win); + const int t = L.t, wv = L.wv, n16 = L.n16, hh = L.hh; + ksA_qkv = P_wqkv + 2u * ((uint)(hd * 3 * 2 * KT * 256) + L.lane8); + uint4 rg[KS_RD]; + + // the block input (all channels of the token) and g1, then the weight ring + uint4 n[KT]; + uint4 g1v[KT]; + { + const uint xt = L.mtok * KW_C + 8u * hh; + [unroll] for (int kt = 0; kt < KT; ++kt) n[kt] = kw_ldg(P_x, xt + 16u * kt); + [unroll] for (int kt4 = 0; kt4 < KT; ++kt4) g1v[kt4] = kw_ldg(P_g1, (uint)(16 * kt4 + 8 * hh)); + [unroll] for (int s0 = 0; s0 < KS_RD; ++s0) rg[s0] = ks_wq(s0); + uint f = 0u; // (the chain starts with the last loads) + [unroll] for (int s1 = KS_RD - 1; s1 >= 0; --s1) f ^= rg[s1].x; + [unroll] for (int kt5 = KT - 1; kt5 >= 0; --kt5) f ^= g1v[kt5].x ^ n[kt5].x; + n[0].x ^= KS_FENCE(f); // the input, g1 and the ring in one wait + } + + // normalization (k_block_wide.hlsl step 1): s = f16(sum f16(x^2)), r = f16(rsq(s)); s = 0 gives r = 0 + { + float ss = 0.0f; + [unroll] for (int kt = 0; kt < KT; ++kt) { + const uint4 sq = kw_opq4(kw_mul8(n[kt], n[kt])); + [unroll] for (uint i = 0; i < 4; ++i) { + ss += f16tof32(sq[i] & 0xffffu); + ss += f16tof32(sq[i] >> 16); + } + } + ss += wg_xor16(ss); + const uint s16 = wg_f2h(ss); + const uint r16 = (s16 & 0x7fffu) == 0u ? 0u : wg_f2h(rsqrt(f16tof32(s16))); + const uint4 rv = kw_splat8(r16); + [unroll] for (int kt = 0; kt < KT; ++kt) + n[kt] = kw_mul8(n[kt], kw_opq4(kw_mul8(rv, g1v[kt]))); + } + + // q, k, v^T of head hd (k_block_wide.hlsl step 2, one round's group) + uint4 q[2], kk[2], vv[2]; + [unroll] for (int ct = 0; ct < 2; ++ct) { + v8f aq = wg_splat(0.0f), ak = wg_splat(0.0f), av = wg_splat(0.0f); + [unroll] for (int kt = 0; kt < KT; ++kt) { + const int s = (ct * KT + kt) * 3; + aq = kw_mma(KS_TAKE(s), n[kt], aq); + KS_NEXT(s); + ak = kw_mma(KS_TAKE(s + 1), n[kt], ak); + KS_NEXT(s + 1); + av = kw_mma(n[kt], KS_TAKE(s + 2), av); + KS_NEXT(s + 2); + } + q[ct] = kw_out(aq); + kk[ct] = kw_out(ak); + vv[ct] = kw_out(av); + } + const uint mrow = (uint)((hd * 64 + t) * 64 + 8 * hh); + uint4 bm[4]; + [unroll] for (int st0 = 0; st0 < 4; ++st0) bm[st0] = kw_ldg(P_mix, mrow + 16u * st0); + [unroll] for (int ct1 = 0; ct1 < 2; ++ct1) { + ks_lds[ks_kidx(t, 2 * ct1 + hh)] = kk[ct1]; // K[token t][channels] + ks_lds[ks_vidx(16 * ct1 + n16, 2 * wv + hh)] = vv[ct1]; // VT[channel][tokens] + } + wg_barrier_lds(); // K / VT of the head complete + + // scores^T = K q + B, the clamped polynomial, softmax weights A (k_block_wide.hlsl, the same order) + uint4 pa[4]; + { + uint4 kf[4][2]; + [unroll] for (int st = 0; st < 4; ++st) + [unroll] for (int ct = 0; ct < 2; ++ct) kf[st][ct] = ks_lds[ks_kidx(16 * st + n16, 2 * ct + hh)]; + v8f sc[4]; + [unroll] for (int st1 = 0; st1 < 4; ++st1) sc[st1] = kw_widen8(bm[st1]); + [unroll] for (int ct = 0; ct < 2; ++ct) + [unroll] for (int st2 = 0; st2 < 4; ++st2) sc[st2] = kw_mma(kf[st2][ct], q[ct], sc[st2]); + uint4 gg[4]; + [unroll] for (int st5 = 0; st5 < 4; ++st5) gg[st5] = kw_softpoly8(kw_out(sc[st5])); + float w[4][8]; + float sum = 0.0f; + [unroll] for (int st3 = 0; st3 < 4; ++st3) + [unroll] for (int j = 0; j < 8; ++j) { + w[st3][j] = exp2(f16tof32((gg[st3][j >> 1] >> (16 * (j & 1))) & 0xffffu)); // v_exp_f32 + sum += w[st3][j]; + } + sum += wg_xor16(sum); + const float inv = rcp(sum); // v_rcp_f32 + [unroll] for (int st4 = 0; st4 < 4; ++st4) pa[st4] = k_smx_weights8(w[st4], inv); + } + // z_h^T = VT A^T (the two channel tiles' chains interleaved, each over st) -> Z tiles 2 hd, 2 hd + 1 + { + uint4 vf[2][4]; + [unroll] for (int ct = 0; ct < 2; ++ct) + [unroll] for (int st = 0; st < 4; ++st) vf[ct][st] = ks_lds[ks_vidx(16 * ct + n16, 2 * st + hh)]; + v8f az[2]; + az[0] = wg_splat(0.0f); + az[1] = wg_splat(0.0f); + [unroll] for (int st = 0; st < 4; ++st) + [unroll] for (int ct2 = 0; ct2 < 2; ++ct2) az[ct2] = kw_mma(vf[ct2][st], pa[st], az[ct2]); + const uint zr = ks_zrow(win, t, hh) + (uint)(32 * hd); + kw_stg(P_sz, zr, kw_out(az[0])); + kw_stg(P_sz, zr + 16u, kw_out(az[1])); + } +} +#endif + +#if KS_PHASE == 2 +// ---- k_split_m_wo: one (window, output tile ot) ---------------------------------------------------------------------- +WG_KERNEL(ks_wo) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KT, ot = bid % KT; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint wo = P_wo + 2u * ((uint)(ot * DT * 256) + L.lane8); // fragment dt at + 512 dt + const uint zr = ks_zrow(win, L.t, hh); + uint4 wf[DT], zf[DT]; + [unroll] for (int d0 = 0; d0 < DT; ++d0) { + wf[d0] = wg_ld128(wo, (uint)(d0 * 512)); + zf[d0] = kw_ldg(P_sz, zr + 16u * d0); + } + const uint4 xr = kw_ldg(P_x, L.mtok * KW_C + 8u * hh + 16u * ot); + const uint4 g2v = kw_ldg(P_g2, (uint)(16 * ot + 8 * hh)); + uint4 bo = kw_ldg(P_bo, (uint)(16 * ot + 8 * hh)); + uint f = bo.x ^ xr.x ^ g2v.x; // (the chain starts with the last loads) + [unroll] for (int d1 = 0; d1 < DT; ++d1) f ^= wf[d1].x ^ zf[d1].x; + bo.x ^= KS_FENCE(f); + v8f yacc = kw_widen8(bo); + [unroll] for (int dt = 0; dt < DT; ++dt) yacc = kw_mma(wf[dt], zf[dt], yacc); + const uint4 x1 = kw_add8(xr, kw_out(yacc)); // x1 = f16(x + y) + const uint cr = ks_crow(win, L.t, hh) + 16u * ot; + kw_stg(P_sx1, cr, x1); + kw_stg(P_sa, cr, kw_mul8(x1, g2v)); // a = f16(x1 g2) +} +#endif + +#if KS_PHASE == 3 +// ---- k_split_m_w1: one (window, KS_HB hidden tiles) ------------------------------------------------------------------ +WG_KERNEL(ks_w1) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / NHG, h0 = (bid % NHG) * KS_HB; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint w1 = P_w1 + 2u * ((uint)(h0 * KT * 256) + L.lane8); // hidden tile h0 + m, kt: + 512 (m KT + kt) + const uint cr = ks_crow(win, L.t, hh); + uint4 a[KT]; + uint4 wf[KS_HB][KT]; + [unroll] for (int kt0 = 0; kt0 < KT; ++kt0) { + [unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = wg_ld128(w1, (uint)((m0 * KT + kt0) * 512)); + a[kt0] = kw_ldg(P_sa, cr + 16u * kt0); + } + uint4 b1v[KS_HB]; + [unroll] for (int m4 = 0; m4 < KS_HB; ++m4) b1v[m4] = kw_ldg(P_b1, (uint)(16 * (h0 + m4) + 8 * hh)); + uint f = 0u; // (the chain starts with the last loads) + [unroll] for (int m5 = 0; m5 < KS_HB; ++m5) f ^= b1v[m5].x; + [unroll] for (int kt1 = KT - 1; kt1 >= 0; --kt1) { + f ^= a[kt1].x; + [unroll] for (int m3 = KS_HB - 1; m3 >= 0; --m3) f ^= wf[m3][kt1].x; + } + v8f p[KS_HB]; + [unroll] for (int m1 = 0; m1 < KS_HB; ++m1) { + b1v[m1].x ^= KS_FENCE(f); + p[m1] = kw_widen8(b1v[m1]); + } + [unroll] for (int kt = 0; kt < KT; ++kt) // the hidden tiles' chains interleaved, each over kt + [unroll] for (int m = 0; m < KS_HB; ++m) p[m] = kw_mma(wf[m][kt], a[kt], p[m]); + const uint hr = ks_hrow(win, L.t, hh); + [unroll] for (int m2 = 0; m2 < KS_HB; ++m2) kw_stg(P_sh, hr + (uint)(16 * (h0 + m2)), kw_act8(kw_out(p[m2]))); +} +#endif + +#if KS_PHASE == 4 +// ---- k_split_m_w2: one (window, output tile ot) ---------------------------------------------------------------------- +// The chain over the HT hidden tiles in batches of KS_B2 products, each batch's operands (W2 fragment (j, ot), h fragment +// of hidden tile j) loaded and fenced before its first product. +static const int NB2 = HT / KS_B2; +WG_KERNEL(ks_w2) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KT, ot = bid % KT; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint w2 = P_w2 + 2u * ((uint)(ot * 256) + L.lane8); // hidden tile j: + 512 j KT + const uint hb = P_sh + 2u * ks_hrow(win, L.t, hh); // hidden tile j: + 32 j + const uint cr = ks_crow(win, L.t, hh) + 16u * ot; + const uint4 x1 = kw_ldg(P_sx1, cr); + uint4 b2v = kw_ldg(P_b2, (uint)(16 * ot + 8 * hh)); + v8f acc; + [unroll] for (int b = 0; b < NB2; ++b) { // the hidden tiles 0 .. HT - 1, in order + uint4 rw[KS_B2], rh[KS_B2]; + [unroll] for (int j0 = 0; j0 < KS_B2; ++j0) { + rw[j0] = wg_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512)); + rh[j0] = wg_ld128(hb, (uint)((b * KS_B2 + j0) * 32)); + } + uint f = b == 0 ? b2v.x ^ x1.x : 0u; + [unroll] for (int j1 = 0; j1 < KS_B2; ++j1) f ^= rw[j1].x ^ rh[j1].x; + if (b == 0) { // the bias after the fence: no early wait on it + b2v.x ^= KS_FENCE(f); + acc = kw_widen8(b2v); + } else { + rh[0].x ^= KS_FENCE(f); + } + [unroll] for (int j = 0; j < KS_B2; ++j) acc = kw_mma(rw[j], rh[j], acc); + } + const uint4 x2 = kw_pin(kw_add8(x1, kw_out(acc))); // x2 = f16(x1 + f16(W2 h + b2)), before the divergent store + if (L.real && P_y != 0u) kw_stg(P_y, (uint)((L.gy * P_w + L.gx) * KW_C + 8 * hh) + 16u * ot, x2); +} +#endif diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 76bb1133..fe61de2c 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -561,6 +561,14 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (warm_line_ < 4 || warm_line_ > 4096 || (warm_line_ & (warm_line_ - 1))) return fail("model K: k_warm_line must be a power of two in 4..4096"); if (warm_groups_ < 1 || warm_groups_ > 1024) return fail("model K: k_warm_groups must be 1..1024"); + // k_split (deep-split; the native D3D12 device, backend 3): "m" (the default) runs stage m as four phase launches + // spread over many more workgroups (k_split_m_attn[__rodd], k_split_m_wo, k_split_m_w1, k_split_m_w2: + // clean/k/dx12/kernels/k_block_split.hlsl), the same arithmetic and output bytes as the one k_stagew4_m launch; the + // create-time self-check runs that form for its m case (launch() redirects k_stagew4_m). "off": the one launch. Other + // devices always take the one launch (the phase kernels are D3D12 ports only). + const std::string ksplit = option(c, "k_split"); + if (!ksplit.empty() && ksplit != "m" && ksplit != "off") return fail("model K: k_split must be m or off"); + split_m_ = c.backend == 3 && ksplit != "off"; // k_fill=<0..255> (test use, changes no value a frame writes): every buffer a frame writes and both history copies are // filled with this byte after allocation, before the first frame, so that a test with a different byte per backend // sees a write a frame did not make (first-frame and reset-only writes included) as a difference @@ -683,6 +691,7 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); if (!dev_->sync()) return dfail(); + if (split_m_ && !split_alloc()) return false; // the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or // corrupt shader fails create here, not at the first frame if (c.backend == 3) { @@ -960,6 +969,9 @@ std::string KRuntime::stage_kernel(const std::string& k, const char* stage) cons bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a0, void* a1, void* a2, void* a3, void* a4, void* a5, void* a6, void* a7) { if (grid == 0) return true; + // k_split: stage m (the frame's and the self-check's) as its phase launches + if (split_m_ && std::strncmp(kernel, "k_stagew4_m", 11) == 0 && (!kernel[11] || std::strcmp(kernel + 11, "__rodd") == 0)) + return split_m(kernel[11] != 0, a0); void* args[8] = {a0, a1, a2, a3, a4, a5, a6, a7}; if (pending_mark_ >= 0) { const int e = pending_mark_; @@ -969,6 +981,45 @@ bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a return dev_->launch(kernel, grid, block, args) || dfail(); } +// k_split: stage m as four launches (clean/k/dx12/kernels/k_block_split.hlsl), joined by the dispatch boundaries: +// attention per (window, head), Wo per (window, output tile), W1 per (window, two hidden tiles), W2 per (window, output +// tile). Each phase reads the block parameters of k_stagew4_m's launch and the scratch the phase before wrote. +bool KRuntime::split_m(bool rodd, void* params) { + const KBlkParams& p = *static_cast(params); + const KStageInfo& st = *k_stage("m"); + const int nwin = p.nwx * p.nwy, kt = st.C / 16, ht = 4 * st.C / 16; + if (nwin > split_cap_) + return fail("model K: k_split: stage m has " + std::to_string(nwin) + " windows, the scratch holds " + + std::to_string(split_cap_)); + return launch(rodd ? "k_split_m_attn__rodd" : "k_split_m_attn", (unsigned)(nwin * st.heads), 128, params, &split_z_, + &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_wo", (unsigned)(nwin * kt), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_w1", (unsigned)(nwin * (ht / 2)), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_w2", (unsigned)(nwin * kt), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_); +} + +// The phases' scratch (FP16): Z [windows][64][8 heads x 32], X1 and A [windows][64][C], H [windows][64][4 C], for the most +// windows stage m has at any window offset (0 .. 7) in this context's frames or in the self-check's m case. +bool KRuntime::split_alloc() { + const KStageInfo& st = *k_stage("m"); + if (st.C != 160 || st.heads != 8) // the phase kernels are compiled for stage m's C = 160, 8 heads + return fail("model K: k_split: stage m is not C = 160 with 8 heads"); + int cap = 0; + auto grow = [&](int wb, int hb) { + int lw, lh; + k_level_size(wb, hb, st.level, &lw, &lh); + for (int oy = 0; oy < 8; ++oy) + for (int ox = 0; ox < 8; ++ox) cap = std::max(cap, k_window_count(lw, lh, ox, oy)); + }; + grow(Bw_, Bh_); + for (const SmxCase& sc : kSmxCases) + if (std::strcmp(sc.stage, "m") == 0) grow(sc.wb, sc.hb); + split_cap_ = cap; + const size_t tok = (size_t)cap * 64; + return alloc(&split_z_, tok * st.heads * 32 * 2) && alloc(&split_x1_, tok * st.C * 2) && + alloc(&split_a_, tok * st.C * 2) && alloc(&split_h_, tok * 4 * st.C * 2); +} + bool KRuntime::upload_inputs(int rw, int rh, const float* color, const float* dep, const float* mot, int channels) { if (rw <= 0 || rh <= 0 || rw > max_rw_ || rh > max_rh_) return fail("render size larger than the maximum"); if (external_io_) return fail("inputs come from the interop (external_io)"); @@ -1448,6 +1499,9 @@ std::vector KRuntime::d3d12_kernels() const { v.push_back(k + "__rodd"); } } + if (split_m_) // k_split: stage m's phase kernels (k_stagew4_m[__rodd] stay listed above, unused while split) + for (const char* k : {"k_split_m_attn", "k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"}) + v.push_back(k); v.push_back("k_mix_probe"); v.push_back("k_mix_probe__rodd"); v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); @@ -1515,7 +1569,7 @@ const char* KRuntime::kernels() { " legacy_color_launches=" + std::to_string(legacy_color_launches_) + " legacy_base_launches=" + std::to_string(legacy_base_launches_) + (fast_ ? " k_fast=1" : "") + (expo_v051_ ? " k_expo_rule=v051" : "") + - smx_note_ + approx_tag() + " hidden=fused_d0" + std::string(" head=") + + smx_note_ + (split_m_ ? " split=m" : "") + approx_tag() + " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + " base=" + std::to_string(Bw_) + "x" + std::to_string(Bh_) + " hdr=" + (hdr_ ? "1" : "0") + " mv=render" + diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index f810eb4f..d1a0c1a8 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -124,6 +124,12 @@ class KRuntime { std::string smx_case_name(int i, int ox, int oy, bool rodd) const; // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe std::string stage_kernel(const std::string& k, const char* stage) const; + // deep-split (runtime option k_split, native D3D12 device): stage m as its four phase launches (k_split_m_*, + // clean/k/dx12/kernels/k_block_split.hlsl) in place of one k_stagew4_m launch; launch() redirects k_stagew4_m[__rodd] + // here, so network() and the create-time self-check both run it. split_alloc (create): the phases' scratch, sized for + // the most windows stage m can have in a frame of this context or in the self-check's m case. + bool split_m(bool rodd, void* params); + bool split_alloc(); int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with WgRtConfig cfg_ = {}; @@ -184,6 +190,11 @@ class KRuntime { return std::string(" k_approx=") + (approx_a1_ ? "a1" : "") + (approx_a1_ && approx_a2_ ? "," : "") + (approx_a2_ ? "a2" : ""); } + // runtime option k_split (backend 3 only; default "m"): stage m runs as its phase launches (split_m); "off": one + // k_stagew4_m launch. split_cap_: windows the scratch holds; split_z_ / x1_ / a_ / h_: the phases' FP16 scratch + bool split_m_ = false; + int split_cap_ = 0; + DevPtr split_z_ = 0, split_x1_ = 0, split_a_ = 0, split_h_ = 0; std::string smx_note_; std::string smx_test_; // option k_smx_test (developer builds only) std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty From f6d091c954378a96d97673fe624a9965b9c189f8 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:33:52 -0500 Subject: [PATCH 02/14] K deep-split: self-check digest covers the split stage m path The native D3D12 device runs stage m's phase kernels (k_split_m_*) for the self-check's m case (KRuntime::launch redirect), but the digest that binds k_smx_selfcheck.h to its sources covered only k_stagew4_m. Add the five phase kernels to the digest's kernel set and mark the k_split option, the redirect and split_m / split_alloc as SMX-SELFCHECK-SOURCE regions; digest test cases for k_block_split.hlsl and the redirect. The header's kSourceDigest is NOT rebound here (the 133 hashes are unchanged; rebinding needs coordinator / oversight OK): until then normal builds refuse this tree and the lever builds --test-only. --- clean/k/dx12/smx_selfcheck_digest.py | 8 ++++++-- clean/k/runtime/runtime_k.cpp | 6 ++++++ clean/k/tests/test_k_smx_selfcheck_digest.py | 5 ++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/clean/k/dx12/smx_selfcheck_digest.py b/clean/k/dx12/smx_selfcheck_digest.py index 00d743af..c4542eb3 100644 --- a/clean/k/dx12/smx_selfcheck_digest.py +++ b/clean/k/dx12/smx_selfcheck_digest.py @@ -3,7 +3,8 @@ The reference hashes in k_smx_selfcheck.h are valid only for the shaders and the synthetic data they were generated from. The digest is a SHA-256 over: -- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines); +- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines), the + phase kernels of a split stage m included (SPLIT: the native device runs them for the self-check's m case); - the text of every source file those kernels compile from (their .hlsl file and its #include closure in clean/k/dx12/hlsl and clean/dx12/hlsl; an include generated into the build folder is covered by its generator, clean/k/dx12/gen_k_fp_tables.py); @@ -28,7 +29,10 @@ # the softmax stage kernels the self-check runs (runtime_k.cpp kSmxCases), both forms BASE = ["k_stage_e2", "k_stagew2_e3", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1", "k_stagew4_e4", "k_stagew4_d4", "k_stagew2_e4", "k_stagew2_d4", "k_stagew4_m"] -KERNELS = sorted(BASE + [k + "__rodd" for k in BASE]) +# the phase kernels the native D3D12 device runs in place of k_stagew4_m[__rodd] (runtime option k_split, default on +# backend 3; KRuntime::launch redirects stage m, so the self-check's m case runs them: kernels/k_block_split.hlsl) +SPLIT = ["k_split_m_attn", "k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"] +KERNELS = sorted(BASE + [k + "__rodd" for k in BASE] + SPLIT) DECL = re.compile(r"^//\s*@wg-kernel\s+(\S+)\s.*$", re.M) INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) REGION = re.compile(r"// SMX-SELFCHECK-SOURCE BEGIN.*?// SMX-SELFCHECK-SOURCE END", re.S) diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index fe61de2c..a9bc8353 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -561,6 +561,7 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (warm_line_ < 4 || warm_line_ > 4096 || (warm_line_ & (warm_line_ - 1))) return fail("model K: k_warm_line must be a power of two in 4..4096"); if (warm_groups_ < 1 || warm_groups_ > 1024) return fail("model K: k_warm_groups must be 1..1024"); + // SMX-SELFCHECK-SOURCE BEGIN (k_split: routes the self-check's m case; the digest covers it) // k_split (deep-split; the native D3D12 device, backend 3): "m" (the default) runs stage m as four phase launches // spread over many more workgroups (k_split_m_attn[__rodd], k_split_m_wo, k_split_m_w1, k_split_m_w2: // clean/k/dx12/kernels/k_block_split.hlsl), the same arithmetic and output bytes as the one k_stagew4_m launch; the @@ -569,6 +570,7 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e const std::string ksplit = option(c, "k_split"); if (!ksplit.empty() && ksplit != "m" && ksplit != "off") return fail("model K: k_split must be m or off"); split_m_ = c.backend == 3 && ksplit != "off"; + // SMX-SELFCHECK-SOURCE END // k_fill=<0..255> (test use, changes no value a frame writes): every buffer a frame writes and both history copies are // filled with this byte after allocation, before the first frame, so that a test with a different byte per backend // sees a write a frame did not make (first-frame and reset-only writes included) as a difference @@ -969,9 +971,11 @@ std::string KRuntime::stage_kernel(const std::string& k, const char* stage) cons bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a0, void* a1, void* a2, void* a3, void* a4, void* a5, void* a6, void* a7) { if (grid == 0) return true; + // SMX-SELFCHECK-SOURCE BEGIN (k_split: stage m's kernel-name redirect; the digest covers it) // k_split: stage m (the frame's and the self-check's) as its phase launches if (split_m_ && std::strncmp(kernel, "k_stagew4_m", 11) == 0 && (!kernel[11] || std::strcmp(kernel + 11, "__rodd") == 0)) return split_m(kernel[11] != 0, a0); + // SMX-SELFCHECK-SOURCE END void* args[8] = {a0, a1, a2, a3, a4, a5, a6, a7}; if (pending_mark_ >= 0) { const int e = pending_mark_; @@ -981,6 +985,7 @@ bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a return dev_->launch(kernel, grid, block, args) || dfail(); } +// SMX-SELFCHECK-SOURCE BEGIN (k_split: the phase launches and scratch; the digest covers them) // k_split: stage m as four launches (clean/k/dx12/kernels/k_block_split.hlsl), joined by the dispatch boundaries: // attention per (window, head), Wo per (window, output tile), W1 per (window, two hidden tiles), W2 per (window, output // tile). Each phase reads the block parameters of k_stagew4_m's launch and the scratch the phase before wrote. @@ -1019,6 +1024,7 @@ bool KRuntime::split_alloc() { return alloc(&split_z_, tok * st.heads * 32 * 2) && alloc(&split_x1_, tok * st.C * 2) && alloc(&split_a_, tok * st.C * 2) && alloc(&split_h_, tok * 4 * st.C * 2); } +// SMX-SELFCHECK-SOURCE END bool KRuntime::upload_inputs(int rw, int rh, const float* color, const float* dep, const float* mot, int channels) { if (rw <= 0 || rh <= 0 || rw > max_rw_ || rh > max_rh_) return fail("render size larger than the maximum"); diff --git a/clean/k/tests/test_k_smx_selfcheck_digest.py b/clean/k/tests/test_k_smx_selfcheck_digest.py index f92bf6dc..dd4b3091 100644 --- a/clean/k/tests/test_k_smx_selfcheck_digest.py +++ b/clean/k/tests/test_k_smx_selfcheck_digest.py @@ -54,7 +54,10 @@ def test_changes_are_caught(self): (os.path.join("clean", "dx12", "hlsl", "wg_math.hlsli"), "float"), (os.path.join("clean", "k", "dx12", "kernels", "k_block_wide.hlsl"), "KW_RD=12"), (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "kSmxSeed = 0x6b5eu"), - (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)")] + (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)"), + (os.path.join("clean", "k", "dx12", "kernels", "k_block_split.hlsl"), "#define KS_RD 20"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "return split_m(kernel[11] != 0, a0);"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "split_m_ = c.backend == 3")] for rel, needle in cases: with self.subTest(file=rel): tmp = tempfile.mkdtemp(prefix="smx_digest_") From 17ead5f2dc74a29646d4da4466cf1d05a9d6e9fb Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:33:52 -0500 Subject: [PATCH 03/14] K deep-split: README notes the digest coverage and k_split parsing on every device --- clean/k/dx12/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index e556207d..e5bd562e 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -123,7 +123,9 @@ dispatch boundaries. Every accumulator is the same WMMA chain (operands, order, FP16 value handed between the launches is the one `k_stagew4_m` keeps at that point, so the output bytes are the same. Default on the native D3D12 device (`k_split=m`; `k_split=off`: the one launch; other devices: the one launch). The runtime redirects `k_stagew4_m[__rodd]` in `KRuntime::launch`, so the create-time self-check runs the split form for its m case -against the unchanged HIP reference hashes (kernels line `split=m`). +against the unchanged HIP reference hashes (kernels line `split=m`). The self-check digest (`smx_selfcheck_digest.py`) +covers the phase kernels and the runtime's `k_split` option, redirect and phase launches. `k_split` is parsed on every +device: a value other than `m` or `off` fails create there too. ## Close approximations (runtime option `k_approx`, opt-in) From ff10f0a47bfda123e48d3ca09d4541bd3fdd50a3 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 13:33:52 -0500 Subject: [PATCH 04/14] K deep-split: rebind the self-check digest to the split-path tree The D3D12 report hook (k_smx_test=report, developer build) on the native D3D12 device reproduced all 133 HIP reference hashes of k_smx_selfcheck.h in both softmax forms (mix and round-to-odd), 0 mismatches, probe 0 / 0, create smx=mix. kSourceDigest is rebound to the digest of this tree (smx_selfcheck_digest.digest()); the header's hashes and names are unchanged. Record: R:\WrongGPU-clean\tmp\kl-speed-K\deep-split\diag\smx-report-diag.json. --- clean/k/runtime/k_smx_selfcheck.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index c4fed657..d019a35c 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -2,13 +2,19 @@ // Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs, // first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases, // KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py). +// kSourceDigest provenance (2026-11-18, deep-split fix round 2, oversight-approved rebind): the D3D12 report hook +// (runtime option k_smx_test=report, developer build with WG_DEV_DIAGNOSTICS) on the native D3D12 device (backend 3, +// no HIP) reproduced all 133 hashes below in both softmax forms (mix and round-to-odd), 0 mismatches, probe 0 / 0. +// Record: R:\WrongGPU-clean\tmp\kl-speed-K\deep-split\diag\smx-report-diag.json (and RECORD.md "Fix round 2"). +// The digest value is the one smx_selfcheck_digest.digest() computes for this tree's sources; the hashes are the HIP +// reference's, unchanged from the previous header. #pragma once #include namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "2ad91491cb5edf84114baa30bc0fe79a4859ac281a90d8989bc57130f8d05d69"; +constexpr char kSourceDigest[] = "1a51cb9d0f4f5b5e8d1acd833f814ff7457a169416d453692e3c449f75688bac"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", From 3ed43f18ce1f3b3b8f4ecca7d5e47618052c813a Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:13:13 -0500 Subject: [PATCH 05/14] K deep-split: fallback trace expects the phase launches under split=m check_fallback_frames expected k_stagew4_m__rodd on backend 3, but with k_split=m (the default) stage m runs as k_split_m_attn__rodd and the k_split_m_wo / w1 / w2 launches. The expectation now follows the option: the phase launches with split=m, and a k_split=off leg expects k_stagew4_m__rodd. The forbidden mix forms follow the same rule (the phases wo / w1 / w2 have no softmax and run in both forms). --- clean/k/tests/test_k_runtime_d3d12.py | 34 +++++++++++++++++++-------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 4f31df9b..d97c6cfa 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -355,23 +355,37 @@ def test_native_softmax_refusals(self): finally: shutil.rmtree(tmp, ignore_errors=True) - def check_fallback_frames(self, frames, big, **size): + def expect_fallback_trace(self, on_dx, big, split): + """The softmax stages of the forced fallback ran their __rodd ports on D3D12 and no mix form did. Stage m runs as + its phase launches with k_split=m (the default) and as k_stagew4_m[__rodd] with k_split=off. The phases + k_split_m_wo / w1 / w2 have no softmax: they are the same launches in both forms, so only the attention phase + has a mix form to forbid.""" + ng = "k_stagew2_" if big else "k_stagew4_" + stages = ["k_stage_e2", "k_stagew2_e3", ng + "e4", ng + "d4", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1"] + if split: + rodd, mix = ["k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"], ["k_split_m_attn"] + else: + rodd, mix = ["k_stagew4_m__rodd"], ["k_stagew4_m"] + self.assertEqual([k for k in stages if not on_dx.get(k + "__rodd")] + [k for k in rodd if not on_dx.get(k)], [], + on_dx) + self.assertEqual([k for k in stages + mix if on_dx.get(k)], [], on_dx) + + def check_fallback_frames(self, frames, big, split=True, **size): """Forced fallback: the frames match HIP (raw outputs and intermediates) and every softmax stage ran its __rodd - port on D3D12, none its mix form.""" + port on D3D12, none its mix form (split: stage m's phase launches; else its one launch).""" self.need_dev_build() - on_dx, _ = self.sequence(3, True, 0, True, frames, "k_smx_test=mix_fail", **size) - ng = "k_stagew2_" if big else "k_stagew4_" - want = ["k_stage_e2", "k_stagew2_e3", ng + "e4", "k_stagew4_m", ng + "d4", "k_stagew2_d3", "k_stage_d2", - "k_stage2_d1"] - self.assertEqual([k for k in want if not on_dx.get(k + "__rodd")], [], on_dx) - self.assertEqual([k for k in want if on_dx.get(k)], [], on_dx) + options = "k_smx_test=mix_fail" + ("" if split else ";k_split=off") + on_dx, _ = self.sequence(3, True, 0, True, frames, options, **size) + self.expect_fallback_trace(on_dx, big, split) def test_native_fallback_frames_1440p_quality(self): self.check_fallback_frames(self.FRAMES_Q, False, **self.QUALITY) + self.check_fallback_frames(self.FRAMES_Q, False, split=False, **self.QUALITY) def test_native_fallback_frames_4k_performance(self): - self.check_fallback_frames([(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)], True, - out=(3840, 2160), max_render=(1920, 1080)) + frames = [(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)] + self.check_fallback_frames(frames, True, out=(3840, 2160), max_render=(1920, 1080)) + self.check_fallback_frames(frames, True, split=False, out=(3840, 2160), max_render=(1920, 1080)) # ---- k_approx: the opt-in close approximations (never the default) -------------------------------------------- def test_k_approx_refusals(self): From 7c547ecdca4aebe1e43a0242386b2e0de4a4d248 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:14:01 -0500 Subject: [PATCH 06/14] K deep-split: fall back to the one launch when the split self-check fails A split stage m (k_split=m) that fails the create-time self-check no longer refuses create. Stage m takes the one k_stagew4_m launch (the same bytes and the same reference hashes), the self-check runs again on that path, and one create note says so, in the pattern of the round-to-odd note. Create refuses only when the one-launch path fails the self-check too; the error names both failures. Developer hook: k_smx_test=split_fail (WG_DEV_DIAGNOSTICS builds) makes the split form count as failed, so the fallback can be tested. With k_split=off it has no effect. --- clean/k/dx12/README.md | 6 +++++- clean/k/runtime/runtime_k.cpp | 33 +++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index e5bd562e..01bc8a45 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -97,7 +97,7 @@ refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-con `build.py` never embeds a test-only, bypassed, define-built, unprovenanced or stale entry, and the payload packager refuses a library whose build manifest does not record that check (`clean/tests/test_dev_diagnostics_off.py`). -The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built +The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `split_fail`, `report`) exists only in runtime libraries built with `build.py --dev-diagnostics`; a release build refuses it (`test_k_runtime_d3d12.py`: `test_native_softmax_switch_is_logged`, `test_native_softmax_refusals`, `test_native_fallback_frames_*`). Negative controls, never shipped: `K_SMX_MUT=1` (the mix form's residual dropped), `K_SMX_MUT4=1` (dropped only at window offset @@ -127,6 +127,10 @@ against the unchanged HIP reference hashes (kernels line `split=m`). The self-ch covers the phase kernels and the runtime's `k_split` option, redirect and phase launches. `k_split` is parsed on every device: a value other than `m` or `off` fails create there too. +If the create-time self-check fails on the split form of stage m, create does not refuse: stage m takes the one launch, +the self-check runs again on that path, and one note of create says so (the output is the same). Create refuses only +when the one-launch path fails the self-check too. The developer hook `k_smx_test=split_fail` forces the first failure. + ## Close approximations (runtime option `k_approx`, opt-in) Every port above is exact, and nothing set is the exact path. `k_approx` selects close approximations of two stages on diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index a9bc8353..6f7c3684 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -585,8 +585,8 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e smx_test_ = option(c, "k_smx_test"); #ifdef WG_DEV_DIAGNOSTICS if (!smx_test_.empty() && smx_test_ != "mix_fail" && smx_test_ != "both_fail" && smx_test_ != "rodd_unchecked" && - smx_test_ != "report") - return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked or report"); + smx_test_ != "split_fail" && smx_test_ != "report") + return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked, split_fail or report"); #else if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only"); #endif @@ -701,7 +701,28 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!dev_->prepare(k)) return dfail(); prepared_ = true; } - if (!smx_select(c.backend, smx_test_)) return false; + // the create-time self-check. A split stage m (k_split=m) that fails it is not a refusal: stage m takes the one launch + // (k_split=off's path: the same bytes, the same reference hashes), the self-check runs again on that path, and create + // refuses only when that fails too (both reasons in the error) + const std::string notes_before = notes_; + if (!smx_select(c.backend, smx_test_)) { + if (!split_m_) return false; + std::string split_why = err_; + for (char& ch : split_why) + if (ch == '\n' || ch == '\r') ch = ' '; + notes_ = notes_before; + split_m_ = false; + if (!smx_select(c.backend, smx_test_)) { + std::string one_launch = err_; + for (char& ch : one_launch) + if (ch == '\n' || ch == '\r') ch = ' '; + return fail(one_launch + " (the split form of stage m failed first: " + split_why + ")"); + } + notes_ += (notes_.empty() ? "" : "\n") + + std::string("model K: the create-time self-check failed on the split form of stage m (") + split_why + + "); stage m takes the one launch (k_stagew4_m), output unchanged"; + err_.clear(); + } if (!approx_note_.empty()) notes_ += (notes_.empty() ? "" : "\n") + approx_note_; return true; } @@ -724,7 +745,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e // prepared at create (KRuntime::create, Device::prepare): a missing or corrupt shader fails create. // Test hook (developer builds only, WG_DEV_DIAGNOSTICS): runtime option k_smx_test = mix_fail (the mix form counts as // failed), both_fail (both forms count as failed), rodd_unchecked (the round-to-odd ports without the check; for the -// frame test's negative control), report (adds the computed hashes as a note; on any device, every possible offset). +// frame test's negative control), split_fail (the split form of stage m counts as failed, so create takes the one +// launch; with k_split=off it has no effect), report (adds the computed hashes as a note; on any device, every possible +// offset). namespace { // SMX-SELFCHECK-SOURCE BEGIN (the cases, the synthetic data, the offsets and the check: k_smx_selfcheck.h's digest covers this text) struct SmxCase { @@ -908,6 +931,8 @@ bool KRuntime::smx_select(int backend, const std::string& hook) { add_note(report("reference", h, -1)); return true; } + if (hook == "split_fail" && split_m_) // test hook: the split form of stage m counts as failed (create's fallback) + return fail("model K: test hook: the split self-check of stage m failed (k_smx_test=split_fail)"); if (hook == "rodd_unchecked") { smx_rodd_ = true; smx_note_ = " smx=rodd(unchecked)"; From 703ec2ef01039c910a7e4c32fa75753a2d7d8990 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:15:01 -0500 Subject: [PATCH 07/14] K deep-split: native tests for the split stage m, its refusals and the fallback Native-against-native tests (no HIP reference run, so they run on the developer device without the HIP legs): - split=m (the default) against k_split=off: the same raw bytes of the output and of every written buffer, history and exposure state after every frame, at 200 x 120 and at an odd output size (201 x 121 from 131 x 79), in RGBA16F and in dense RGB with auto-exposure. Each side starts from its own sentinel (0x5A, 0xC3), so a word neither side wrote fails. - The launch trace: the phase launches with k_split=m, k_stagew4_m with k_split=off, and the round-to-odd rounding of the softmax stages with k_smx_test=mix_fail on both. - A k_split value other than m or off fails create in one line. - The kernels line names split=m only when stage m is split. - k_smx_test=split_fail: create falls back to the one launch, one note names both paths, and the frames match k_split=off. The both_fail refusal names the split form's failure as well as the one-launch path's, and with k_split=off it does not. --- clean/k/tests/test_k_runtime_d3d12.py | 134 +++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 2 deletions(-) diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index d97c6cfa..a0f060da 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -81,6 +81,12 @@ def tonemapped_psnr(a, b): ALWAYS = ["k_expo_scalars", "k_out_pixel_c"] TILED = ["kx/k_in_tiled", "kx/k_in_tiled_uniform"] LEGACY = ["kx/k_in_color", "kx/k_in_base"] # (+ kx/k_in_reset on a reset frame) +ALT_SENTINEL = 0xC3 # the second native runtime of a native-against-native comparison (the first is D3D12_SENTINEL) + + +def opts(*parts): + """A runtime option string from the non-empty parts.""" + return ";".join(p for p in parts if p) def trace_since(pos): @@ -331,6 +337,11 @@ def test_native_softmax_refusals(self): self.assertFalse(both["ok"], both) self.assertIn("not exact on this driver", both["error"]) self.assertNotIn("\n", both["error"].strip()) + # with the split form on, the first refusal is the split form's and the one-launch path's is named too + self.assertIn("(the split form of stage m failed first:", both["error"]) + both_off = self.leg("k_smx_test=both_fail;k_split=off") + self.assertFalse(both_off["ok"], both_off) + self.assertNotIn("split form", both_off["error"]) # a copy of the build (its top-level files and dx12/) with one flipped byte in one stage shader: a softmax stage's # __rodd port and a fixed-weight stage (e0); create prepares every selectable kernel, so it refuses in one line for victim in ("k_stagew2_e3__rodd", "k_stage2_e0"): @@ -387,6 +398,124 @@ def test_native_fallback_frames_4k_performance(self): self.check_fallback_frames(frames, True, out=(3840, 2160), max_render=(1920, 1080)) self.check_fallback_frames(frames, True, split=False, out=(3840, 2160), max_render=(1920, 1080)) + # ---- deep-split: stage m's phase launches (k_split) on the native device; native against native, no HIP run ----- + def native_pair(self, left, right, frames, hdr=True, mode=0, half=True, out=OUT, max_render=MAX_RENDER): + """Two native runtimes of this build, options `left` and `right`, the same inputs for every frame. After every + frame the output and the written buffers, histories and exposure state are compared raw. Each side starts from + its own sentinel (0x5A and 0xC3, k_fill and before every frame), so a word neither side wrote fails.""" + auto = mode == 2 + a = self.make(3, hdr, auto, opts(left, f"k_fill={D3D12_SENTINEL}"), out, max_render) + b = self.make(3, hdr, auto, opts(right, f"k_fill={ALT_SENTINEL}"), out, max_render) + head = "k.N" if "k_head=full" in left + right else "k.Np" + try: + for i, fr in enumerate(frames): + rw, rh, reset, E, pi = fr[:5] + x = inputs(np.random.default_rng(i), rw, rh, hdr, reset, jitter=fr[5] if len(fr) > 5 else None) + x["pre_exposure"], x["E_cur"] = pi, float(np.float32(E)) + if i == 2: + x["mv_scale"], x["mv_offset"] = (0.75, 1.25), (0.125, -0.25) + for rt, byte in ((a, D3D12_SENTINEL), (b, ALT_SENTINEL)): + poke(rt, rt.bufs["out"], rt.bufs_size["out"], byte) + if i: + for name in WRITTEN + [head]: + p, n = rt.buffer_info(name) + poke(rt, p, n, byte) + a.frame(x, mode, half=half) + b.frame(x, mode, half=half) + compare_raw(f"frame {i}: output", out_bytes(a, half), out_bytes(b, half), + hip_sentinel=D3D12_SENTINEL, d3d12_sentinel=ALT_SENTINEL) + for name in WRITTEN + STATE + [head]: + compare_raw(f"frame {i}: {name}", a.buffer(name, np.uint8), b.buffer(name, np.uint8), + hip_sentinel=D3D12_SENTINEL, d3d12_sentinel=ALT_SENTINEL) + finally: + del a, b + + def native_kernels(self, options, frames, out=OUT, max_render=MAX_RENDER): + """The kernels a native runtime launches for these frames (the launch trace); no reference run.""" + rt = self.make(3, True, False, opts(options, f"k_fill={D3D12_SENTINEL}"), out, max_render) + pos = os.path.getsize(TRACE) if os.path.exists(TRACE) else 0 + ran = {} + try: + for i, fr in enumerate(frames): + rw, rh, reset, E, pi = fr[:5] + x = inputs(np.random.default_rng(i), rw, rh, True, reset) + x["pre_exposure"], x["E_cur"] = pi, float(np.float32(E)) + rt.frame(x, 0, half=True) + lines, pos = trace_since(pos) + for ln in lines: + parts = ln.split() + if parts and parts[-1] == "d3d12": + ran[parts[0]] = ran.get(parts[0], 0) + 1 + finally: + del rt + return ran + + ODD = dict(out=(201, 121), max_render=(131, 79)) + FRAMES_ODD = [(131, 79, True, 1.0, 1.0), (131, 79, False, 1.0, 1.0), (131, 79, False, 1.25, 0.5), + (101, 61, False, 1.0, 1.0), (101, 61, True, 0.75, 1.0)] + + def test_native_split_m_matches_one_launch(self): + """k_split=m (the default on the native device) against k_split=off: the same bytes of the output and of every + written buffer, history and exposure state, frame by frame, at the 200 x 120 output and at an odd output size + (201 x 121 from 131 x 79).""" + self.native_pair("", "k_split=off", self.FRAMES) + self.native_pair("", "k_split=off", self.FRAMES_ODD, **self.ODD) + + def test_native_split_m_auto_exposure_dense(self): + """The same parity in the dense RGB output with auto-exposure (the exposure kernels' own path).""" + self.native_pair("", "k_split=off", self.FRAMES, hdr=True, mode=2, half=False) + + def test_native_split_trace(self): + """Stage m's launches on the native device: the phase launches with k_split=m (the default), k_stagew4_m with + k_split=off; with the developer hook k_smx_test=mix_fail the rounding is the round-to-odd port in both.""" + on = self.native_kernels("", self.FRAMES_Q, **self.QUALITY) + phases = ["k_split_m_attn", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"] + self.assertTrue(all(on.get(k) for k in phases), sorted(on)) + self.assertFalse([k for k in on if k in ("k_stagew4_m", "k_stagew4_m__rodd")], sorted(on)) + off = self.native_kernels("k_split=off", self.FRAMES_Q, **self.QUALITY) + self.assertTrue(off.get("k_stagew4_m"), sorted(off)) + self.assertFalse([k for k in off if k.startswith("k_split_m_")], sorted(off)) + self.need_dev_build() + self.expect_fallback_trace(self.native_kernels("k_smx_test=mix_fail", self.FRAMES_Q, **self.QUALITY), False, True) + self.expect_fallback_trace(self.native_kernels("k_smx_test=mix_fail;k_split=off", self.FRAMES_Q, **self.QUALITY), + False, False) + + def test_native_k_split_refusals(self): + """A k_split value other than m or off fails create on the native device in one line.""" + for value in ("maybe", "M", "on", "2"): + with self.subTest(value=value): + got = self.leg("k_split=" + value) + self.assertFalse(got["ok"], got) + self.assertIn("k_split must be m or off", got["error"]) + self.assertNotIn("\n", got["error"].strip()) + + def test_native_split_kernels_line(self): + """The kernels line names the split form (split=m) only when stage m is split; the self-check form is unchanged.""" + plain, off = self.leg(""), self.leg("k_split=off") + self.assertTrue(plain["ok"], plain) + self.assertTrue(off["ok"], off) + self.assertEqual((plain["split"], off["split"]), ("m", "")) + self.assertEqual((plain["smx"], plain["notes"]), (off["smx"], off["notes"])) + explicit = self.leg("k_split=m") + self.assertEqual(explicit["split"], "m", explicit) + + def test_native_split_fallback(self): + """k_smx_test=split_fail (developer build): the split form of stage m fails the create-time self-check, so create + takes the one launch: create succeeds, the kernels line has no split=m, the rounding is the mix form, one note of + create names the split form and the one-launch path, and the frames are the same bytes as with k_split=off.""" + self.need_dev_build() + fb = self.leg("k_smx_test=split_fail") + self.assertTrue(fb["ok"], fb) + self.assertEqual((fb["split"], fb["smx"]), ("", "mix")) + self.assertEqual(len(fb["notes"]), 1, fb) + note = fb["notes"][0] + self.assertIn("split form of stage m", note) + self.assertIn("one launch", note) + self.assertIn("test hook", note) + self.assertNotIn("\n", note) + self.native_pair("k_smx_test=split_fail", "k_split=off", self.FRAMES) + self.native_pair("k_smx_test=split_fail", "k_split=off", self.FRAMES_ODD, **self.ODD) + # ---- k_approx: the opt-in close approximations (never the default) -------------------------------------------- def test_k_approx_refusals(self): """k_approx runs on the native D3D12 device only (HIP and the hybrid device refuse it), takes only its known @@ -595,16 +724,17 @@ def softmax_leg(mdir, map_path, options, build): try: rt = KRuntimeExt(build, mdir, map_path, OUT, MAX_RENDER, True, False, False, options, backend=3) except RuntimeError as e: - print(json.dumps({"ok": False, "smx": "", "notes": [], "k_approx": "", "error": str(e)})) + print(json.dumps({"ok": False, "smx": "", "notes": [], "k_approx": "", "split": "", "error": str(e)})) return rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] rt.lib.wg_rt_notes.restype = ctypes.c_char_p kern = rt.lib.wg_rt_kernels(rt.rt).decode().split() smx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("smx=")] approx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("k_approx=")] + split = [kv.split("=", 1)[1] for kv in kern if kv.startswith("split=")] notes = [n for n in rt.lib.wg_rt_notes(rt.rt).decode().split("\n") if n] print(json.dumps({"ok": True, "smx": smx[0] if smx else "", "notes": notes, "k_approx": approx[0] if approx else "", - "error": ""})) + "split": split[0] if split else "", "error": ""})) if __name__ == "__main__": From 6607365f1edb1f1865c62a77ceb5cd558886fe72 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:15:27 -0500 Subject: [PATCH 08/14] K deep-split: header carries only the generator's text The provenance lines added by hand to k_smx_selfcheck.h named a local path and the internal review process. The generator (clean/k/dx12/gen_k_smx_selfcheck.py) does not write them: its output for the same entries and digest is now byte-identical to the header. The provenance of the rebound digest is in its commit message. --- clean/k/runtime/k_smx_selfcheck.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index d019a35c..0447b48b 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -2,12 +2,6 @@ // Generated by clean/k/dx12/gen_k_smx_selfcheck.py; do not edit. The reference hashes (SHA-256 of the outputs, // first 8 bytes) of model K's softmax self-check cases at every possible window offset (runtime_k.cpp kSmxCases, // KRuntime::smx_select) on the HIP reference, and the source digest they belong to (smx_selfcheck_digest.py). -// kSourceDigest provenance (2026-11-18, deep-split fix round 2, oversight-approved rebind): the D3D12 report hook -// (runtime option k_smx_test=report, developer build with WG_DEV_DIAGNOSTICS) on the native D3D12 device (backend 3, -// no HIP) reproduced all 133 hashes below in both softmax forms (mix and round-to-odd), 0 mismatches, probe 0 / 0. -// Record: R:\WrongGPU-clean\tmp\kl-speed-K\deep-split\diag\smx-report-diag.json (and RECORD.md "Fix round 2"). -// The digest value is the one smx_selfcheck_digest.digest() computes for this tree's sources; the hashes are the HIP -// reference's, unchanged from the previous header. #pragma once #include From 7d1d157259a06e880618df60d954c4a2eb4d2a68 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:16:04 -0500 Subject: [PATCH 09/14] K deep-split: k_block_split.hlsl refuses tile sizes that do not divide HT The W2 batches (HT / KS_B2) and the W1 groups (HT / KS_HB) need HT = 4 KW_C / 16 divisible by both sizes; a #error now refuses any other choice at compile time. The comment ties the runtime's W1 grid (HT / 2 groups per window) to KS_HB = 2, so a change of KS_HB changes that launch as well. --- clean/k/dx12/kernels/k_block_split.hlsl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/clean/k/dx12/kernels/k_block_split.hlsl b/clean/k/dx12/kernels/k_block_split.hlsl index b7ef48d0..2387060e 100644 --- a/clean/k/dx12/kernels/k_block_split.hlsl +++ b/clean/k/dx12/kernels/k_block_split.hlsl @@ -53,6 +53,12 @@ WG_ARGS_END #ifndef KS_HB #define KS_HB 2 // hidden tiles per workgroup (k_split_m_w1) #endif +// The hidden tiles HT = 4 KW_C / 16 are split into the W2 batches (HT / KS_B2) and the W1 groups (HT / KS_HB). The +// runtime's W1 launch (runtime_k.cpp, split_m) runs HT / 2 groups per window: that is this grid only for KS_HB = 2, so a +// change of KS_HB must change that launch too. +#if (4 * KW_C / 16) % KS_B2 != 0 || (4 * KW_C / 16) % KS_HB != 0 +#error "k_block_split.hlsl: KS_B2 and KS_HB must divide HT = 4 KW_C / 16 (the W2 batches and the W1 groups)" +#endif // Operand fence: the D3D12 driver's scheduler moves each global load of a short, low-pressure chain down to its // product (one memory latency per product, measured in the RGA ISA of the first build). A phase therefore loads a From c9bb052275f75570e2fd527cc95a6f6df6bc19a4 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:16:09 -0500 Subject: [PATCH 10/14] K deep-split: the split scratch is allocated before the k_fill block split_alloc now runs before the k_fill block, and the four scratch buffers (Z, X1, A, H) join the fill list, so k_fill's sentinel covers them as it covers the rest of what a frame writes. Their byte sizes are recorded at allocation (split_bytes_). The W1 launch carries a comment that ties its HT / 2 groups per window to KS_HB = 2 in the kernel source. --- clean/k/runtime/runtime_k.cpp | 15 ++++++++++++--- clean/k/runtime/runtime_k.h | 1 + 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 6f7c3684..b0d2ab7b 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -677,6 +677,7 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e return fail("model K: an input-stage buffer is not allocated (it would be packed as arena offset 0)"); // the own output buffer (tests): dense RGB f32, dense RGBA f32 (FrameIo.out_rgba32f) or RGBA16F rows that fit if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 16, true)) return false; + if (split_m_ && !split_alloc()) return false; // k_split's scratch (filled with the rest by k_fill, below) if (fill_ >= 0) { // k_fill (test hook): the sentinel in everything a frame writes, before the first frame auto fillb = [&](DevPtr p, size_t bytes) { return !p || dev_->memset(p, fill_, std::max(bytes, 256)); }; bool ok = true; @@ -686,6 +687,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e ok = ok && fillb(ctap_, R * 8) && fillb(ctapT_, R * 2) && fillb(cpre_, R * 16) && fillb(cpre2_, R * 8) && fillb(xs_, 16) && fillb(kexpo_, 16) && fillb(color_, R * 16) && fillb(depth_, R * 4) && fillb(motion_, R * 8); // the own input copies (unused with caller buffers) + if (split_m_) // k_split's scratch: written by the phases of stage m, so filled like the rest + ok = ok && fillb(split_z_, split_bytes_[0]) && fillb(split_x1_, split_bytes_[1]) && + fillb(split_a_, split_bytes_[2]) && fillb(split_h_, split_bytes_[3]); if (!ok) return dfail(); } // the filter-shape table (spec 43 section 7) of every level, computed once; a frame reads its level's set @@ -693,7 +697,6 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e for (int l = 0; l < 8; ++l) k_filter_table(l, 1, tab.data() + (size_t)l * kFilterEntries * kFilterValues); if (!dev_->upload(tables_, tab.data(), tab.size() * 2)) return dfail(); if (!dev_->sync()) return dfail(); - if (split_m_ && !split_alloc()) return false; // the native device prepares (loads, verifies and compiles) every kernel this context can select now: a missing or // corrupt shader fails create here, not at the first frame if (c.backend == 3) { @@ -1021,6 +1024,8 @@ bool KRuntime::split_m(bool rodd, void* params) { if (nwin > split_cap_) return fail("model K: k_split: stage m has " + std::to_string(nwin) + " windows, the scratch holds " + std::to_string(split_cap_)); + // k_split_m_w1 runs HT / 2 groups per window: one group per two hidden tiles (KS_HB = 2 in + // clean/k/dx12/kernels/k_block_split.hlsl). A change of KS_HB must change this grid too. return launch(rodd ? "k_split_m_attn__rodd" : "k_split_m_attn", (unsigned)(nwin * st.heads), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && launch("k_split_m_wo", (unsigned)(nwin * kt), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && @@ -1046,8 +1051,12 @@ bool KRuntime::split_alloc() { if (std::strcmp(sc.stage, "m") == 0) grow(sc.wb, sc.hb); split_cap_ = cap; const size_t tok = (size_t)cap * 64; - return alloc(&split_z_, tok * st.heads * 32 * 2) && alloc(&split_x1_, tok * st.C * 2) && - alloc(&split_a_, tok * st.C * 2) && alloc(&split_h_, tok * 4 * st.C * 2); + split_bytes_[0] = tok * st.heads * 32 * 2; // Z + split_bytes_[1] = tok * st.C * 2; // X1 + split_bytes_[2] = tok * st.C * 2; // A + split_bytes_[3] = tok * 4 * st.C * 2; // H + return alloc(&split_z_, split_bytes_[0]) && alloc(&split_x1_, split_bytes_[1]) && + alloc(&split_a_, split_bytes_[2]) && alloc(&split_h_, split_bytes_[3]); } // SMX-SELFCHECK-SOURCE END diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index d1a0c1a8..a1d3540a 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -195,6 +195,7 @@ class KRuntime { bool split_m_ = false; int split_cap_ = 0; DevPtr split_z_ = 0, split_x1_ = 0, split_a_ = 0, split_h_ = 0; + size_t split_bytes_[4] = {0, 0, 0, 0}; // the bytes of split_z_, split_x1_, split_a_, split_h_ (split_alloc) std::string smx_note_; std::string smx_test_; // option k_smx_test (developer builds only) std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty From af642648c24d5fc83d9428752bfd01551b3971da Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:16:19 -0500 Subject: [PATCH 11/14] K deep-split: ISA check covers the split stage m's wo, w1 and w2 phases The three phases have no softmax, so they must have no v_fma_mix. The check records that, next to the attention phase's count. --- clean/dx12/isa_checks.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clean/dx12/isa_checks.json b/clean/dx12/isa_checks.json index 1d8cf87b..44840ee6 100644 --- a/clean/dx12/isa_checks.json +++ b/clean/dx12/isa_checks.json @@ -8,6 +8,9 @@ "k_stagew2_e4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew4_m": {"v_fma_mix(lo|hi)_f16": 64}, "k_split_m_attn": {"v_fma_mix(lo|hi)_f16": 32}, + "k_split_m_wo": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_w1": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_w2": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_d4": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew2_d4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew2_d3": {"v_fma_mix(lo|hi)_f16": 64}, From af123009a59fc31cf7bcf83b982dafbc879a7ffe Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:16:19 -0500 Subject: [PATCH 12/14] K deep-split: README states the window counts at the sizes that are measured Stage m's window count is 6 to 24 at 1080p and 1440p output, more at 4K and in the self-check; the sentence said only the first range. --- clean/k/dx12/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 01bc8a45..ffa259df 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -115,7 +115,7 @@ Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs a ## Phase-split stage m (runtime option `k_split`, native D3D12 device) -`kernels/k_block_split.hlsl` runs stage m (`k_stagew4_m`: 6 to 24 windows, one 512-lane workgroup each) as four launches +`kernels/k_block_split.hlsl` runs stage m (`k_stagew4_m`: 6 to 24 windows at 1080p/1440p output (more at 4K and in the self-check), one 512-lane workgroup each) as four launches spread over many more workgroups: `k_split_m_attn` (one per window and head: normalization, q / k / v, softmax, z; `__rodd` twin), `k_split_m_wo` (one per window and output tile: Wo, x1, a), `k_split_m_w1` (one per window and two hidden tiles: W1, activation) and `k_split_m_w2` (one per window and output tile: W2, x2, the block output), joined by the From ad46b02eea25bb58f5049157da3e05498033a149 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:29:01 -0500 Subject: [PATCH 13/14] K deep-split: rebind the self-check digest to the reviewed tree kSourceDigest moves from 1a51cb9d... to ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b, the value smx_selfcheck_digest.digest() gives for this tree (stale() is None). Only the digest line changes: the 133 names and the reference hashes are unchanged. Provenance: dev-diagnostics runtime built from this tree (DLL sha256 b8b97de8...), k_smx_test=report on backend 3 (D3D12, native, no HIP), record kl-speed-K deep-split smx-report-r3, 2026-11-18: - mix: 133/133 equal to the HIP reference hashes, 0 mismatches - rodd: 133/133 equal to the HIP reference hashes, 0 mismatches --- clean/k/runtime/k_smx_selfcheck.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index 0447b48b..01c8aaf0 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "1a51cb9d0f4f5b5e8d1acd833f814ff7457a169416d453692e3c449f75688bac"; +constexpr char kSourceDigest[] = "ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", From 4cac84b9b2aba37d40817a813e34be100345c575 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Wed, 18 Nov 2026 14:40:43 -0500 Subject: [PATCH 14/14] K deep-split: raw compare covers buffers of any byte length compare_raw required a multiple of 4 bytes, so the native odd-size legs of test_native_split_m_matches_one_launch and test_native_split_fallback stopped at k.ctapT (2 bytes per render pixel: 2 x 131 x 79 = 20698 bytes) before any state or head buffer was compared. The whole 4-byte words are still compared as uint32; the 1 to 3 trailing bytes are compared byte for byte under the same sentinel rules. Equal data still passes; a changed trailing byte, an untouched trailing byte (counted) and a size mismatch fail. CPU tests in test_k_raw_compare.py cover odd lengths 1, 2, 3 mod 4 and 20698: equal passes, a differing last byte and first trailing byte fail, a trailing byte neither side wrote is counted and limited, a write only on D3D12 fails, a size mismatch fails. --- clean/k/tests/raw_compare.py | 48 +++++++++++++++-------- clean/k/tests/test_k_raw_compare.py | 61 +++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 16 deletions(-) diff --git a/clean/k/tests/raw_compare.py b/clean/k/tests/raw_compare.py index 4fcbd853..5d020044 100644 --- a/clean/k/tests/raw_compare.py +++ b/clean/k/tests/raw_compare.py @@ -3,8 +3,10 @@ Before a frame every output and intermediate buffer is filled with a sentinel byte: 0xA5 on HIP, 0x5A on D3D12. After it, the two buffers are compared as 4-byte words of raw bit patterns (no float conversion: an FP16 NaN keeps its -payload, -0 stays -0). A word that differs is a failure, except a word that neither side wrote (HIP's sentinel on one -side and D3D12's on the other): those are counted, and a caller that knows the buffer is written in full allows none. +payload, -0 stays -0). A buffer whose size is not a multiple of 4 (ctapT is 2 bytes a render pixel, so an odd render +size gives one) has its 1 to 3 trailing bytes compared byte for byte under the same rules, so every byte is checked. +A unit that differs is a failure, except a unit that neither side wrote (HIP's sentinel on one side and D3D12's on the +other): those are counted, and a caller that knows the buffer is written in full allows none. A write that D3D12 dropped leaves its sentinel where HIP stored a value, and a write that only D3D12 made leaves HIP's sentinel: both fail, which the negative controls in test_k_raw_compare.py show. """ @@ -18,21 +20,35 @@ def sentinel_word(byte): return np.uint32(byte * 0x01010101) -def compare_raw(what, hip, d3d12, allow_untouched=0, hip_sentinel=HIP_SENTINEL, d3d12_sentinel=D3D12_SENTINEL): - """hip, d3d12: bytes-like of equal length (a multiple of 4). Raises AssertionError on a differing word or on more - than allow_untouched words written by neither; returns the number of such words.""" - hip, d3d12 = bytes(hip), bytes(d3d12) - if len(hip) != len(d3d12) or len(hip) % 4: - raise AssertionError(f"{what}: sizes {len(hip)} and {len(d3d12)} (need equal, a multiple of 4)") - h, d = np.frombuffer(hip, np.uint32), np.frombuffer(d3d12, np.uint32) +def _compare_units(what, h, d, hip_sentinel, d3d12_sentinel, unit, base, width): + """h, d: arrays of equal length of one unit (words or bytes); base: index of the first unit in the buffer (bytes, for + the trailing bytes). Returns (units written by neither side, units compared); raises on a differing unit.""" differ = h != d - untouched = differ & (h == sentinel_word(hip_sentinel)) & (d == sentinel_word(d3d12_sentinel)) + untouched = differ & (h == hip_sentinel) & (d == d3d12_sentinel) bad = differ & ~untouched if bad.any(): i = int(np.flatnonzero(bad)[0]) - raise AssertionError(f"{what}: {int(bad.sum())} of {len(h)} words differ; first at word {i}: HIP {h[i]:08x}, " - f"D3D12 {d[i]:08x}") - n = int(untouched.sum()) - if n > allow_untouched: - raise AssertionError(f"{what}: {n} of {len(h)} words written by neither side (allowed {allow_untouched})") - return n + raise AssertionError(f"{what}: {int(bad.sum())} of {len(h)} {unit}s differ; first at {unit} {base + i}: " + f"HIP {int(h[i]):0{width}x}, D3D12 {int(d[i]):0{width}x}") + return int(untouched.sum()), len(h) + + +def compare_raw(what, hip, d3d12, allow_untouched=0, hip_sentinel=HIP_SENTINEL, d3d12_sentinel=D3D12_SENTINEL): + """hip, d3d12: bytes-like of equal length, any length. Raises AssertionError on a differing word or byte, or on more + than allow_untouched units written by neither side; returns the number of such units (words and trailing bytes).""" + hip, d3d12 = bytes(hip), bytes(d3d12) + if len(hip) != len(d3d12): + raise AssertionError(f"{what}: sizes {len(hip)} and {len(d3d12)} (need equal)") + whole = len(hip) - len(hip) % 4 + untouched, total = _compare_units( + what, np.frombuffer(hip[:whole], np.uint32), np.frombuffer(d3d12[:whole], np.uint32), + sentinel_word(hip_sentinel), sentinel_word(d3d12_sentinel), "word", 0, 8) + if whole < len(hip): + n, m = _compare_units( + what, np.frombuffer(hip[whole:], np.uint8), np.frombuffer(d3d12[whole:], np.uint8), + hip_sentinel, d3d12_sentinel, "byte", whole, 2) + untouched, total = untouched + n, total + m + if untouched > allow_untouched: + raise AssertionError(f"{what}: {untouched} of {total} units (words, trailing bytes) written by neither side " + f"(allowed {allow_untouched})") + return untouched diff --git a/clean/k/tests/test_k_raw_compare.py b/clean/k/tests/test_k_raw_compare.py index e0a74878..46bc6b27 100644 --- a/clean/k/tests/test_k_raw_compare.py +++ b/clean/k/tests/test_k_raw_compare.py @@ -61,6 +61,67 @@ def test_size_mismatch_fails(self): with self.assertRaises(AssertionError): compare_raw("size", h.tobytes(), d[:-1].tobytes()) + def odd_bytes(self, n, seed=7): + """Ordinary bytes of length n (no sentinel byte, so every byte is data).""" + rng = np.random.default_rng(seed) + b = rng.integers(0, 256, n, dtype=np.uint8) + b[(b == HIP_SENTINEL) | (b == D3D12_SENTINEL)] = 1 + return b + + def test_odd_length_equal_passes(self): + """A buffer whose size is not a multiple of 4 compares byte for byte: 2 x 131 x 79 = 20698 bytes (ctapT at the + odd render size) and the other trailing-byte lengths 1, 2 and 3 mod 4.""" + for n in (1, 2, 3, 6, 4 * 5 + 1, 4 * 5 + 3, 2 * 131 * 79): + with self.subTest(n=n): + b = self.odd_bytes(n) + self.assertEqual(compare_raw("odd", b.tobytes(), b.copy().tobytes()), 0) + + def test_odd_length_last_byte_differs_fails(self): + """A change in the last byte of an odd-length buffer (the trailing bytes are compared, not skipped).""" + for n in (4 * 5 + 1, 4 * 5 + 2, 4 * 5 + 3, 2 * 131 * 79): + with self.subTest(n=n): + h = self.odd_bytes(n) + d = h.copy() + d[-1] ^= 1 + with self.assertRaisesRegex(AssertionError, f"differ; first at byte {n - 1}: HIP"): + compare_raw("odd-last", h.tobytes(), d.tobytes()) + + def test_odd_length_first_trailing_byte_differs_fails(self): + """The first trailing byte (the first byte after the last whole word) is compared too.""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + d[(n // 4) * 4] ^= 0x80 + with self.assertRaisesRegex(AssertionError, f"first at byte {(n // 4) * 4}"): + compare_raw("odd-tail0", h.tobytes(), d.tobytes()) + + def test_odd_length_untouched_trailing_byte_counted(self): + """A trailing byte neither side wrote (HIP's sentinel on one side, D3D12's on the other) is counted like a word; + a full buffer allows none.""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + h[-1], d[-1] = HIP_SENTINEL, D3D12_SENTINEL + self.assertEqual(compare_raw("odd-pad", h.tobytes(), d.tobytes(), allow_untouched=1), 1) + with self.assertRaisesRegex(AssertionError, "neither"): + compare_raw("odd-pad", h.tobytes(), d.tobytes()) + + def test_odd_length_write_only_on_d3d12_fails(self): + """A trailing byte D3D12 wrote where HIP left its sentinel fails (the sentinel pair is not the only way to pass).""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + h[-2] = HIP_SENTINEL + d[-2] = 9 + with self.assertRaisesRegex(AssertionError, "differ"): + compare_raw("odd-extra", h.tobytes(), d.tobytes()) + + def test_odd_length_size_mismatch_fails(self): + n = 2 * 131 * 79 + h = self.odd_bytes(n) + with self.assertRaisesRegex(AssertionError, "sizes"): + compare_raw("odd-size", h.tobytes(), h[:-1].tobytes()) + if __name__ == "__main__": unittest.main()