diff --git a/clean/dx12/isa_checks.json b/clean/dx12/isa_checks.json index c9c4a79e..44840ee6 100644 --- a/clean/dx12/isa_checks.json +++ b/clean/dx12/isa_checks.json @@ -1,5 +1,5 @@ { - "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", + "_doc": "build_dx12.py ISA check: for each kernel listed, the RGA (gfx1201) ISA must contain exactly the given number of instructions whose mnemonic matches each pattern (a regular expression, matched at the start of the mnemonic). Model K's softmax weights (clean/k/dx12/hlsl/k_smx.hlsli) are exact only when the driver folds the mix form p + e into one v_fma_mixlo/hi_f16 rounding per weight: 32 weights per lane per softmax round, i.e. 32 x heads x windows per workgroup for the narrow stages (e2 / d2: 2 heads, 1 window; d1: 2 heads, 2 windows) and 32 x heads / NG for the wide stages (e3 / d3 and the NG = 2 e4 / d4: 2 rounds; NG = 4 e4 / d4: 1; m: 8 heads / NG 4 = 2; the phase-split m attention k_split_m_attn: one head per workgroup = 1). A missing fold leaves v_cvt_f16_f32 after an FP32 add (a second rounding) and fails here. The __rodd ports (round to odd) must have none. The native D3D12 runtime checks the same fold on the player's driver at create (k_mix_probe). WG_DX12_ISA_CHECK=0 skips the check.", "k_stage_e2": {"v_fma_mix(lo|hi)_f16": 64}, "k_stage_d2": {"v_fma_mix(lo|hi)_f16": 64}, "k_stage2_d1": {"v_fma_mix(lo|hi)_f16": 128}, @@ -7,6 +7,10 @@ "k_stagew4_e4": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew2_e4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew4_m": {"v_fma_mix(lo|hi)_f16": 64}, + "k_split_m_attn": {"v_fma_mix(lo|hi)_f16": 32}, + "k_split_m_wo": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_w1": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_w2": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_d4": {"v_fma_mix(lo|hi)_f16": 32}, "k_stagew2_d4": {"v_fma_mix(lo|hi)_f16": 64}, "k_stagew2_d3": {"v_fma_mix(lo|hi)_f16": 64}, @@ -18,6 +22,7 @@ "k_stagew4_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_e4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_m__rodd": {"v_fma_mix(lo|hi)_f16": 0}, + "k_split_m_attn__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew4_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_d4__rodd": {"v_fma_mix(lo|hi)_f16": 0}, "k_stagew2_d3__rodd": {"v_fma_mix(lo|hi)_f16": 0}, diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 07fe9ce0..c15b4c92 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -464,6 +464,26 @@ "k_out_pixel_direct", "k_io.co" ], + [ + "k_split_m_attn", + "k_block_wide.co" + ], + [ + "k_split_m_attn__rodd", + "k_block_wide.co" + ], + [ + "k_split_m_w1", + "k_block_wide.co" + ], + [ + "k_split_m_w2", + "k_block_wide.co" + ], + [ + "k_split_m_wo", + "k_block_wide.co" + ], [ "k_stage2_d0_hidden", "k_block.co" diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index a02f8a4f..ffa259df 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -97,7 +97,7 @@ refuse a header whose digest differs; `WG_DX12_SMX_DIGEST=0` lets a negative-con `build.py` never embeds a test-only, bypassed, define-built, unprovenanced or stale entry, and the payload packager refuses a library whose build manifest does not record that check (`clean/tests/test_dev_diagnostics_off.py`). -The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `report`) exists only in runtime libraries built +The test hook `k_smx_test` (`mix_fail`, `both_fail`, `rodd_unchecked`, `split_fail`, `report`) exists only in runtime libraries built with `build.py --dev-diagnostics`; a release build refuses it (`test_k_runtime_d3d12.py`: `test_native_softmax_switch_is_logged`, `test_native_softmax_refusals`, `test_native_fallback_frames_*`). Negative controls, never shipped: `K_SMX_MUT=1` (the mix form's residual dropped), `K_SMX_MUT4=1` (dropped only at window offset @@ -113,6 +113,24 @@ identical. Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero and HIP keeps them. `k_expo_scalars` only meets one with a pre-exposure above about 8.5e37, so the check's inputs keep clear of them. +## Phase-split stage m (runtime option `k_split`, native D3D12 device) + +`kernels/k_block_split.hlsl` runs stage m (`k_stagew4_m`: 6 to 24 windows at 1080p/1440p output (more at 4K and in the self-check), one 512-lane workgroup each) as four launches +spread over many more workgroups: `k_split_m_attn` (one per window and head: normalization, q / k / v, softmax, z; +`__rodd` twin), `k_split_m_wo` (one per window and output tile: Wo, x1, a), `k_split_m_w1` (one per window and two hidden +tiles: W1, activation) and `k_split_m_w2` (one per window and output tile: W2, x2, the block output), joined by the +dispatch boundaries. Every accumulator is the same WMMA chain (operands, order, initial value) as in `k_stagew4_m` and every +FP16 value handed between the launches is the one `k_stagew4_m` keeps at that point, so the output bytes are the same. +Default on the native D3D12 device (`k_split=m`; `k_split=off`: the one launch; other devices: the one launch). The runtime +redirects `k_stagew4_m[__rodd]` in `KRuntime::launch`, so the create-time self-check runs the split form for its m case +against the unchanged HIP reference hashes (kernels line `split=m`). The self-check digest (`smx_selfcheck_digest.py`) +covers the phase kernels and the runtime's `k_split` option, redirect and phase launches. `k_split` is parsed on every +device: a value other than `m` or `off` fails create there too. + +If the create-time self-check fails on the split form of stage m, create does not refuse: stage m takes the one launch, +the self-check runs again on that path, and one note of create says so (the output is the same). Create refuses only +when the one-launch path fails the self-check too. The developer hook `k_smx_test=split_fail` forces the first failure. + ## Close approximations (runtime option `k_approx`, opt-in) Every port above is exact, and nothing set is the exact path. `k_approx` selects close approximations of two stages on diff --git a/clean/k/dx12/kernels/k_block_split.hlsl b/clean/k/dx12/kernels/k_block_split.hlsl new file mode 100644 index 00000000..2387060e --- /dev/null +++ b/clean/k/dx12/kernels/k_block_split.hlsl @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: MIT +// Phase-split form of model K's wide stage m (deep-split lever, probe: stage m only). WrongGPU's own code. D3D12 only: +// the same arithmetic as k_stagew4_m (kernels/k_block_wide.hlsl, hlsl/k_wide.hlsli; HIP: k_block_wide.co), whose one +// workgroup per window runs every phase of the block (normalization, q / k / v, attention, Wo, MLP) in sequence on 16 +// waves. At 6 .. 24 windows that leaves most CUs idle and the window's dependent chain of weight loads and WMMAs sets the +// time. Here the window's phases run as four launches, each spread over many more workgroups, joined by the dispatch +// boundaries (the device's UAV barrier between launches): +// k_split_m_attn one workgroup (4 waves) per (window, head): normalization, q / k / v of the head, scores, softmax, +// z of the head -> Z. Every head's K / V is computed once (no redundant K / V). +// k_split_m_wo one workgroup per (window, output tile): y = Wo z + bo over the z tiles in order, x1 = f16(x + y), +// a = f16(x1 g2) -> X1, A. +// k_split_m_w1 one workgroup per (window, KS_HB hidden tiles): h = act(f16(W1 a + b1)) -> H. +// k_split_m_w2 one workgroup per (window, output tile): x2 = f16(x1 + f16(W2 h + b2)) over the hidden tiles in +// order, stored for the real positions of the block output map. +// Wave wv of every workgroup holds the window's tokens 16 wv .. 16 wv + 15 (lane: token 16 wv + lane % 16, channel half +// lane / 16), as in k_stagew4_m. Exactness: every accumulator is the same chain of v_wmma_f32_16x16x16_f16 with the same +// A / B operands in the same order from the same initial value as in k_stagew4_m (bias first; q / k / v over kt; scores +// bias then ct; z over st; Wo over the z tiles 0 .. DT - 1; W1 over kt; W2 over the hidden tiles 0 .. HT - 1), and every +// FP16 value handed between the phases (z, x1, a, h) is the FP16 value k_stagew4_m keeps in LDS or registers at that +// point (the driver's RNE matrix conversion, then the same FP16 operations): the output bytes are k_stagew4_m's. The +// helpers (normalization operations, softmax polynomial, weights k_smx.hlsli, activation, mirror) are k_wide.hlsli's +// and k_smx.hlsli's, unchanged. The create-time self-check runs this form for the m case when the runtime takes it +// (KRuntime::launch redirects k_stagew4_m), against the same HIP reference hashes (k_smx_selfcheck.h). +// Scratch (runtime_k.cpp split_*_, FP16, per window w and token t): Z [w][64][DT 16], X1 / A [w][64][C], H [w][64][HT 16]. +// Arguments: KBlkParams by value (s184:p19.b32, the stage's own), then the four scratch pointers. +// @wg-kernel k_split_m_attn co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_attn defines=KS_PHASE=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_attn__rodd co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_attn defines=K_SMX=1,KS_PHASE=1,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_wo co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_wo defines=KS_PHASE=2,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_w1 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w1 defines=KS_PHASE=3,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +// @wg-kernel k_split_m_w2 co=k_block_wide.co block=128 wave=32 args=s184:p19.b32,p,p,p,p entry=ks_w2 defines=KS_PHASE=4,KW_C=160,KW_HEADS=8,KW_IN=0,KW_CIN=160,KW_CNEXT=0,KW_NG=4 +#include "wg_platform.hlsli" + +WG_ARGS_BEGIN + uint P_x, P_skip, P_y, P_merged, P_head, P_g1, P_g2, P_wqkv, P_mix, P_wo, P_bo, P_w1, P_b1, P_w2, P_b2, P_wa, P_ba, + P_wz, P_bz; // pointers (arena offsets; 0 = null) + int P_w, P_h, P_ox, P_oy, P_nwx, P_nwy, P_cw, P_ch; + uint P_sz, P_sx1, P_sa, P_sh; // the split scratch: Z, X1, A, H +WG_ARGS_END + +#include "k_wide.hlsli" +#include "k_smx.hlsli" + +#if KW_IN != 0 || KW_CNEXT != 0 +#error "k_block_split.hlsl: map input, no merge (stage m) only" +#endif + +#ifndef KS_RD +#define KS_RD 20 // weight ring depth of the q / k / v chains (k_split_m_attn) +#endif +#ifndef KS_B2 +#define KS_B2 20 // W2 products per fenced operand batch (k_split_m_w2) +#endif +#ifndef KS_HB +#define KS_HB 2 // hidden tiles per workgroup (k_split_m_w1) +#endif +// The hidden tiles HT = 4 KW_C / 16 are split into the W2 batches (HT / KS_B2) and the W1 groups (HT / KS_HB). The +// runtime's W1 launch (runtime_k.cpp, split_m) runs HT / 2 groups per window: that is this grid only for KS_HB = 2, so a +// change of KS_HB must change that launch too. +#if (4 * KW_C / 16) % KS_B2 != 0 || (4 * KW_C / 16) % KS_HB != 0 +#error "k_block_split.hlsl: KS_B2 and KS_HB must divide HT = 4 KW_C / 16 (the W2 batches and the W1 groups)" +#endif + +// Operand fence: the D3D12 driver's scheduler moves each global load of a short, low-pressure chain down to its +// product (one memory latency per product, measured in the RGA ISA of the first build). A phase therefore loads a +// product group's operands, XORs one word of each into f and XORs KS_FENCE(f) into its first operand (the bias of the +// accumulator, or the first fragment): kw_zero() is 0 at +// run time (grids are below 2^31) but unknown to the compiler, so every load is issued before the group's first product +// and waited once. No bit of any operand changes. +#define KS_FENCE(f) ((f) & kw_zero()) + +static const int SA = 2 * KT * 3; // q / k / v products of one head +static const int NHG = HT / KS_HB; // hidden-tile groups of a window + +// The lane's window geometry (as kw_stage): token t, the mirrored member it reads, whether it is a real position. +struct KsLane { + uint lane, lane8; + int wv, n16, hh, t, gx, gy; + bool real; + uint mtok; +}; +KsLane ks_lane(int win) +{ + KsLane L; + const uint tid = wg_lane_id(); + L.lane = tid & 31u; + L.lane8 = L.lane * 8u; + L.wv = wg_uniform((int)(tid >> 5)); + L.n16 = (int)(L.lane & 15u); + L.hh = (int)(L.lane >> 4); + L.t = 16 * L.wv + L.n16; + const int wa = win % P_nwx, wb = win / P_nwx; + L.gx = 8 * wa - P_ox + (L.t & 7); + L.gy = 8 * wb - P_oy + (L.t >> 3); + L.real = L.gx >= 0 && L.gx < P_w && L.gy >= 0 && L.gy < P_h; + L.mtok = (uint)(kw_mirror(L.gy, P_h) * P_w + kw_mirror(L.gx, P_w)); // mirrored member (spec 41) + return L; +} + +// scratch element offsets (halfs) of token t of window win, channel half hh +uint ks_zrow(int win, int t, int hh) { return (uint)((win * 64 + t) * DT * 16 + 8 * hh); } +uint ks_crow(int win, int t, int hh) { return (uint)((win * 64 + t) * KW_C + 8 * hh); } +uint ks_hrow(int win, int t, int hh) { return (uint)((win * 64 + t) * HT * 16 + 8 * hh); } + +#if KS_PHASE == 1 +// ---- k_split_m_attn: one (window, head) ------------------------------------------------------------------------------ +// LDS: K [64 tokens][4 chunks] and VT [32 channels][8 chunks] of the head, 16-byte chunks xor-swizzled (k_wide.hlsli's +// NG = 4 layout of one group); 8 KB. +groupshared uint4 ks_lds[512]; +uint ks_kidx(int row, int c) { return (uint)(row * 4 + (c ^ (row & 3))) & 511u; } +uint ks_vidx(int row, int c) { return (uint)(256 + row * 8 + (c ^ (row & 7))) & 511u; } + +static uint ksA_qkv; +// product s = (ct KT + kt) 3 + {q, k, v} of the head's q / k / v chains (k_block_wide.hlsl's stream order of a round) +uint4 ks_wq(int s) { return wg_ld128(ksA_qkv, (uint)(((s % 3) * 2 * KT + s / 3) * 512)); } +#define KS_TAKE(s) rg[(s) % KS_RD] +#define KS_NEXT(s) { if ((s) + KS_RD < SA) rg[(s) % KS_RD] = ks_wq((s) + KS_RD); } + +WG_KERNEL(ks_attn) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KW_HEADS, hd = bid % KW_HEADS; + const KsLane L = ks_lane(win); + const int t = L.t, wv = L.wv, n16 = L.n16, hh = L.hh; + ksA_qkv = P_wqkv + 2u * ((uint)(hd * 3 * 2 * KT * 256) + L.lane8); + uint4 rg[KS_RD]; + + // the block input (all channels of the token) and g1, then the weight ring + uint4 n[KT]; + uint4 g1v[KT]; + { + const uint xt = L.mtok * KW_C + 8u * hh; + [unroll] for (int kt = 0; kt < KT; ++kt) n[kt] = kw_ldg(P_x, xt + 16u * kt); + [unroll] for (int kt4 = 0; kt4 < KT; ++kt4) g1v[kt4] = kw_ldg(P_g1, (uint)(16 * kt4 + 8 * hh)); + [unroll] for (int s0 = 0; s0 < KS_RD; ++s0) rg[s0] = ks_wq(s0); + uint f = 0u; // (the chain starts with the last loads) + [unroll] for (int s1 = KS_RD - 1; s1 >= 0; --s1) f ^= rg[s1].x; + [unroll] for (int kt5 = KT - 1; kt5 >= 0; --kt5) f ^= g1v[kt5].x ^ n[kt5].x; + n[0].x ^= KS_FENCE(f); // the input, g1 and the ring in one wait + } + + // normalization (k_block_wide.hlsl step 1): s = f16(sum f16(x^2)), r = f16(rsq(s)); s = 0 gives r = 0 + { + float ss = 0.0f; + [unroll] for (int kt = 0; kt < KT; ++kt) { + const uint4 sq = kw_opq4(kw_mul8(n[kt], n[kt])); + [unroll] for (uint i = 0; i < 4; ++i) { + ss += f16tof32(sq[i] & 0xffffu); + ss += f16tof32(sq[i] >> 16); + } + } + ss += wg_xor16(ss); + const uint s16 = wg_f2h(ss); + const uint r16 = (s16 & 0x7fffu) == 0u ? 0u : wg_f2h(rsqrt(f16tof32(s16))); + const uint4 rv = kw_splat8(r16); + [unroll] for (int kt = 0; kt < KT; ++kt) + n[kt] = kw_mul8(n[kt], kw_opq4(kw_mul8(rv, g1v[kt]))); + } + + // q, k, v^T of head hd (k_block_wide.hlsl step 2, one round's group) + uint4 q[2], kk[2], vv[2]; + [unroll] for (int ct = 0; ct < 2; ++ct) { + v8f aq = wg_splat(0.0f), ak = wg_splat(0.0f), av = wg_splat(0.0f); + [unroll] for (int kt = 0; kt < KT; ++kt) { + const int s = (ct * KT + kt) * 3; + aq = kw_mma(KS_TAKE(s), n[kt], aq); + KS_NEXT(s); + ak = kw_mma(KS_TAKE(s + 1), n[kt], ak); + KS_NEXT(s + 1); + av = kw_mma(n[kt], KS_TAKE(s + 2), av); + KS_NEXT(s + 2); + } + q[ct] = kw_out(aq); + kk[ct] = kw_out(ak); + vv[ct] = kw_out(av); + } + const uint mrow = (uint)((hd * 64 + t) * 64 + 8 * hh); + uint4 bm[4]; + [unroll] for (int st0 = 0; st0 < 4; ++st0) bm[st0] = kw_ldg(P_mix, mrow + 16u * st0); + [unroll] for (int ct1 = 0; ct1 < 2; ++ct1) { + ks_lds[ks_kidx(t, 2 * ct1 + hh)] = kk[ct1]; // K[token t][channels] + ks_lds[ks_vidx(16 * ct1 + n16, 2 * wv + hh)] = vv[ct1]; // VT[channel][tokens] + } + wg_barrier_lds(); // K / VT of the head complete + + // scores^T = K q + B, the clamped polynomial, softmax weights A (k_block_wide.hlsl, the same order) + uint4 pa[4]; + { + uint4 kf[4][2]; + [unroll] for (int st = 0; st < 4; ++st) + [unroll] for (int ct = 0; ct < 2; ++ct) kf[st][ct] = ks_lds[ks_kidx(16 * st + n16, 2 * ct + hh)]; + v8f sc[4]; + [unroll] for (int st1 = 0; st1 < 4; ++st1) sc[st1] = kw_widen8(bm[st1]); + [unroll] for (int ct = 0; ct < 2; ++ct) + [unroll] for (int st2 = 0; st2 < 4; ++st2) sc[st2] = kw_mma(kf[st2][ct], q[ct], sc[st2]); + uint4 gg[4]; + [unroll] for (int st5 = 0; st5 < 4; ++st5) gg[st5] = kw_softpoly8(kw_out(sc[st5])); + float w[4][8]; + float sum = 0.0f; + [unroll] for (int st3 = 0; st3 < 4; ++st3) + [unroll] for (int j = 0; j < 8; ++j) { + w[st3][j] = exp2(f16tof32((gg[st3][j >> 1] >> (16 * (j & 1))) & 0xffffu)); // v_exp_f32 + sum += w[st3][j]; + } + sum += wg_xor16(sum); + const float inv = rcp(sum); // v_rcp_f32 + [unroll] for (int st4 = 0; st4 < 4; ++st4) pa[st4] = k_smx_weights8(w[st4], inv); + } + // z_h^T = VT A^T (the two channel tiles' chains interleaved, each over st) -> Z tiles 2 hd, 2 hd + 1 + { + uint4 vf[2][4]; + [unroll] for (int ct = 0; ct < 2; ++ct) + [unroll] for (int st = 0; st < 4; ++st) vf[ct][st] = ks_lds[ks_vidx(16 * ct + n16, 2 * st + hh)]; + v8f az[2]; + az[0] = wg_splat(0.0f); + az[1] = wg_splat(0.0f); + [unroll] for (int st = 0; st < 4; ++st) + [unroll] for (int ct2 = 0; ct2 < 2; ++ct2) az[ct2] = kw_mma(vf[ct2][st], pa[st], az[ct2]); + const uint zr = ks_zrow(win, t, hh) + (uint)(32 * hd); + kw_stg(P_sz, zr, kw_out(az[0])); + kw_stg(P_sz, zr + 16u, kw_out(az[1])); + } +} +#endif + +#if KS_PHASE == 2 +// ---- k_split_m_wo: one (window, output tile ot) ---------------------------------------------------------------------- +WG_KERNEL(ks_wo) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KT, ot = bid % KT; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint wo = P_wo + 2u * ((uint)(ot * DT * 256) + L.lane8); // fragment dt at + 512 dt + const uint zr = ks_zrow(win, L.t, hh); + uint4 wf[DT], zf[DT]; + [unroll] for (int d0 = 0; d0 < DT; ++d0) { + wf[d0] = wg_ld128(wo, (uint)(d0 * 512)); + zf[d0] = kw_ldg(P_sz, zr + 16u * d0); + } + const uint4 xr = kw_ldg(P_x, L.mtok * KW_C + 8u * hh + 16u * ot); + const uint4 g2v = kw_ldg(P_g2, (uint)(16 * ot + 8 * hh)); + uint4 bo = kw_ldg(P_bo, (uint)(16 * ot + 8 * hh)); + uint f = bo.x ^ xr.x ^ g2v.x; // (the chain starts with the last loads) + [unroll] for (int d1 = 0; d1 < DT; ++d1) f ^= wf[d1].x ^ zf[d1].x; + bo.x ^= KS_FENCE(f); + v8f yacc = kw_widen8(bo); + [unroll] for (int dt = 0; dt < DT; ++dt) yacc = kw_mma(wf[dt], zf[dt], yacc); + const uint4 x1 = kw_add8(xr, kw_out(yacc)); // x1 = f16(x + y) + const uint cr = ks_crow(win, L.t, hh) + 16u * ot; + kw_stg(P_sx1, cr, x1); + kw_stg(P_sa, cr, kw_mul8(x1, g2v)); // a = f16(x1 g2) +} +#endif + +#if KS_PHASE == 3 +// ---- k_split_m_w1: one (window, KS_HB hidden tiles) ------------------------------------------------------------------ +WG_KERNEL(ks_w1) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / NHG, h0 = (bid % NHG) * KS_HB; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint w1 = P_w1 + 2u * ((uint)(h0 * KT * 256) + L.lane8); // hidden tile h0 + m, kt: + 512 (m KT + kt) + const uint cr = ks_crow(win, L.t, hh); + uint4 a[KT]; + uint4 wf[KS_HB][KT]; + [unroll] for (int kt0 = 0; kt0 < KT; ++kt0) { + [unroll] for (int m0 = 0; m0 < KS_HB; ++m0) wf[m0][kt0] = wg_ld128(w1, (uint)((m0 * KT + kt0) * 512)); + a[kt0] = kw_ldg(P_sa, cr + 16u * kt0); + } + uint4 b1v[KS_HB]; + [unroll] for (int m4 = 0; m4 < KS_HB; ++m4) b1v[m4] = kw_ldg(P_b1, (uint)(16 * (h0 + m4) + 8 * hh)); + uint f = 0u; // (the chain starts with the last loads) + [unroll] for (int m5 = 0; m5 < KS_HB; ++m5) f ^= b1v[m5].x; + [unroll] for (int kt1 = KT - 1; kt1 >= 0; --kt1) { + f ^= a[kt1].x; + [unroll] for (int m3 = KS_HB - 1; m3 >= 0; --m3) f ^= wf[m3][kt1].x; + } + v8f p[KS_HB]; + [unroll] for (int m1 = 0; m1 < KS_HB; ++m1) { + b1v[m1].x ^= KS_FENCE(f); + p[m1] = kw_widen8(b1v[m1]); + } + [unroll] for (int kt = 0; kt < KT; ++kt) // the hidden tiles' chains interleaved, each over kt + [unroll] for (int m = 0; m < KS_HB; ++m) p[m] = kw_mma(wf[m][kt], a[kt], p[m]); + const uint hr = ks_hrow(win, L.t, hh); + [unroll] for (int m2 = 0; m2 < KS_HB; ++m2) kw_stg(P_sh, hr + (uint)(16 * (h0 + m2)), kw_act8(kw_out(p[m2]))); +} +#endif + +#if KS_PHASE == 4 +// ---- k_split_m_w2: one (window, output tile ot) ---------------------------------------------------------------------- +// The chain over the HT hidden tiles in batches of KS_B2 products, each batch's operands (W2 fragment (j, ot), h fragment +// of hidden tile j) loaded and fenced before its first product. +static const int NB2 = HT / KS_B2; +WG_KERNEL(ks_w2) +{ + WG_SETUP(); + const int bid = (int)wg_block_id(); + const int win = bid / KT, ot = bid % KT; + const KsLane L = ks_lane(win); + const int hh = L.hh; + const uint w2 = P_w2 + 2u * ((uint)(ot * 256) + L.lane8); // hidden tile j: + 512 j KT + const uint hb = P_sh + 2u * ks_hrow(win, L.t, hh); // hidden tile j: + 32 j + const uint cr = ks_crow(win, L.t, hh) + 16u * ot; + const uint4 x1 = kw_ldg(P_sx1, cr); + uint4 b2v = kw_ldg(P_b2, (uint)(16 * ot + 8 * hh)); + v8f acc; + [unroll] for (int b = 0; b < NB2; ++b) { // the hidden tiles 0 .. HT - 1, in order + uint4 rw[KS_B2], rh[KS_B2]; + [unroll] for (int j0 = 0; j0 < KS_B2; ++j0) { + rw[j0] = wg_ld128(w2, (uint)((b * KS_B2 + j0) * KT * 512)); + rh[j0] = wg_ld128(hb, (uint)((b * KS_B2 + j0) * 32)); + } + uint f = b == 0 ? b2v.x ^ x1.x : 0u; + [unroll] for (int j1 = 0; j1 < KS_B2; ++j1) f ^= rw[j1].x ^ rh[j1].x; + if (b == 0) { // the bias after the fence: no early wait on it + b2v.x ^= KS_FENCE(f); + acc = kw_widen8(b2v); + } else { + rh[0].x ^= KS_FENCE(f); + } + [unroll] for (int j = 0; j < KS_B2; ++j) acc = kw_mma(rw[j], rh[j], acc); + } + const uint4 x2 = kw_pin(kw_add8(x1, kw_out(acc))); // x2 = f16(x1 + f16(W2 h + b2)), before the divergent store + if (L.real && P_y != 0u) kw_stg(P_y, (uint)((L.gy * P_w + L.gx) * KW_C + 8 * hh) + 16u * ot, x2); +} +#endif diff --git a/clean/k/dx12/smx_selfcheck_digest.py b/clean/k/dx12/smx_selfcheck_digest.py index 00d743af..c4542eb3 100644 --- a/clean/k/dx12/smx_selfcheck_digest.py +++ b/clean/k/dx12/smx_selfcheck_digest.py @@ -3,7 +3,8 @@ The reference hashes in k_smx_selfcheck.h are valid only for the shaders and the synthetic data they were generated from. The digest is a SHA-256 over: -- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines); +- the @wg-kernel definition lines of every self-check kernel (both forms: the names, code objects and defines), the + phase kernels of a split stage m included (SPLIT: the native device runs them for the self-check's m case); - the text of every source file those kernels compile from (their .hlsl file and its #include closure in clean/k/dx12/hlsl and clean/dx12/hlsl; an include generated into the build folder is covered by its generator, clean/k/dx12/gen_k_fp_tables.py); @@ -28,7 +29,10 @@ # the softmax stage kernels the self-check runs (runtime_k.cpp kSmxCases), both forms BASE = ["k_stage_e2", "k_stagew2_e3", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1", "k_stagew4_e4", "k_stagew4_d4", "k_stagew2_e4", "k_stagew2_d4", "k_stagew4_m"] -KERNELS = sorted(BASE + [k + "__rodd" for k in BASE]) +# the phase kernels the native D3D12 device runs in place of k_stagew4_m[__rodd] (runtime option k_split, default on +# backend 3; KRuntime::launch redirects stage m, so the self-check's m case runs them: kernels/k_block_split.hlsl) +SPLIT = ["k_split_m_attn", "k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"] +KERNELS = sorted(BASE + [k + "__rodd" for k in BASE] + SPLIT) DECL = re.compile(r"^//\s*@wg-kernel\s+(\S+)\s.*$", re.M) INCLUDE = re.compile(r'^\s*#\s*include\s+"([^"]+)"', re.M) REGION = re.compile(r"// SMX-SELFCHECK-SOURCE BEGIN.*?// SMX-SELFCHECK-SOURCE END", re.S) diff --git a/clean/k/runtime/k_smx_selfcheck.h b/clean/k/runtime/k_smx_selfcheck.h index c4fed657..01c8aaf0 100644 --- a/clean/k/runtime/k_smx_selfcheck.h +++ b/clean/k/runtime/k_smx_selfcheck.h @@ -8,7 +8,7 @@ namespace k_smx_selfcheck { -constexpr char kSourceDigest[] = "2ad91491cb5edf84114baa30bc0fe79a4859ac281a90d8989bc57130f8d05d69"; +constexpr char kSourceDigest[] = "ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b"; constexpr int kCount = 133; constexpr const char* kNames[kCount] = { "k_stage_e2@512x384/o4,4", diff --git a/clean/k/runtime/runtime_k.cpp b/clean/k/runtime/runtime_k.cpp index 76bb1133..b0d2ab7b 100644 --- a/clean/k/runtime/runtime_k.cpp +++ b/clean/k/runtime/runtime_k.cpp @@ -561,6 +561,16 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (warm_line_ < 4 || warm_line_ > 4096 || (warm_line_ & (warm_line_ - 1))) return fail("model K: k_warm_line must be a power of two in 4..4096"); if (warm_groups_ < 1 || warm_groups_ > 1024) return fail("model K: k_warm_groups must be 1..1024"); + // SMX-SELFCHECK-SOURCE BEGIN (k_split: routes the self-check's m case; the digest covers it) + // k_split (deep-split; the native D3D12 device, backend 3): "m" (the default) runs stage m as four phase launches + // spread over many more workgroups (k_split_m_attn[__rodd], k_split_m_wo, k_split_m_w1, k_split_m_w2: + // clean/k/dx12/kernels/k_block_split.hlsl), the same arithmetic and output bytes as the one k_stagew4_m launch; the + // create-time self-check runs that form for its m case (launch() redirects k_stagew4_m). "off": the one launch. Other + // devices always take the one launch (the phase kernels are D3D12 ports only). + const std::string ksplit = option(c, "k_split"); + if (!ksplit.empty() && ksplit != "m" && ksplit != "off") return fail("model K: k_split must be m or off"); + split_m_ = c.backend == 3 && ksplit != "off"; + // SMX-SELFCHECK-SOURCE END // k_fill=<0..255> (test use, changes no value a frame writes): every buffer a frame writes and both history copies are // filled with this byte after allocation, before the first frame, so that a test with a different byte per backend // sees a write a frame did not make (first-frame and reset-only writes included) as a difference @@ -575,8 +585,8 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e smx_test_ = option(c, "k_smx_test"); #ifdef WG_DEV_DIAGNOSTICS if (!smx_test_.empty() && smx_test_ != "mix_fail" && smx_test_ != "both_fail" && smx_test_ != "rodd_unchecked" && - smx_test_ != "report") - return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked or report"); + smx_test_ != "split_fail" && smx_test_ != "report") + return fail("model K: k_smx_test must be mix_fail, both_fail, rodd_unchecked, split_fail or report"); #else if (!smx_test_.empty()) return fail("model K: k_smx_test is a test hook of developer builds only"); #endif @@ -667,6 +677,7 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e return fail("model K: an input-stage buffer is not allocated (it would be packed as arena offset 0)"); // the own output buffer (tests): dense RGB f32, dense RGBA f32 (FrameIo.out_rgba32f) or RGBA16F rows that fit if (!external_io_ && !alloc(&out_, (size_t)W_ * H_ * 16, true)) return false; + if (split_m_ && !split_alloc()) return false; // k_split's scratch (filled with the rest by k_fill, below) if (fill_ >= 0) { // k_fill (test hook): the sentinel in everything a frame writes, before the first frame auto fillb = [&](DevPtr p, size_t bytes) { return !p || dev_->memset(p, fill_, std::max(bytes, 256)); }; bool ok = true; @@ -676,6 +687,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e ok = ok && fillb(ctap_, R * 8) && fillb(ctapT_, R * 2) && fillb(cpre_, R * 16) && fillb(cpre2_, R * 8) && fillb(xs_, 16) && fillb(kexpo_, 16) && fillb(color_, R * 16) && fillb(depth_, R * 4) && fillb(motion_, R * 8); // the own input copies (unused with caller buffers) + if (split_m_) // k_split's scratch: written by the phases of stage m, so filled like the rest + ok = ok && fillb(split_z_, split_bytes_[0]) && fillb(split_x1_, split_bytes_[1]) && + fillb(split_a_, split_bytes_[2]) && fillb(split_h_, split_bytes_[3]); if (!ok) return dfail(); } // the filter-shape table (spec 43 section 7) of every level, computed once; a frame reads its level's set @@ -690,7 +704,28 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e if (!dev_->prepare(k)) return dfail(); prepared_ = true; } - if (!smx_select(c.backend, smx_test_)) return false; + // the create-time self-check. A split stage m (k_split=m) that fails it is not a refusal: stage m takes the one launch + // (k_split=off's path: the same bytes, the same reference hashes), the self-check runs again on that path, and create + // refuses only when that fails too (both reasons in the error) + const std::string notes_before = notes_; + if (!smx_select(c.backend, smx_test_)) { + if (!split_m_) return false; + std::string split_why = err_; + for (char& ch : split_why) + if (ch == '\n' || ch == '\r') ch = ' '; + notes_ = notes_before; + split_m_ = false; + if (!smx_select(c.backend, smx_test_)) { + std::string one_launch = err_; + for (char& ch : one_launch) + if (ch == '\n' || ch == '\r') ch = ' '; + return fail(one_launch + " (the split form of stage m failed first: " + split_why + ")"); + } + notes_ += (notes_.empty() ? "" : "\n") + + std::string("model K: the create-time self-check failed on the split form of stage m (") + split_why + + "); stage m takes the one launch (k_stagew4_m), output unchanged"; + err_.clear(); + } if (!approx_note_.empty()) notes_ += (notes_.empty() ? "" : "\n") + approx_note_; return true; } @@ -713,7 +748,9 @@ bool KRuntime::create(const WgRtConfig& c, Device* dev, AllocFn alloc_fn, bool e // prepared at create (KRuntime::create, Device::prepare): a missing or corrupt shader fails create. // Test hook (developer builds only, WG_DEV_DIAGNOSTICS): runtime option k_smx_test = mix_fail (the mix form counts as // failed), both_fail (both forms count as failed), rodd_unchecked (the round-to-odd ports without the check; for the -// frame test's negative control), report (adds the computed hashes as a note; on any device, every possible offset). +// frame test's negative control), split_fail (the split form of stage m counts as failed, so create takes the one +// launch; with k_split=off it has no effect), report (adds the computed hashes as a note; on any device, every possible +// offset). namespace { // SMX-SELFCHECK-SOURCE BEGIN (the cases, the synthetic data, the offsets and the check: k_smx_selfcheck.h's digest covers this text) struct SmxCase { @@ -897,6 +934,8 @@ bool KRuntime::smx_select(int backend, const std::string& hook) { add_note(report("reference", h, -1)); return true; } + if (hook == "split_fail" && split_m_) // test hook: the split form of stage m counts as failed (create's fallback) + return fail("model K: test hook: the split self-check of stage m failed (k_smx_test=split_fail)"); if (hook == "rodd_unchecked") { smx_rodd_ = true; smx_note_ = " smx=rodd(unchecked)"; @@ -960,6 +999,11 @@ std::string KRuntime::stage_kernel(const std::string& k, const char* stage) cons bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a0, void* a1, void* a2, void* a3, void* a4, void* a5, void* a6, void* a7) { if (grid == 0) return true; + // SMX-SELFCHECK-SOURCE BEGIN (k_split: stage m's kernel-name redirect; the digest covers it) + // k_split: stage m (the frame's and the self-check's) as its phase launches + if (split_m_ && std::strncmp(kernel, "k_stagew4_m", 11) == 0 && (!kernel[11] || std::strcmp(kernel + 11, "__rodd") == 0)) + return split_m(kernel[11] != 0, a0); + // SMX-SELFCHECK-SOURCE END void* args[8] = {a0, a1, a2, a3, a4, a5, a6, a7}; if (pending_mark_ >= 0) { const int e = pending_mark_; @@ -969,6 +1013,53 @@ bool KRuntime::launch(const char* kernel, unsigned grid, unsigned block, void* a return dev_->launch(kernel, grid, block, args) || dfail(); } +// SMX-SELFCHECK-SOURCE BEGIN (k_split: the phase launches and scratch; the digest covers them) +// k_split: stage m as four launches (clean/k/dx12/kernels/k_block_split.hlsl), joined by the dispatch boundaries: +// attention per (window, head), Wo per (window, output tile), W1 per (window, two hidden tiles), W2 per (window, output +// tile). Each phase reads the block parameters of k_stagew4_m's launch and the scratch the phase before wrote. +bool KRuntime::split_m(bool rodd, void* params) { + const KBlkParams& p = *static_cast(params); + const KStageInfo& st = *k_stage("m"); + const int nwin = p.nwx * p.nwy, kt = st.C / 16, ht = 4 * st.C / 16; + if (nwin > split_cap_) + return fail("model K: k_split: stage m has " + std::to_string(nwin) + " windows, the scratch holds " + + std::to_string(split_cap_)); + // k_split_m_w1 runs HT / 2 groups per window: one group per two hidden tiles (KS_HB = 2 in + // clean/k/dx12/kernels/k_block_split.hlsl). A change of KS_HB must change this grid too. + return launch(rodd ? "k_split_m_attn__rodd" : "k_split_m_attn", (unsigned)(nwin * st.heads), 128, params, &split_z_, + &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_wo", (unsigned)(nwin * kt), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_w1", (unsigned)(nwin * (ht / 2)), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_) && + launch("k_split_m_w2", (unsigned)(nwin * kt), 128, params, &split_z_, &split_x1_, &split_a_, &split_h_); +} + +// The phases' scratch (FP16): Z [windows][64][8 heads x 32], X1 and A [windows][64][C], H [windows][64][4 C], for the most +// windows stage m has at any window offset (0 .. 7) in this context's frames or in the self-check's m case. +bool KRuntime::split_alloc() { + const KStageInfo& st = *k_stage("m"); + if (st.C != 160 || st.heads != 8) // the phase kernels are compiled for stage m's C = 160, 8 heads + return fail("model K: k_split: stage m is not C = 160 with 8 heads"); + int cap = 0; + auto grow = [&](int wb, int hb) { + int lw, lh; + k_level_size(wb, hb, st.level, &lw, &lh); + for (int oy = 0; oy < 8; ++oy) + for (int ox = 0; ox < 8; ++ox) cap = std::max(cap, k_window_count(lw, lh, ox, oy)); + }; + grow(Bw_, Bh_); + for (const SmxCase& sc : kSmxCases) + if (std::strcmp(sc.stage, "m") == 0) grow(sc.wb, sc.hb); + split_cap_ = cap; + const size_t tok = (size_t)cap * 64; + split_bytes_[0] = tok * st.heads * 32 * 2; // Z + split_bytes_[1] = tok * st.C * 2; // X1 + split_bytes_[2] = tok * st.C * 2; // A + split_bytes_[3] = tok * 4 * st.C * 2; // H + return alloc(&split_z_, split_bytes_[0]) && alloc(&split_x1_, split_bytes_[1]) && + alloc(&split_a_, split_bytes_[2]) && alloc(&split_h_, split_bytes_[3]); +} +// SMX-SELFCHECK-SOURCE END + bool KRuntime::upload_inputs(int rw, int rh, const float* color, const float* dep, const float* mot, int channels) { if (rw <= 0 || rh <= 0 || rw > max_rw_ || rh > max_rh_) return fail("render size larger than the maximum"); if (external_io_) return fail("inputs come from the interop (external_io)"); @@ -1448,6 +1539,9 @@ std::vector KRuntime::d3d12_kernels() const { v.push_back(k + "__rodd"); } } + if (split_m_) // k_split: stage m's phase kernels (k_stagew4_m[__rodd] stay listed above, unused while split) + for (const char* k : {"k_split_m_attn", "k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"}) + v.push_back(k); v.push_back("k_mix_probe"); v.push_back("k_mix_probe__rodd"); v.push_back((compact_head_ ? "k_out_pixel_c" : "k_out_pixel") + d); @@ -1515,7 +1609,7 @@ const char* KRuntime::kernels() { " legacy_color_launches=" + std::to_string(legacy_color_launches_) + " legacy_base_launches=" + std::to_string(legacy_base_launches_) + (fast_ ? " k_fast=1" : "") + (expo_v051_ ? " k_expo_rule=v051" : "") + - smx_note_ + approx_tag() + " hidden=fused_d0" + std::string(" head=") + + smx_note_ + (split_m_ ? " split=m" : "") + approx_tag() + " hidden=fused_d0" + std::string(" head=") + (compact_head_ ? "compact" : "full") + " base=" + std::to_string(Bw_) + "x" + std::to_string(Bh_) + " hdr=" + (hdr_ ? "1" : "0") + " mv=render" + diff --git a/clean/k/runtime/runtime_k.h b/clean/k/runtime/runtime_k.h index f810eb4f..a1d3540a 100644 --- a/clean/k/runtime/runtime_k.h +++ b/clean/k/runtime/runtime_k.h @@ -124,6 +124,12 @@ class KRuntime { std::string smx_case_name(int i, int ox, int oy, bool rodd) const; // the stage kernel network() launches: the __rodd form of a softmax stage after a failed probe std::string stage_kernel(const std::string& k, const char* stage) const; + // deep-split (runtime option k_split, native D3D12 device): stage m as its four phase launches (k_split_m_*, + // clean/k/dx12/kernels/k_block_split.hlsl) in place of one k_stagew4_m launch; launch() redirects k_stagew4_m[__rodd] + // here, so network() and the create-time self-check both run it. split_alloc (create): the phases' scratch, sized for + // the most windows stage m can have in a frame of this context or in the self-check's m case. + bool split_m(bool rodd, void* params); + bool split_alloc(); int fill_ = -1; // option k_fill (test hook): the byte every frame-written buffer and the histories start with WgRtConfig cfg_ = {}; @@ -184,6 +190,12 @@ class KRuntime { return std::string(" k_approx=") + (approx_a1_ ? "a1" : "") + (approx_a1_ && approx_a2_ ? "," : "") + (approx_a2_ ? "a2" : ""); } + // runtime option k_split (backend 3 only; default "m"): stage m runs as its phase launches (split_m); "off": one + // k_stagew4_m launch. split_cap_: windows the scratch holds; split_z_ / x1_ / a_ / h_: the phases' FP16 scratch + bool split_m_ = false; + int split_cap_ = 0; + DevPtr split_z_ = 0, split_x1_ = 0, split_a_ = 0, split_h_ = 0; + size_t split_bytes_[4] = {0, 0, 0, 0}; // the bytes of split_z_, split_x1_, split_a_, split_h_ (split_alloc) std::string smx_note_; std::string smx_test_; // option k_smx_test (developer builds only) std::string notes_; // notes of create for the caller's log (notes()), one per line, or empty diff --git a/clean/k/tests/raw_compare.py b/clean/k/tests/raw_compare.py index 4fcbd853..5d020044 100644 --- a/clean/k/tests/raw_compare.py +++ b/clean/k/tests/raw_compare.py @@ -3,8 +3,10 @@ Before a frame every output and intermediate buffer is filled with a sentinel byte: 0xA5 on HIP, 0x5A on D3D12. After it, the two buffers are compared as 4-byte words of raw bit patterns (no float conversion: an FP16 NaN keeps its -payload, -0 stays -0). A word that differs is a failure, except a word that neither side wrote (HIP's sentinel on one -side and D3D12's on the other): those are counted, and a caller that knows the buffer is written in full allows none. +payload, -0 stays -0). A buffer whose size is not a multiple of 4 (ctapT is 2 bytes a render pixel, so an odd render +size gives one) has its 1 to 3 trailing bytes compared byte for byte under the same rules, so every byte is checked. +A unit that differs is a failure, except a unit that neither side wrote (HIP's sentinel on one side and D3D12's on the +other): those are counted, and a caller that knows the buffer is written in full allows none. A write that D3D12 dropped leaves its sentinel where HIP stored a value, and a write that only D3D12 made leaves HIP's sentinel: both fail, which the negative controls in test_k_raw_compare.py show. """ @@ -18,21 +20,35 @@ def sentinel_word(byte): return np.uint32(byte * 0x01010101) -def compare_raw(what, hip, d3d12, allow_untouched=0, hip_sentinel=HIP_SENTINEL, d3d12_sentinel=D3D12_SENTINEL): - """hip, d3d12: bytes-like of equal length (a multiple of 4). Raises AssertionError on a differing word or on more - than allow_untouched words written by neither; returns the number of such words.""" - hip, d3d12 = bytes(hip), bytes(d3d12) - if len(hip) != len(d3d12) or len(hip) % 4: - raise AssertionError(f"{what}: sizes {len(hip)} and {len(d3d12)} (need equal, a multiple of 4)") - h, d = np.frombuffer(hip, np.uint32), np.frombuffer(d3d12, np.uint32) +def _compare_units(what, h, d, hip_sentinel, d3d12_sentinel, unit, base, width): + """h, d: arrays of equal length of one unit (words or bytes); base: index of the first unit in the buffer (bytes, for + the trailing bytes). Returns (units written by neither side, units compared); raises on a differing unit.""" differ = h != d - untouched = differ & (h == sentinel_word(hip_sentinel)) & (d == sentinel_word(d3d12_sentinel)) + untouched = differ & (h == hip_sentinel) & (d == d3d12_sentinel) bad = differ & ~untouched if bad.any(): i = int(np.flatnonzero(bad)[0]) - raise AssertionError(f"{what}: {int(bad.sum())} of {len(h)} words differ; first at word {i}: HIP {h[i]:08x}, " - f"D3D12 {d[i]:08x}") - n = int(untouched.sum()) - if n > allow_untouched: - raise AssertionError(f"{what}: {n} of {len(h)} words written by neither side (allowed {allow_untouched})") - return n + raise AssertionError(f"{what}: {int(bad.sum())} of {len(h)} {unit}s differ; first at {unit} {base + i}: " + f"HIP {int(h[i]):0{width}x}, D3D12 {int(d[i]):0{width}x}") + return int(untouched.sum()), len(h) + + +def compare_raw(what, hip, d3d12, allow_untouched=0, hip_sentinel=HIP_SENTINEL, d3d12_sentinel=D3D12_SENTINEL): + """hip, d3d12: bytes-like of equal length, any length. Raises AssertionError on a differing word or byte, or on more + than allow_untouched units written by neither side; returns the number of such units (words and trailing bytes).""" + hip, d3d12 = bytes(hip), bytes(d3d12) + if len(hip) != len(d3d12): + raise AssertionError(f"{what}: sizes {len(hip)} and {len(d3d12)} (need equal)") + whole = len(hip) - len(hip) % 4 + untouched, total = _compare_units( + what, np.frombuffer(hip[:whole], np.uint32), np.frombuffer(d3d12[:whole], np.uint32), + sentinel_word(hip_sentinel), sentinel_word(d3d12_sentinel), "word", 0, 8) + if whole < len(hip): + n, m = _compare_units( + what, np.frombuffer(hip[whole:], np.uint8), np.frombuffer(d3d12[whole:], np.uint8), + hip_sentinel, d3d12_sentinel, "byte", whole, 2) + untouched, total = untouched + n, total + m + if untouched > allow_untouched: + raise AssertionError(f"{what}: {untouched} of {total} units (words, trailing bytes) written by neither side " + f"(allowed {allow_untouched})") + return untouched diff --git a/clean/k/tests/test_k_raw_compare.py b/clean/k/tests/test_k_raw_compare.py index e0a74878..46bc6b27 100644 --- a/clean/k/tests/test_k_raw_compare.py +++ b/clean/k/tests/test_k_raw_compare.py @@ -61,6 +61,67 @@ def test_size_mismatch_fails(self): with self.assertRaises(AssertionError): compare_raw("size", h.tobytes(), d[:-1].tobytes()) + def odd_bytes(self, n, seed=7): + """Ordinary bytes of length n (no sentinel byte, so every byte is data).""" + rng = np.random.default_rng(seed) + b = rng.integers(0, 256, n, dtype=np.uint8) + b[(b == HIP_SENTINEL) | (b == D3D12_SENTINEL)] = 1 + return b + + def test_odd_length_equal_passes(self): + """A buffer whose size is not a multiple of 4 compares byte for byte: 2 x 131 x 79 = 20698 bytes (ctapT at the + odd render size) and the other trailing-byte lengths 1, 2 and 3 mod 4.""" + for n in (1, 2, 3, 6, 4 * 5 + 1, 4 * 5 + 3, 2 * 131 * 79): + with self.subTest(n=n): + b = self.odd_bytes(n) + self.assertEqual(compare_raw("odd", b.tobytes(), b.copy().tobytes()), 0) + + def test_odd_length_last_byte_differs_fails(self): + """A change in the last byte of an odd-length buffer (the trailing bytes are compared, not skipped).""" + for n in (4 * 5 + 1, 4 * 5 + 2, 4 * 5 + 3, 2 * 131 * 79): + with self.subTest(n=n): + h = self.odd_bytes(n) + d = h.copy() + d[-1] ^= 1 + with self.assertRaisesRegex(AssertionError, f"differ; first at byte {n - 1}: HIP"): + compare_raw("odd-last", h.tobytes(), d.tobytes()) + + def test_odd_length_first_trailing_byte_differs_fails(self): + """The first trailing byte (the first byte after the last whole word) is compared too.""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + d[(n // 4) * 4] ^= 0x80 + with self.assertRaisesRegex(AssertionError, f"first at byte {(n // 4) * 4}"): + compare_raw("odd-tail0", h.tobytes(), d.tobytes()) + + def test_odd_length_untouched_trailing_byte_counted(self): + """A trailing byte neither side wrote (HIP's sentinel on one side, D3D12's on the other) is counted like a word; + a full buffer allows none.""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + h[-1], d[-1] = HIP_SENTINEL, D3D12_SENTINEL + self.assertEqual(compare_raw("odd-pad", h.tobytes(), d.tobytes(), allow_untouched=1), 1) + with self.assertRaisesRegex(AssertionError, "neither"): + compare_raw("odd-pad", h.tobytes(), d.tobytes()) + + def test_odd_length_write_only_on_d3d12_fails(self): + """A trailing byte D3D12 wrote where HIP left its sentinel fails (the sentinel pair is not the only way to pass).""" + n = 2 * 131 * 79 + h = self.odd_bytes(n) + d = h.copy() + h[-2] = HIP_SENTINEL + d[-2] = 9 + with self.assertRaisesRegex(AssertionError, "differ"): + compare_raw("odd-extra", h.tobytes(), d.tobytes()) + + def test_odd_length_size_mismatch_fails(self): + n = 2 * 131 * 79 + h = self.odd_bytes(n) + with self.assertRaisesRegex(AssertionError, "sizes"): + compare_raw("odd-size", h.tobytes(), h[:-1].tobytes()) + if __name__ == "__main__": unittest.main() diff --git a/clean/k/tests/test_k_runtime_d3d12.py b/clean/k/tests/test_k_runtime_d3d12.py index 4f31df9b..a0f060da 100644 --- a/clean/k/tests/test_k_runtime_d3d12.py +++ b/clean/k/tests/test_k_runtime_d3d12.py @@ -81,6 +81,12 @@ def tonemapped_psnr(a, b): ALWAYS = ["k_expo_scalars", "k_out_pixel_c"] TILED = ["kx/k_in_tiled", "kx/k_in_tiled_uniform"] LEGACY = ["kx/k_in_color", "kx/k_in_base"] # (+ kx/k_in_reset on a reset frame) +ALT_SENTINEL = 0xC3 # the second native runtime of a native-against-native comparison (the first is D3D12_SENTINEL) + + +def opts(*parts): + """A runtime option string from the non-empty parts.""" + return ";".join(p for p in parts if p) def trace_since(pos): @@ -331,6 +337,11 @@ def test_native_softmax_refusals(self): self.assertFalse(both["ok"], both) self.assertIn("not exact on this driver", both["error"]) self.assertNotIn("\n", both["error"].strip()) + # with the split form on, the first refusal is the split form's and the one-launch path's is named too + self.assertIn("(the split form of stage m failed first:", both["error"]) + both_off = self.leg("k_smx_test=both_fail;k_split=off") + self.assertFalse(both_off["ok"], both_off) + self.assertNotIn("split form", both_off["error"]) # a copy of the build (its top-level files and dx12/) with one flipped byte in one stage shader: a softmax stage's # __rodd port and a fixed-weight stage (e0); create prepares every selectable kernel, so it refuses in one line for victim in ("k_stagew2_e3__rodd", "k_stage2_e0"): @@ -355,23 +366,155 @@ def test_native_softmax_refusals(self): finally: shutil.rmtree(tmp, ignore_errors=True) - def check_fallback_frames(self, frames, big, **size): + def expect_fallback_trace(self, on_dx, big, split): + """The softmax stages of the forced fallback ran their __rodd ports on D3D12 and no mix form did. Stage m runs as + its phase launches with k_split=m (the default) and as k_stagew4_m[__rodd] with k_split=off. The phases + k_split_m_wo / w1 / w2 have no softmax: they are the same launches in both forms, so only the attention phase + has a mix form to forbid.""" + ng = "k_stagew2_" if big else "k_stagew4_" + stages = ["k_stage_e2", "k_stagew2_e3", ng + "e4", ng + "d4", "k_stagew2_d3", "k_stage_d2", "k_stage2_d1"] + if split: + rodd, mix = ["k_split_m_attn__rodd", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"], ["k_split_m_attn"] + else: + rodd, mix = ["k_stagew4_m__rodd"], ["k_stagew4_m"] + self.assertEqual([k for k in stages if not on_dx.get(k + "__rodd")] + [k for k in rodd if not on_dx.get(k)], [], + on_dx) + self.assertEqual([k for k in stages + mix if on_dx.get(k)], [], on_dx) + + def check_fallback_frames(self, frames, big, split=True, **size): """Forced fallback: the frames match HIP (raw outputs and intermediates) and every softmax stage ran its __rodd - port on D3D12, none its mix form.""" + port on D3D12, none its mix form (split: stage m's phase launches; else its one launch).""" self.need_dev_build() - on_dx, _ = self.sequence(3, True, 0, True, frames, "k_smx_test=mix_fail", **size) - ng = "k_stagew2_" if big else "k_stagew4_" - want = ["k_stage_e2", "k_stagew2_e3", ng + "e4", "k_stagew4_m", ng + "d4", "k_stagew2_d3", "k_stage_d2", - "k_stage2_d1"] - self.assertEqual([k for k in want if not on_dx.get(k + "__rodd")], [], on_dx) - self.assertEqual([k for k in want if on_dx.get(k)], [], on_dx) + options = "k_smx_test=mix_fail" + ("" if split else ";k_split=off") + on_dx, _ = self.sequence(3, True, 0, True, frames, options, **size) + self.expect_fallback_trace(on_dx, big, split) def test_native_fallback_frames_1440p_quality(self): self.check_fallback_frames(self.FRAMES_Q, False, **self.QUALITY) + self.check_fallback_frames(self.FRAMES_Q, False, split=False, **self.QUALITY) def test_native_fallback_frames_4k_performance(self): - self.check_fallback_frames([(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)], True, - out=(3840, 2160), max_render=(1920, 1080)) + frames = [(1920, 1080, True, 1.0, 1.0), (1920, 1080, False, 1.0, 1.0)] + self.check_fallback_frames(frames, True, out=(3840, 2160), max_render=(1920, 1080)) + self.check_fallback_frames(frames, True, split=False, out=(3840, 2160), max_render=(1920, 1080)) + + # ---- deep-split: stage m's phase launches (k_split) on the native device; native against native, no HIP run ----- + def native_pair(self, left, right, frames, hdr=True, mode=0, half=True, out=OUT, max_render=MAX_RENDER): + """Two native runtimes of this build, options `left` and `right`, the same inputs for every frame. After every + frame the output and the written buffers, histories and exposure state are compared raw. Each side starts from + its own sentinel (0x5A and 0xC3, k_fill and before every frame), so a word neither side wrote fails.""" + auto = mode == 2 + a = self.make(3, hdr, auto, opts(left, f"k_fill={D3D12_SENTINEL}"), out, max_render) + b = self.make(3, hdr, auto, opts(right, f"k_fill={ALT_SENTINEL}"), out, max_render) + head = "k.N" if "k_head=full" in left + right else "k.Np" + try: + for i, fr in enumerate(frames): + rw, rh, reset, E, pi = fr[:5] + x = inputs(np.random.default_rng(i), rw, rh, hdr, reset, jitter=fr[5] if len(fr) > 5 else None) + x["pre_exposure"], x["E_cur"] = pi, float(np.float32(E)) + if i == 2: + x["mv_scale"], x["mv_offset"] = (0.75, 1.25), (0.125, -0.25) + for rt, byte in ((a, D3D12_SENTINEL), (b, ALT_SENTINEL)): + poke(rt, rt.bufs["out"], rt.bufs_size["out"], byte) + if i: + for name in WRITTEN + [head]: + p, n = rt.buffer_info(name) + poke(rt, p, n, byte) + a.frame(x, mode, half=half) + b.frame(x, mode, half=half) + compare_raw(f"frame {i}: output", out_bytes(a, half), out_bytes(b, half), + hip_sentinel=D3D12_SENTINEL, d3d12_sentinel=ALT_SENTINEL) + for name in WRITTEN + STATE + [head]: + compare_raw(f"frame {i}: {name}", a.buffer(name, np.uint8), b.buffer(name, np.uint8), + hip_sentinel=D3D12_SENTINEL, d3d12_sentinel=ALT_SENTINEL) + finally: + del a, b + + def native_kernels(self, options, frames, out=OUT, max_render=MAX_RENDER): + """The kernels a native runtime launches for these frames (the launch trace); no reference run.""" + rt = self.make(3, True, False, opts(options, f"k_fill={D3D12_SENTINEL}"), out, max_render) + pos = os.path.getsize(TRACE) if os.path.exists(TRACE) else 0 + ran = {} + try: + for i, fr in enumerate(frames): + rw, rh, reset, E, pi = fr[:5] + x = inputs(np.random.default_rng(i), rw, rh, True, reset) + x["pre_exposure"], x["E_cur"] = pi, float(np.float32(E)) + rt.frame(x, 0, half=True) + lines, pos = trace_since(pos) + for ln in lines: + parts = ln.split() + if parts and parts[-1] == "d3d12": + ran[parts[0]] = ran.get(parts[0], 0) + 1 + finally: + del rt + return ran + + ODD = dict(out=(201, 121), max_render=(131, 79)) + FRAMES_ODD = [(131, 79, True, 1.0, 1.0), (131, 79, False, 1.0, 1.0), (131, 79, False, 1.25, 0.5), + (101, 61, False, 1.0, 1.0), (101, 61, True, 0.75, 1.0)] + + def test_native_split_m_matches_one_launch(self): + """k_split=m (the default on the native device) against k_split=off: the same bytes of the output and of every + written buffer, history and exposure state, frame by frame, at the 200 x 120 output and at an odd output size + (201 x 121 from 131 x 79).""" + self.native_pair("", "k_split=off", self.FRAMES) + self.native_pair("", "k_split=off", self.FRAMES_ODD, **self.ODD) + + def test_native_split_m_auto_exposure_dense(self): + """The same parity in the dense RGB output with auto-exposure (the exposure kernels' own path).""" + self.native_pair("", "k_split=off", self.FRAMES, hdr=True, mode=2, half=False) + + def test_native_split_trace(self): + """Stage m's launches on the native device: the phase launches with k_split=m (the default), k_stagew4_m with + k_split=off; with the developer hook k_smx_test=mix_fail the rounding is the round-to-odd port in both.""" + on = self.native_kernels("", self.FRAMES_Q, **self.QUALITY) + phases = ["k_split_m_attn", "k_split_m_wo", "k_split_m_w1", "k_split_m_w2"] + self.assertTrue(all(on.get(k) for k in phases), sorted(on)) + self.assertFalse([k for k in on if k in ("k_stagew4_m", "k_stagew4_m__rodd")], sorted(on)) + off = self.native_kernels("k_split=off", self.FRAMES_Q, **self.QUALITY) + self.assertTrue(off.get("k_stagew4_m"), sorted(off)) + self.assertFalse([k for k in off if k.startswith("k_split_m_")], sorted(off)) + self.need_dev_build() + self.expect_fallback_trace(self.native_kernels("k_smx_test=mix_fail", self.FRAMES_Q, **self.QUALITY), False, True) + self.expect_fallback_trace(self.native_kernels("k_smx_test=mix_fail;k_split=off", self.FRAMES_Q, **self.QUALITY), + False, False) + + def test_native_k_split_refusals(self): + """A k_split value other than m or off fails create on the native device in one line.""" + for value in ("maybe", "M", "on", "2"): + with self.subTest(value=value): + got = self.leg("k_split=" + value) + self.assertFalse(got["ok"], got) + self.assertIn("k_split must be m or off", got["error"]) + self.assertNotIn("\n", got["error"].strip()) + + def test_native_split_kernels_line(self): + """The kernels line names the split form (split=m) only when stage m is split; the self-check form is unchanged.""" + plain, off = self.leg(""), self.leg("k_split=off") + self.assertTrue(plain["ok"], plain) + self.assertTrue(off["ok"], off) + self.assertEqual((plain["split"], off["split"]), ("m", "")) + self.assertEqual((plain["smx"], plain["notes"]), (off["smx"], off["notes"])) + explicit = self.leg("k_split=m") + self.assertEqual(explicit["split"], "m", explicit) + + def test_native_split_fallback(self): + """k_smx_test=split_fail (developer build): the split form of stage m fails the create-time self-check, so create + takes the one launch: create succeeds, the kernels line has no split=m, the rounding is the mix form, one note of + create names the split form and the one-launch path, and the frames are the same bytes as with k_split=off.""" + self.need_dev_build() + fb = self.leg("k_smx_test=split_fail") + self.assertTrue(fb["ok"], fb) + self.assertEqual((fb["split"], fb["smx"]), ("", "mix")) + self.assertEqual(len(fb["notes"]), 1, fb) + note = fb["notes"][0] + self.assertIn("split form of stage m", note) + self.assertIn("one launch", note) + self.assertIn("test hook", note) + self.assertNotIn("\n", note) + self.native_pair("k_smx_test=split_fail", "k_split=off", self.FRAMES) + self.native_pair("k_smx_test=split_fail", "k_split=off", self.FRAMES_ODD, **self.ODD) # ---- k_approx: the opt-in close approximations (never the default) -------------------------------------------- def test_k_approx_refusals(self): @@ -581,16 +724,17 @@ def softmax_leg(mdir, map_path, options, build): try: rt = KRuntimeExt(build, mdir, map_path, OUT, MAX_RENDER, True, False, False, options, backend=3) except RuntimeError as e: - print(json.dumps({"ok": False, "smx": "", "notes": [], "k_approx": "", "error": str(e)})) + print(json.dumps({"ok": False, "smx": "", "notes": [], "k_approx": "", "split": "", "error": str(e)})) return rt.lib.wg_rt_notes.argtypes = [ctypes.c_void_p] rt.lib.wg_rt_notes.restype = ctypes.c_char_p kern = rt.lib.wg_rt_kernels(rt.rt).decode().split() smx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("smx=")] approx = [kv.split("=", 1)[1] for kv in kern if kv.startswith("k_approx=")] + split = [kv.split("=", 1)[1] for kv in kern if kv.startswith("split=")] notes = [n for n in rt.lib.wg_rt_notes(rt.rt).decode().split("\n") if n] print(json.dumps({"ok": True, "smx": smx[0] if smx else "", "notes": notes, "k_approx": approx[0] if approx else "", - "error": ""})) + "split": split[0] if split else "", "error": ""})) if __name__ == "__main__": diff --git a/clean/k/tests/test_k_smx_selfcheck_digest.py b/clean/k/tests/test_k_smx_selfcheck_digest.py index f92bf6dc..dd4b3091 100644 --- a/clean/k/tests/test_k_smx_selfcheck_digest.py +++ b/clean/k/tests/test_k_smx_selfcheck_digest.py @@ -54,7 +54,10 @@ def test_changes_are_caught(self): (os.path.join("clean", "dx12", "hlsl", "wg_math.hlsli"), "float"), (os.path.join("clean", "k", "dx12", "kernels", "k_block_wide.hlsl"), "KW_RD=12"), (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "kSmxSeed = 0x6b5eu"), - (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)")] + (os.path.join("clean", "k", "runtime", "model_k.cpp"), "(2 * oxm - 4)"), + (os.path.join("clean", "k", "dx12", "kernels", "k_block_split.hlsl"), "#define KS_RD 20"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "return split_m(kernel[11] != 0, a0);"), + (os.path.join("clean", "k", "runtime", "runtime_k.cpp"), "split_m_ = c.backend == 3")] for rel, needle in cases: with self.subTest(file=rel): tmp = tempfile.mkdtemp(prefix="smx_digest_")