diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index b2b49543..2bee0356 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -296,6 +296,46 @@ "k_expo_scalars_direct", "k_io.co" ], + [ + "k_stage2_d0_hidden", + "k_block.co" + ], + [ + "k_stage2_d0_hp", + "k_block.co" + ], + [ + "k_stage2_d1", + "k_block.co" + ], + [ + "k_stage2_e0", + "k_block.co" + ], + [ + "k_stage2_e1", + "k_block.co" + ], + [ + "k_stage_d1", + "k_block.co" + ], + [ + "k_stage_d2", + "k_block.co" + ], + [ + "k_stage_e0", + "k_block.co" + ], + [ + "k_stage_e1", + "k_block.co" + ], + [ + "k_stage_e2", + "k_block.co" + ], [ "k_warm", "k_block_wide.co" diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 427cd926..595fb21a 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -32,9 +32,10 @@ // module [] (zero or more) // buf [fill] device buffer: the file's bytes (padded with fill when the file is // shorter), or every byte set to fill (0..255, default 0) with "-" -// step ... a launch; each arg is @ (a buffer's pointer), a 32-bit decimal +// step ... a launch; each arg is @[+] (a buffer's pointer, optionally +// that many bytes into it), null (a null pointer), a 32-bit decimal // value, B: (a struct passed by value: the file's bytes) or -// S:[:=@[,=@...]] (a struct with pointers +// S:[:=@[+][,...]] (a struct with pointers // inside, manifest argument kind s:...: the file's bytes with the 8 // bytes at each byte offset replaced by that buffer's device pointer) // save after the run, the buffer's bytes are written to @@ -386,6 +387,14 @@ int run_chain(Device* d, Chain& c, const std::string& base) { if (b.id == id) return &b; return nullptr; }; + // "" or "+": a buffer's device pointer, optionally that many bytes into it + auto resolve = [&](const std::string& ref, DevPtr* out) -> bool { + const size_t plus = ref.find('+'); + ChainBuf* b = find(ref.substr(0, plus)); + if (!b) return false; + *out = b->ptr + (plus == std::string::npos ? 0 : (DevPtr)std::stoull(ref.substr(plus + 1))); + return true; + }; int rc = 0; for (auto& b : c.bufs) { std::vector data(b.bytes, (uint8_t)(b.fill & 0xff)); @@ -431,10 +440,13 @@ int run_chain(Device* d, Chain& c, const std::string& base) { const size_t comma = a.find(',', at); const std::string item = a.substr(at, comma == std::string::npos ? std::string::npos : comma - at); const size_t eq = item.find("=@"); - ChainBuf* b = eq == std::string::npos ? nullptr : find(item.substr(eq + 2)); + DevPtr target = 0; const size_t off = eq == std::string::npos ? 0 : (size_t)std::stoull(item.substr(0, eq)); - if (!b || off + 8 > blob.size()) { ok = false; break; } - for (int k = 0; k < 8; ++k) blob[off + k] = (uint8_t)((uint64_t)b->ptr >> (8 * k)); + if (eq == std::string::npos || off + 8 > blob.size() || !resolve(item.substr(eq + 2), &target)) { + ok = false; + break; + } + for (int k = 0; k < 8; ++k) blob[off + k] = (uint8_t)((uint64_t)target >> (8 * k)); at = comma == std::string::npos ? std::string::npos : comma + 1; } if (!ok) { @@ -444,10 +456,17 @@ int run_chain(Device* d, Chain& c, const std::string& base) { } blobs[i].push_back(std::move(blob)); argv[i].push_back(blobs[i].back().data()); + } else if (a == "null") { // a null pointer + ptrs[i].push_back(0); + argv[i].push_back(&ptrs[i].back()); } else if (!a.empty() && a[0] == '@') { - ChainBuf* b = find(a.substr(1)); - if (!b) { std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); rc = 1; break; } - ptrs[i].push_back(b->ptr); + DevPtr target = 0; + if (!resolve(a.substr(1), &target)) { + std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); + rc = 1; + break; + } + ptrs[i].push_back(target); argv[i].push_back(&ptrs[i].back()); } else { vals[i].push_back((uint32_t)std::stoll(a)); diff --git a/clean/k/dx12/README.md b/clean/k/dx12/README.md index 20f33cba..8a5af6b5 100644 --- a/clean/k/dx12/README.md +++ b/clean/k/dx12/README.md @@ -30,8 +30,17 @@ and starts filled with its run's own sentinel (HIP `0xA5`, D3D12 `0x5A`); the de slack must keep its sentinel; special values are among the inputs; a negative control (the port given another input than HIP) must be caught; Model M's `scale_rgb` runs through the same replay as a control. Output: counts and verdicts. -Families: `expo` (`k_expo_scalars`) against `k_io.co`; `warm` (`k_warm`, a cache warm-up with no HIP kernel: checked -against its definition, through the struct argument path). +Families: `narrow` (the narrow network stages of `k_block.co`: `k_stage_e2`, `k_stage_e0`, `k_stage_e1`, `k_stage_d2`, +`k_stage_d1`, `k_stage2_e0`, `k_stage2_e1`, `k_stage2_d1`, `k_stage2_d0_hp`, `k_stage2_d0_hidden`) on random stage weights +and inputs, the stage runner's launch plans run on HIP and replayed on D3D12 (`k_plan_bridge.py`); the inputs of every +third case carry special values (infinity, +-65504, zero, -0, subnormals) and of every third case NaN as well; the +half-grid control launches a stage with half its workgroups. `expo` (`k_expo_scalars`) against `k_io.co`; `warm` +(`k_warm`, a cache warm-up with no HIP kernel: checked against its definition, through the struct argument path). + +NaN: IEEE 754 does not define the sign or payload of a NaN, and where a NaN is generated or propagated the two devices +can differ in it (the decoder stages, with NaN in the input). For the cases with NaN in the input the check compares the +raw bytes and, where they differ, reports the elements that are NaN on both sides separately; every other byte must be +identical. Without NaN in the input the raw bytes are identical. Known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero and HIP keeps them. `k_expo_scalars` only meets one with a pre-exposure above about 8.5e37, so the check's inputs keep clear of them. diff --git a/clean/k/dx12/hlsl/k_blkparams.hlsli b/clean/k/dx12/hlsl/k_blkparams.hlsli new file mode 100644 index 00000000..a9c019e6 --- /dev/null +++ b/clean/k/dx12/hlsl/k_blkparams.hlsli @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: MIT +// Model K's stage kernel arguments for the D3D12 ports (WrongGPU's own code): KBlkParams of clean/k/kernels/k_block.hip +// as the manifest token s184:p19.b32 lays it out (clean/runtime/struct_args.h). Include before WG_ARGS_BEGIN and put +// KBLK_PARAMS_DECL inside the argument block; the kernel's other headers (k_block.hlsli, k_common.hlsli) come after +// WG_ARGS_END. +#pragma once + +// KBlkParams (clean/k/kernels/k_block.hip, 184 bytes) as the manifest token s184:p19.b32 lays it out: 19 pointers (one +// root constant each: the arena byte offset, 0 = null), then 8 ints. Declare inside WG_ARGS_BEGIN / WG_ARGS_END. +#define KBLK_PARAMS_DECL \ + uint P_x, P_skip, P_y, P_merged, P_head, P_g1, P_g2, P_wqkv, P_mix, P_wo, P_bo, P_w1, P_b1, P_w2, P_b2, P_wa, \ + P_ba, P_wz, P_bz; \ + int P_w, P_h, P_ox, P_oy, P_nwx, P_nwy, P_cw, P_ch; diff --git a/clean/k/dx12/hlsl/k_block.hlsli b/clean/k/dx12/hlsl/k_block.hlsli new file mode 100644 index 00000000..473877f2 --- /dev/null +++ b/clean/k/dx12/hlsl/k_block.hlsli @@ -0,0 +1,588 @@ +// SPDX-License-Identifier: MIT +// Model K block of spec 41 with its fused stage neighbours, for the D3D12 ports: the HLSL counterpart of the k_block +// template of clean/k/kernels/k_block.hip (WrongGPU's own code), FP32 accumulation only (template parameter F16 = 0: +// the network's default exact path; the FP16-accumulating k_stageh_* / k_stage2h_* forms and the FP8 forms are not +// ported). Steps, layouts and orders are those of k_block.hip (read its header comment): one 128-lane workgroup (4 waves) +// per G windows of 8x8 tokens; wave w owns window-local tokens 16 w .. 16 w + 15 as the WMMA columns; lane l holds +// token 16 w + l % 16 and channels 16 kt + 8 (l / 16) + 0..7 of it in every fragment. +// +// Every floating-point operation is the HIP kernel's, in its order and with its roundings (k_common.hlsli says how +// FP16 operations, conversions and the softmax weights reproduce the gfx1201 instructions); every FP32 accumulator +// starts from the same bias and runs over the same tiles in the same order, and is rounded once. Differences in form +// only (bit-identical): +// * driver-channel rules of model M (clean/dx12/README.md): each group of matrix products loads all its weight / +// operand fragments, in source order, before its first multiply (the channel calls are UAV atomics: the compiler +// never moves loads across them); no branches around the products (the mirror, the window clamps and the +// per-token conditions are selects; only stores are predicated); +// * LDS (one groupshared array of KB_LDS_HALFS: 8192 halfs = 16 KB, what every two-window stage needs; the one-window +// e0 / e1 forms declare their exact 6144 / 4096 halfs in k_block.hlsl, more workgroups per WGP) holds the +// same fragment-major K / VT tiles, the same x staging and the same NB (1 or 2 head buffers) as HIP; +// * the XG step 3 re-read of x for a member outside the map reads the mirrored member's slot of P.y instead and +// discards it (HIP loads nothing there): the value is the same zero; the address stays inside the map; +// * the hidden-history epilogue's exact rsqrt (k_fp.h kf_rsqrt: v_rsq_f32 estimate + binary64 rounding tests) uses +// HLSL doubles (DoublePrecisionFloatShaderOps incl. fma, as the RR / exposure ports); its binary64 fallback +// kd_rsqrt32 is reached on gfx1201 only for x = +inf (-> 0) and NaN (-> NaN). +#pragma once + +#include "k_blkparams.hlsli" +#include "k_common.hlsli" +#include "k_norm.hlsli" + +// KB_LEAN: the opaque zero only where a result feeds an FP16 add or multiply (k_common.hlsli wgk_*n); 0: everywhere +#ifndef KB_LEAN +#define KB_LEAN 2 +#endif +#if KB_LEAN >= 2 // also the softmax polynomial's 1 - k u^2 (feeds a multiply) and u (1 - k u^2) (feeds the f32 widening) +#define KB_SUB8S wgk_sub8n +#define KB_MUL8S wgk_mul8n +#else +#define KB_SUB8S wgk_sub8 +#define KB_MUL8S wgk_mul8 +#endif +#if KB_LEAN +#define KB_MUL8W wgk_mul8n +#define KB_ADD8W wgk_add8n +#define KB_CLAMP8W wgk_clamp8cn +#define KB_MAXC8W wgk_maxc8n +#define KB_ACT8W wgk_act8n +#else +#define KB_MUL8W wgk_mul8 +#define KB_ADD8W wgk_add8 +#define KB_CLAMP8W wgk_clamp8c +#define KB_MAXC8W wgk_maxc8 +#define KB_ACT8W wgk_act8 +#endif + +#define KMIX_FIXED 0 +#define KMIX_SOFTMAX 1 +#define KIN_MAP 0 +#define KIN_PADDED 1 +#define KIN_EMBED 2 +#define KIN_EXPAND 3 + +#define WGK_SOFTMAX_C 0x4d69u // 21.640625 +#define WGK_SOFTMAX_KAPPA 0x11d5u // 0.0007119178771972656 + +#define KB_FRAG 256 // halfs per 16 x 16 tile (32 lanes x 8 halfs, fragment order) +#ifndef KB_LDS_HALFS +#define KB_LDS_HALFS 8192 // LDS of the stage kernels of k_block.hlsl (NB / LH1 / LH2 as in HIP: 16 KB each) +#endif +groupshared uint4 kb_lds[KB_LDS_HALFS / 8]; +uint4 kb_ldl(uint off_h) { return kb_lds[off_h >> 3]; } +void kb_stl(uint off_h, uint4 v) { kb_lds[off_h >> 3] = v; } + +// KB_PREF (default 0): every weight section of the stage is touched once per workgroup at the start (one dword per +// 128-byte line, spread over the 128 lanes), so a workgroup that finds them cold (in the frame, after the previous +// stage streamed its activations) waits for one round of parallel misses instead of one miss per weight tile along its +// serial chain. The loaded words only feed a pin (0 at run time) added to one later load's base: no arithmetic changes. +// Measured in the game-like hipbench (2026-11-02): on all six stages +0.019 ms per frame, on e1 alone -0.001 ms: off. +#ifndef KB_PREF +#define KB_PREF 0 +#endif +uint kb_touch(uint base, int halfs, uint tid) +{ + uint v = 0u; + [unroll] for (int j = 0; j < (2 * halfs + 16383) / 16384; ++j) + v ^= wg_ld32(base, min(tid * 128u + (uint)j * 16384u, (uint)(2 * halfs) - 4u)); + return v; +} + +// ---- hidden-history epilogue (spec 43 section 6.8; HIP wgk_hidden_norm2 / wgk_store_hidden*) -------------------------- +float wgk_hidden_norm2(float h[4]) +{ + float ss = h[1] * h[1]; + ss = mad(h[0], h[0], ss); + ss = mad(h[2], h[2], ss); + ss = mad(h[3], h[3], ss); + return ss + 1.0f; +} + +// k_fp.h kf_rsqrt: 1 / sqrt(x) correctly rounded to FP32 for x >= 1 (estimate v_rsq_f32, then up to three exact +// binary64 midpoint tests moving it by one ulp); kd_rsqrt32 (binary64 sqrt + division) for the rest, which here is only +// x = +inf (-> +0) or NaN (the hidden norm x = sum of squares + 1 is never below 1). +float wgk_kf_rsqrt(float x) +{ + const bool fin = x >= 1.0f && x - x == 0.0f; + float y = wg_rsq(x); + bool done = !fin; + const double xd = (double)x; + [unroll] for (int it = 0; it < 3; ++it) { + const uint b = asuint(y); + const double yd = (double)y; + const double ulp = (double)asfloat((b & 0x7F800000u) - (23u << 23)); + const double up = yd + 0.5 * ulp; + const double lo = yd - ((b & 0x7FFFFFu) == 0u ? 0.25 : 0.5) * ulp; + const bool inc = fma(up * up, xd, -1.0) < 0.0; + const bool dec = !inc && fma(lo * lo, xd, -1.0) > 0.0; + const uint nb = inc ? b + 1u : (dec ? b - 1u : b); + y = done ? y : asfloat(nb); + done = done || (!inc && !dec); + } + // not finite / below 1 (only +inf and NaN reach here): kd_rsqrt32 + const float other = isnan(x) ? asfloat(asuint(x) | 0x400000u) : (x == asfloat(0x7f800000u) ? 0.0f : y); + return fin ? y : other; +} + +bool wgk_hidden_half_safe(float p) +{ + const uint a = asuint(p) & 0x7fffffffu; + const uint low = a & 0x1fffu; + return a >= 0x38800000u && a <= 0x477fe000u && (low < 0x0ff8u || low > 0x1008u); +} + +// HIP wgk_store_hidden_guarded: the hardware estimate when every product is provably on the same side of its FP16 +// rounding midpoint (gfx1201 bound: rsq is within one float of kf_rsqrt for x in [1, 2^34]), else the exact form. +// The four FP16 values go out as one 8-byte store at dst (the same bytes as HIP's four 2-byte stores). +void wgk_store_hidden(uint base, uint off_h, float h[4]) +{ + const float x = wgk_hidden_norm2(h); + const float est = wg_rsq(x); + bool safe = x >= 1.0f && x <= 17179869184.0f; + float pe[4]; + [unroll] for (int k = 0; k < 4; ++k) { + pe[k] = h[k] * est; + safe = safe && wgk_hidden_half_safe(pe[k]); + } + float pr[4]; + if (!safe) { + const float nu = wgk_kf_rsqrt(x); + [unroll] for (int k2 = 0; k2 < 4; ++k2) pr[k2] = h[k2] * nu; + } else { + [unroll] for (int k3 = 0; k3 < 4; ++k3) pr[k3] = pe[k3]; + } + wg_st64(base, off_h * 2u, uint2(wg_f2h(pr[0]) | (wg_f2h(pr[1]) << 16), wg_f2h(pr[2]) | (wg_f2h(pr[3]) << 16))); +} + +// ---- the block ---------------------------------------------------------------------------------------------------- +// Template parameters as k_block.hip (F16 = 0): C channels, HEADS, MIX (KMIX_*), IN (KIN_MAP / KIN_EMBED / KIN_EXPAND; +// KIN_PADDED as well), CIN input channels of EMBED / EXPAND, CNEXT merge output channels (0: none), HEAD (32 -> 40 output +// head), G windows per workgroup, HIDDEN (hidden-history epilogue; G = 2 head stage; hidden = its pointer), PIX (compact +// pixel head), XG (decoder: x = f16(e + S) through P.y instead of LDS). blk: the workgroup's window group (windows +// G blk .. G blk + G - 1): wg_block_id(). +template +void k_block(uint hidden, uint blk) +{ + const int KT = C / 16; + const int DT = HEADS * 2; + const int HT = 4 * C / 16; + const int QKV = MIX == KMIX_SOFTMAX ? 3 : 1; + const bool XS = IN == KIN_EMBED || IN == KIN_EXPAND; + const bool MS = CNEXT > 0; + const int KH = QKV == 3 ? 8 * KB_FRAG : 0, VH = 8 * KB_FRAG, XH = (XS || MS) ? 64 * C : 0; + const bool XA = (MS && !XS) || XG; + const int LH1 = XA ? max(G * (KH + VH), G * XH) : G * (KH + VH) + G * XH; + const int LH2 = XA ? max(G * 2 * (KH + VH), G * XH) : G * 2 * (KH + VH) + G * XH; + // kb_wg_per_wgp(bytes) = 131072 / bytes; NB = 2 if wg(2 LH2) >= min(8, wg(2 LH1)). KB_NB_FORCE (a kernel's + // defines): that NB instead (the number of head buffers changes only which LDS slots and barriers are used) +#ifndef KB_NB_FORCE +#define KB_NB_FORCE 0 +#endif + const int NB = KB_NB_FORCE ? KB_NB_FORCE : ((131072 / (2 * LH2)) >= min(8, 131072 / (2 * LH1)) ? 2 : 1); + const uint lds_x = XA ? 0u : (uint)(G * NB * (KH + VH)); + const uint KVW = (uint)(NB * (KH + VH)); // LDS halfs between the windows' K / VT sets + + const uint tid = wg_lane_id(); + const uint lane = tid & 31u; + const uint wave = (uint)wg_uniform((int)(tid >> 5)); // wave-uniform (scalar): the loops over tiles by wave + const int n16 = (int)(lane & 15u), hh = (int)(lane >> 4); + const int nwin = P_nwx * P_nwy; + const int t = 16 * (int)wave + n16; + int wa[G], wb[G], gx[G], gy[G]; + bool real[G]; + uint mtok[G], xt[G], xl[G], ytok[G]; + [unroll] for (int g = 0; g < G; ++g) { + const int wraw = (int)blk * G + g; + const int win = select(wraw < nwin, wraw, nwin - 1); + wa[g] = (int)((uint)win % (uint)P_nwx); + wb[g] = (int)((uint)win / (uint)P_nwx); + gx[g] = 8 * wa[g] - P_ox + (t & 7); + gy[g] = 8 * wb[g] - P_oy + (t >> 3); + real[g] = and(and(wraw < nwin, and(gx[g] >= 0, gx[g] < P_w)), and(gy[g] >= 0, gy[g] < P_h)); + mtok[g] = (uint)(wgk_mirror(gy[g], P_h) * P_w + wgk_mirror(gx[g], P_w)); + const uint tok = IN == KIN_PADDED ? (uint)((8 * wb[g] + (t >> 3)) * (8 * P_nwx) + 8 * wa[g] + (t & 7)) : mtok[g]; + xt[g] = tok * (uint)C + 8u * (uint)hh; // halfs from P_x (MAP / PADDED) + xl[g] = lds_x + (uint)(g * XH + t * C + 8 * hh); // halfs in LDS (XS / MS) + ytok[g] = real[g] ? (uint)(gy[g] * P_w + gx[g]) : mtok[g]; // this member's map token (in range) + } + + // g1 first: its loads overlap the block input's + uint4 g1v[C / 16]; + [unroll] for (int k0 = 0; k0 < KT; ++k0) g1v[k0] = wgk_ld8(P_g1, (uint)(16 * k0 + 8 * hh)); + + // 1. the block input + uint4 n[G][C / 16]; + if (IN == KIN_EMBED) { // e(t) = max(f16(We in(t) + be), 0) of the mirrored member + uint4 inp[G], ea[C / 16], eb[C / 16]; + [unroll] for (int g = 0; g < G; ++g) inp[g] = wgk_ld8(P_x, mtok[g] * 16u + 8u * (uint)hh); + [unroll] for (int o0 = 0; o0 < KT; ++o0) { + ea[o0] = wgk_ld8(P_wa, (uint)(o0 * 256) + lane * 8u); + eb[o0] = wgk_ld8(P_ba, (uint)(16 * o0 + 8 * hh)); + } + [unroll] for (int ot = 0; ot < KT; ++ot) { + const v8f b0 = wgk_widen8(eb[ot]); + [unroll] for (int g = 0; g < G; ++g) { + n[g][ot] = KB_MAXC8W(wgk_round8(wgk_mma(ea[ot], inp[g], b0)), 0u); + kb_stl(xl[g] + (uint)(16 * ot), n[g][ot]); + } + } + } else if (IN == KIN_EXPAND) { + // the window's 4 x 4 coarse tokens: column j = lane % 16 is coarse (R0 + j / 4, C0 + j % 4), mirrored + const int KTI = CIN / 16, TOT = 4 * C / 16; + const int jr = n16 >> 2, jc = n16 & 3; + uint4 yb[G][CIN / 16]; + [unroll] for (int g = 0; g < G; ++g) { + const int cr = (8 * wb[g] - P_oy) / 2 + jr, cc = (8 * wa[g] - P_ox) / 2 + jc; // even offsets: exact halves + const uint yc = (uint)(wgk_mirror(cr, P_ch) * P_cw + wgk_mirror(cc, P_cw)) * (uint)CIN + 8u * (uint)hh; + [unroll] for (int kt = 0; kt < KTI; ++kt) yb[g][kt] = wgk_ld8(P_x, yc + (uint)(16 * kt)); + } + [loop] for (int tt = (int)wave; tt < TOT; tt += 4) { // output tile tt of 4C: sub-position s, channels c0 .. + uint4 wt[CIN / 16]; + const uint4 bb = wgk_ld8(P_ba, (uint)(16 * tt + 8 * hh)); + [unroll] for (int k1 = 0; k1 < KTI; ++k1) wt[k1] = wgk_ld8(P_wa, (uint)((tt * KTI + k1) * 256) + lane * 8u); + v8f e[G]; + const v8f b0 = wgk_widen8(bb); + [unroll] for (int g = 0; g < G; ++g) e[g] = b0; + [unroll] for (int kt = 0; kt < KTI; ++kt) + [unroll] for (int g = 0; g < G; ++g) e[g] = wgk_mma(wt[kt], yb[g][kt], e[g]); + const int s = (16 * tt) / C, c0 = 16 * tt - s * C; + const int ft = (2 * jr + (s >> 1)) * 8 + 2 * jc + (s & 1); // the window member this sub-position feeds + [unroll] for (int g = 0; g < G; ++g) + kb_stl(lds_x + (uint)(g * XH + ft * C + c0 + 8 * hh), wgk_round8(e[g])); + } + wg_barrier_lds(); + [unroll] for (int g = 0; g < G; ++g) { + const uint sk = mtok[g] * (uint)C + 8u * (uint)hh; + uint4 sv[C / 16]; + [unroll] for (int k2 = 0; k2 < KT; ++k2) sv[k2] = wgk_ld8(P_skip, sk + (uint)(16 * k2)); + [unroll] for (int kt = 0; kt < KT; ++kt) { + n[g][kt] = wgk_add8(kb_ldl(xl[g] + (uint)(16 * kt)), sv[kt]); // x = f16(e + S); own slots only + if (!XG) + kb_stl(xl[g] + (uint)(16 * kt), n[g][kt]); + else if (real[g]) + wgk_st8(P_y, ytok[g] * (uint)C + 8u * (uint)hh + (uint)(16 * kt), n[g][kt]); + } + } + if (XG) wg_barrier_lds(); // the staging is read; LDS goes to K / VT + } else { + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int kt = 0; kt < KT; ++kt) n[g][kt] = wgk_ld8(P_x, xt[g] + (uint)(16 * kt)); + } + + // (after the block input's loads: the touches' wait coincides with theirs) + uint pref = 0u; + if (KB_PREF) { + pref ^= kb_touch(P_wqkv, HEADS * QKV * 2 * KT * 256, tid); + pref ^= kb_touch(P_mix, HEADS * 64 * 64, tid); + pref ^= kb_touch(P_wo, KT * DT * 256, tid); + pref ^= kb_touch(P_w1, HT * KT * 256, tid); + pref ^= kb_touch(P_w2, HT * KT * 256, tid); + if (MS) pref ^= kb_touch(P_wz, (CNEXT / 16) * (4 * C / 16) * 256, tid); + } + + // normalization: s = f16(sum f16(x^2)) (FP32 sum, channel order), r = f16(1 / sqrt(s)); s = 0 gives r = 0 + float ss[G]; + [unroll] for (int g = 0; g < G; ++g) ss[g] = 0.0f; + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) ss[g] = wgk_sumsq8(n[g][kt], ss[g]); + [unroll] for (int g = 0; g < G; ++g) ss[g] = ss[g] + wgk_xor16(ss[g]); + [unroll] for (int g = 0; g < G; ++g) { + const uint4 rv = wgk_splat8h(wgk_norm_reciprocal(wg_f2h(ss[g]))); + [unroll] for (int kt = 0; kt < KT; ++kt) n[g][kt] = KB_MUL8W(n[g][kt], wgk_mul8(rv, g1v[kt])); + } + + // 2. token mixing, head by head; y accumulates the output projection + v8f yacc[G][C / 16]; + { + uint4 bo[C / 16]; + [unroll] for (int o1 = 0; o1 < KT; ++o1) bo[o1] = wgk_ld8(P_bo, (uint)(16 * o1 + 8 * hh)); + [unroll] for (int ot = 0; ot < KT; ++ot) { + const v8f b0 = wgk_widen8(bo[ot]); + [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = b0; + } + } + + const uint wl = lane * 8u; + const uint pmix = KB_PREF ? P_mix + wgk_pin(pref) : P_mix; // KB_PREF: the touched words arrive before the heads +#ifndef KB_GRPLATE +#define KB_GRPLATE 0 +#endif + // (the head and MLP loops are left to the driver, which unrolls them: an opaque trip count that keeps them rolled + // measured slower here, d1 +7 us / e1 +10 us with the MLP rolled one tile per iteration, no gain with two) + [loop] for (int hd = 0; hd < HEADS; ++hd) { + const uint wh = wl + (uint)(hd * QKV * 2 * KT * 256); + const uint kv = (uint)((NB == 2 ? (hd & 1) : 0) * (KH + VH)) + lane * 8u; // this lane's slot, window 0 + uint4 q[G][2]; + [unroll] for (int ct = 0; ct < 2; ++ct) { + if (MIX == KMIX_SOFTMAX) { + uint4 wq[C / 16], wk[C / 16]; + [unroll] for (int k3 = 0; k3 < KT; ++k3) { + wq[k3] = wgk_ld8(P_wqkv, wh + (uint)((0 * 2 * KT + ct * KT + k3) * 256)); + wk[k3] = wgk_ld8(P_wqkv, wh + (uint)((1 * 2 * KT + ct * KT + k3) * 256)); + } + v8f aq[G], ak[G]; + [unroll] for (int g = 0; g < G; ++g) { aq[g] = wgk_zero8f(); ak[g] = wgk_zero8f(); } + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) { + aq[g] = wgk_mma(wq[kt], n[g][kt], aq[g]); + ak[g] = wgk_mma(wk[kt], n[g][kt], ak[g]); + } + if (NB == 1 && ct == 0 && hd > 0) wg_barrier_lds(); // every wave has read the previous head's K / VT + [unroll] for (int g = 0; g < G; ++g) { + q[g][ct] = wgk_round8(aq[g]); + kb_stl(kv + (uint)g * KVW + (uint)((2 * (int)wave + ct) * KB_FRAG), wgk_round8(ak[g])); // K tile + } + } + // v^T: rows = channels 16 ct + lane % 16 (B columns), elements = tokens 16 wave + 8 hh + 0..7 + uint4 wv[C / 16]; + [unroll] for (int k4 = 0; k4 < KT; ++k4) + wv[k4] = wgk_ld8(P_wqkv, wh + (uint)(((QKV - 1) * 2 * KT + ct * KT + k4) * 256)); + v8f av[G]; + [unroll] for (int g = 0; g < G; ++g) av[g] = wgk_zero8f(); + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) av[g] = wgk_mma(n[g][kt], wv[kt], av[g]); + if (NB == 1 && MIX == KMIX_FIXED && ct == 0 && hd > 0) wg_barrier_lds(); + [unroll] for (int g = 0; g < G; ++g) + kb_stl(kv + (uint)g * KVW + (uint)(KH + (4 * ct + (int)wave) * KB_FRAG), wgk_round8(av[g])); // VT + } + wg_barrier_lds(); + + const uint mrow = (uint)((hd * 64 + t) * 64 + 8 * hh); + uint4 mx[4]; // A^T = M[h][t][s] (fixed) or the bias B[h][t][s] (softmax): the same for every window + [unroll] for (int s0 = 0; s0 < 4; ++s0) mx[s0] = wgk_ld8(pmix, mrow + (uint)(16 * s0)); + uint4 z[G][2]; + [unroll] for (int g = 0; g < G; ++g) { + const uint lk = kv + (uint)g * KVW; + const uint lv = lk + (uint)KH; + uint4 pa[4]; + if (MIX == KMIX_FIXED) { + [unroll] for (int s1 = 0; s1 < 4; ++s1) pa[s1] = mx[s1]; + } else { + uint4 kf[8]; + [unroll] for (int f0 = 0; f0 < 8; ++f0) kf[f0] = kb_ldl(lk + (uint)(f0 * KB_FRAG)); + uint4 gg[4]; + [unroll] for (int st = 0; st < 4; ++st) { + v8f sc = wgk_widen8(mx[st]); // + B[h][t][s] + [unroll] for (int ct = 0; ct < 2; ++ct) sc = wgk_mma(kf[2 * st + ct], q[g][ct], sc); + const uint4 u = KB_CLAMP8W(wgk_round8(sc), WGK_SOFTMAX_C | 0x8000u, WGK_SOFTMAX_C); + const uint4 u2 = wgk_mul8(u, u); + const uint4 k2 = wgk_mul8(wgk_splat8h(WGK_SOFTMAX_KAPPA), u2); + const uint4 o = KB_SUB8S(wgk_splat8h(WGK_H_ONE), k2); + gg[st] = KB_MUL8S(u, o); + } + wgk_softmax_weights(gg, pa); + } + // z_h^T = VT A^T (KB_GRPLATE: each channel tile's 4 VT fragments just before its 4 products) + uint4 vf[8]; + if (!KB_GRPLATE) + [unroll] for (int f1 = 0; f1 < 8; ++f1) vf[f1] = kb_ldl(lv + (uint)(f1 * KB_FRAG)); + [unroll] for (int ct = 0; ct < 2; ++ct) { + if (KB_GRPLATE) + [unroll] for (int f2 = 0; f2 < 4; ++f2) vf[4 * ct + f2] = kb_ldl(lv + (uint)((4 * ct + f2) * KB_FRAG)); + v8f az = wgk_zero8f(); + [unroll] for (int st = 0; st < 4; ++st) az = wgk_mma(vf[4 * ct + st], pa[st], az); + z[g][ct] = wgk_round8(az); + } + } + // the output projection: each Wo tile once for the G windows (KB_GRPLATE: one channel tile's Wo tiles at a time) + uint4 wo[2][C / 16]; + if (!KB_GRPLATE) + [unroll] for (int c1 = 0; c1 < 2; ++c1) + [unroll] for (int o2 = 0; o2 < KT; ++o2) + wo[c1][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + c1) * 256) + lane * 8u); + [unroll] for (int ct = 0; ct < 2; ++ct) { + if (KB_GRPLATE) + [unroll] for (int o2 = 0; o2 < KT; ++o2) + wo[ct][o2] = wgk_ld8(P_wo, (uint)((o2 * DT + 2 * hd + ct) * 256) + lane * 8u); + [unroll] for (int ot = 0; ot < KT; ++ot) + [unroll] for (int g = 0; g < G; ++g) yacc[g][ot] = wgk_mma(wo[ct][ot], z[g][ct], yacc[g][ot]); + } + } + + // 3. x1 = f16(x + y); 4. a = f16(x1 g2) + uint4 x1[G][C / 16], a[G][C / 16]; + { + uint4 g2v[C / 16], xin[G][C / 16]; + [unroll] for (int k5 = 0; k5 < KT; ++k5) g2v[k5] = wgk_ld8(P_g2, (uint)(16 * k5 + 8 * hh)); + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) { + if (XG) + xin[g][kt] = wgk_ld8(P_y, ytok[g] * (uint)C + 8u * (uint)hh + (uint)(16 * kt)); + else if (XS) + xin[g][kt] = kb_ldl(xl[g] + (uint)(16 * kt)); + else + xin[g][kt] = wgk_ld8(P_x, xt[g] + (uint)(16 * kt)); + } + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) { + const uint4 xv = XG ? (real[g] ? xin[g][kt] : wgk_zero8()) : xin[g][kt]; + x1[g][kt] = wgk_add8(xv, wgk_round8(yacc[g][kt])); + a[g][kt] = KB_MUL8W(x1[g][kt], g2v[kt]); + } + } + + // x1 waits in LDS during the MLP (X1L: the two-window C = 64 stages, whose MLP is the register peak): each lane + // its own x staging slots (xl); where x shares the K / VT space (XA) every wave must have read the last head first +#ifndef KB_X1LDS +#define KB_X1LDS 0 +#endif + const bool X1L = KB_X1LDS && G == 2 && C >= 64 && XH > 0; + if (X1L) { + if (XA) wg_barrier_lds(); + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int kt = 0; kt < KT; ++kt) kb_stl(xl[g] + (uint)(16 * kt), x1[g][kt]); + } + + // 4. MLP: per hidden tile p = f16(W1 a + b1), h = act(p), acc += W2 h (FP32, started from b2) + v8f acc[G][C / 16]; + { + uint4 b2v[C / 16]; + [unroll] for (int o3 = 0; o3 < KT; ++o3) b2v[o3] = wgk_ld8(P_b2, (uint)(16 * o3 + 8 * hh)); + [unroll] for (int ot = 0; ot < KT; ++ot) { + const v8f b0 = wgk_widen8(b2v[ot]); + [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = b0; + } + } + [loop] for (int ht = 0; ht < HT; ++ht) { + uint4 w1t[C / 16], w2t[C / 16]; + const uint4 b1v = wgk_ld8(P_b1, (uint)(16 * ht + 8 * hh)); + [unroll] for (int k6 = 0; k6 < KT; ++k6) w1t[k6] = wgk_ld8(P_w1, (uint)((ht * KT + k6) * 256) + lane * 8u); +#ifndef KB_W2LATE +#define KB_W2LATE 0 +#endif + if (!KB_W2LATE) + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); + v8f p[G]; + const v8f b0 = wgk_widen8(b1v); + [unroll] for (int g = 0; g < G; ++g) p[g] = b0; + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) p[g] = wgk_mma(w1t[kt], a[g][kt], p[g]); + uint4 hv[G]; + [unroll] for (int g = 0; g < G; ++g) hv[g] = KB_ACT8W(wgk_round8(p[g])); + if (KB_W2LATE) // W2's tiles just before their multiplies (fewer live registers across the first GEMM) + [unroll] for (int o4 = 0; o4 < KT; ++o4) w2t[o4] = wgk_ld8(P_w2, (uint)((ht * KT + o4) * 256) + lane * 8u); + [unroll] for (int ot = 0; ot < KT; ++ot) + [unroll] for (int g = 0; g < G; ++g) acc[g][ot] = wgk_mma(w2t[ot], hv[g], acc[g][ot]); + } + + // 5. x2, stored for the real positions (the stores' base pinned to every x2 fragment: products before the branch) + uint pin5 = 0u; + if (X1L) + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int kt = 0; kt < KT; ++kt) x1[g][kt] = kb_ldl(xl[g] + (uint)(16 * kt)); + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int ot = 0; ot < KT; ++ot) { + x1[g][ot] = KB_ADD8W(x1[g][ot], wgk_round8(acc[g][ot])); // x1 now holds x2 + pin5 ^= x1[g][ot].x; + } + const uint py = P_y + wgk_pin(pin5); + [unroll] for (int g = 0; g < G; ++g) { + if (real[g] && P_y != 0u) { + [unroll] for (int o5 = 0; o5 < KT; ++o5) + wgk_st8(py, ytok[g] * (uint)C + 8u * (uint)hh + (uint)(16 * o5), x1[g][o5]); + } + } + + if (MS) { // 2x2 merge of each window's 16 groups: f16(Wm concat(TL, TR, BL, BR) + bm) + if (XA && !X1L) wg_barrier_lds(); // x shares the K / VT space: every wave has read the last head's + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int ot = 0; ot < KT; ++ot) kb_stl(xl[g] + (uint)(16 * ot), real[g] ? x1[g][ot] : wgk_zero8()); + wg_barrier_lds(); + const int KTM = 4 * C / 16, NTM = CNEXT / 16; + const int gr = n16 >> 2, gc = n16 & 3; // group (column n16): window rows 2 gr, 2 gr + 1, columns 2 gc, 2 gc + 1 + const int mw = (P_w + 1) / 2, mh = (P_h + 1) / 2; + [loop] for (int tt = (int)wave; tt < NTM; tt += 4) { + v8f m[G]; + const v8f b0 = wgk_widen8(wgk_ld8(P_bz, (uint)(16 * tt + 8 * hh))); + [unroll] for (int g = 0; g < G; ++g) m[g] = b0; + [unroll] for (int k7 = 0; k7 < KTM; k7 += 4) { // in groups of 4 input tiles (HIP: unroll 4) + uint4 wm[4], xm[G][4]; + [unroll] for (int i = 0; i < 4; ++i) { + const int kt = k7 + i; + const int s = (16 * kt) / C, c0 = 16 * kt - s * C; + const int ft = (2 * gr + (s >> 1)) * 8 + 2 * gc + (s & 1); + wm[i] = wgk_ld8(P_wz, (uint)((tt * KTM + kt) * 256) + lane * 8u); + [unroll] for (int g = 0; g < G; ++g) xm[g][i] = kb_ldl(lds_x + (uint)(g * XH + ft * C + c0 + 8 * hh)); + } + [unroll] for (int i2 = 0; i2 < 4; ++i2) + [unroll] for (int g = 0; g < G; ++g) m[g] = wgk_mma(wm[i2], xm[g][i2], m[g]); + } + uint4 mo[G]; + uint pinm = 0u; + [unroll] for (int g = 0; g < G; ++g) { + mo[g] = wgk_round8(m[g]); + pinm ^= mo[g].x; + } + const uint pm = P_merged + wgk_pin(pinm); + [unroll] for (int g = 0; g < G; ++g) { + const int my = (8 * wb[g] - P_oy) / 2 + gr, mxx = (8 * wa[g] - P_ox) / 2 + gc; + const bool here = (int)blk * G + g < nwin; + if (here && my >= 0 && my < mh && mxx >= 0 && mxx < mw) + wgk_st8(pm, (uint)((my * mw + mxx) * CNEXT + 16 * tt + 8 * hh), mo[g]); + } + } + } + + if (HEAD) { // out = f16(Wh x2 + bh), 40 of 48 padded rows + uint4 heads[G][3]; + uint4 whd[3][C / 16], bhd[3]; + [unroll] for (int o6 = 0; o6 < 3; ++o6) { + bhd[o6] = wgk_ld8(P_bz, (uint)(16 * o6 + 8 * hh)); + [unroll] for (int k8 = 0; k8 < KT; ++k8) whd[o6][k8] = wgk_ld8(P_wz, (uint)((o6 * KT + k8) * 256) + lane * 8u); + } + [unroll] for (int ot = 0; ot < 3; ++ot) { + v8f o[G]; + const v8f b0 = wgk_widen8(bhd[ot]); + [unroll] for (int g = 0; g < G; ++g) o[g] = b0; + [unroll] for (int kt = 0; kt < KT; ++kt) + [unroll] for (int g = 0; g < G; ++g) o[g] = wgk_mma(whd[ot][kt], x1[g][kt], o[g]); + [unroll] for (int g = 0; g < G; ++g) heads[g][ot] = wgk_round8(o[g]); + } + uint pinh = 0u; + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int o8 = 0; o8 < 3; ++o8) pinh ^= heads[g][o8].x; + const uint phead = P_head + wgk_pin(pinh); // the head products before the predicated stores + if (!PIX) { + [unroll] for (int g = 0; g < G; ++g) + [unroll] for (int o7 = 0; o7 < 3; ++o7) + if (real[g] && 16 * o7 + 8 * hh < 40) + wgk_st8(phead, ytok[g] * 40u + (uint)(16 * o7 + 8 * hh), heads[g][o7]); + } else { + // Compact pixel head (k_block.hip): P.head is [2 h][2 w][4] FP16; slot k of token (gx, gy) at row + // 2 gy + (k >> 1), column 2 gx + (k & 1) holds (channel 10 k, 10 k + 1, 10 k + 9, 0). A lane of half hh has + // channels 10 k, 10 k + 1 of slots k = hh and 2 + hh and gets the third value from its partner lane. + [unroll] for (int g = 0; g < G; ++g) { + const uint4 h0 = heads[g][0], h1 = heads[g][1], h2 = heads[g][2]; + const uint a0 = hh ? wgk_el(h0, 2u) : wgk_el(h0, 0u), a1 = hh ? wgk_el(h0, 3u) : wgk_el(h0, 1u); + const uint c0 = hh ? wgk_el(h1, 6u) : wgk_el(h1, 4u), c1 = hh ? wgk_el(h1, 7u) : wgk_el(h1, 5u); + // to the partner: channel 9 + 10 k of its slots (hh = 0 holds 19, 39; hh = 1 holds 9, 29) + const uint snd = (hh ? wgk_el(h0, 1u) : wgk_el(h1, 3u)) | ((hh ? wgk_el(h1, 5u) : wgk_el(h2, 7u)) << 16); + const uint rcv = wgk_xor16_u(snd); + if (real[g]) { + const uint d = ((uint)(2 * gy[g]) * (uint)(2 * P_w) + (uint)(2 * gx[g] + hh)) * 4u; + wg_st64(phead, d * 2u, uint2(a0 | (a1 << 16), rcv & 0xffffu)); + wg_st64(phead, (d + (uint)(2 * P_w) * 4u) * 2u, uint2(c0 | (c1 << 16), rcv >> 16)); + } + } + } + if (HIDDEN) { + // Each lane half owns one of the G = 2 windows for this epilogue; the partner lane sends the opposite + // window's fragment, so xor16 supplies the missing half. All lanes exchange before the real-position guard. + float hs[4]; + [unroll] for (int k = 0; k < 4; ++k) { + float v[5]; + [unroll] for (int j = 0; j < 5; ++j) { + const int c = 10 * k + 3 + j; + const float own = hh == 0 ? wgk_elf(heads[0][c / 16], (uint)(c % 8)) : wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)); + const float snd = hh == 0 ? wgk_elf(heads[G - 1][c / 16], (uint)(c % 8)) : wgk_elf(heads[0][c / 16], (uint)(c % 8)); + const float other = wgk_xor16(snd); + v[j] = ((c % 16) / 8) == hh ? own : other; + } + // opaque: (0.2 s) est must stay two products (the driver regroups multiplication chains) + hs[k] = asfloat(wgk_opq(asuint(0.2f * ((((v[0] + v[1]) + v[2]) + v[3]) + v[4])))); + } + const bool rh = hh == 0 ? real[0] : real[G - 1]; + const uint tk = hh == 0 ? ytok[0] : ytok[G - 1]; + if (rh) wgk_store_hidden(hidden, tk * 4u, hs); + } + } +} diff --git a/clean/k/dx12/hlsl/k_common.hlsli b/clean/k/dx12/hlsl/k_common.hlsli new file mode 100644 index 00000000..187a0f0d --- /dev/null +++ b/clean/k/dx12/hlsl/k_common.hlsli @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: MIT +// Model K network helpers for the D3D12 ports: the HLSL counterpart of clean/k/kernels/k_common.h (WrongGPU's own code), +// with the same names (wgk_*) and the same bits as the HIP kernels built from it (gfx1201, k_block.co: WGK_PAIR_CVT = 1, +// WGK_ASM_CLAMP = 1, FP32 accumulation; the FP16-accumulating KAcc forms are not ported). +// +// Include AFTER the kernel's WG_ARGS_END: the opaque zero below reads the runtime constant wg_grid_. +// +// Types: a fragment of 8 FP16 values (HIP v8h) is a uint4, element i in dword i / 2, low half first; an FP32 +// accumulator (HIP v8f) is the platform's v8f. WMMA layout and lane mapping as in k_common.h: weights = A operand +// (rows = output channels), tokens = B operand (columns = tokens); lane l holds column l % 16 and the 8 contiguous +// K elements 8 (l / 16) + 0..7; the gfx12 accumulator rows d_row(l / 16, 0..7) = 8 (l / 16) + 0..7 (the packer +// absorbs any other order, k_pack.py; the v^T product with swapped operands relies on this gfx12 layout as in HIP). +// +// Exactness on this driver (clean/dx12/hlsl/wg_math.hlsli, clean/dx12/README.md "Driver exceptions"): +// * FP16 elementwise operations are native half2 operations (v_pk_mul_f16 / v_pk_add_f16 / v_pk_max_num_f16 ..., +// each one IEEE FP16 operation rounded to nearest even, FP16 denormals kept), exactly the packed operations of the +// HIP kernels (k_common.h: contraction off, one rounding per operation). The driver may fuse an FP16 multiply with a +// following add (v_fma_mix) or regroup multiplication chains even under -Gis, so every FP16 result that feeds +// another floating-point operation goes through an opaque zero (wgk_opq: x ^ (wg_grid_ >> 31), 0 at run time and +// not provable by the compiler): the next operation sees an integer result and cannot fuse with or regroup it. +// * FP32 -> FP16 of whole accumulator fragments (wgk_round8, HIP v_cvt_f16_f32 + v_fma_mixhi_f16 x * 1 + -0, round to +// nearest even) is the driver's matrix conversion (op 0x32, RNE; the plain f32tof16 / (half) casts round toward +// zero on this driver). Its input is always a WMMA result or an integer-made value (nothing to fold into it). +// Scalars: wg_f2h (software RNE). +// * Driver channel calls (WMMA, conversion) only in wave-uniform control flow, with branchless neighbours. +#pragma once + +// ---- the opaque zero and half2 views --------------------------------------------------------------------------------- +// wgk_zero() is 0 at run time (grids < 2^31) and unknown to the compiler; wgk_opq xors it into BOTH halves of a packed +// pair (x 0x10001): with the plain 0 / 1 value the driver proved the high half's xor zero and dropped it (then fused +// that half's multiply and add into v_pk_fma_f16). +uint wgk_zero() { return wg_grid_ >> 31; } +uint wgk_opq(uint v) { return v ^ (wgk_zero() * 0x10001u); } +uint4 wgk_opq4(uint4 v) { return v ^ (wgk_zero() * 0x10001u); } +half2 wgk_h2(uint u) { return asfloat16(uint16_t2((uint16_t)(u & 0xffffu), (uint16_t)(u >> 16))); } +uint wgk_u(half2 h) { uint16_t2 v = asuint16(h); return (uint)v.x | ((uint)v.y << 16); } +half2 wgk_c2(uint h) { return wgk_h2((h & 0xffffu) * 0x10001u); } // splatted FP16 constant (bits; no cast rounding) +// FP16 element i (0..7) of a fragment, as bits / as FP32 (exact) +uint wgk_el(uint4 v, uint i) { return (v[i >> 1] >> ((i & 1u) * 16u)) & 0xffffu; } +float wgk_elf(uint4 v, uint i) { return f16tof32(wgk_el(v, i)); } + +// Pin: 0 at run time, a wave-wide reduction of v (WaveActiveBitOr of v & the opaque zero). Added to the base pointer of +// a predicated store, it keeps the matrix products that feed the stored values in wave-uniform control flow: without it +// the driver sank a whole GEMM (its v_wmma included) into the store's exec-masked branch, where lanes outside the map +// were off and the products of the other lanes came out wrong (k_stage_e2's merge at the map border). A reduction +// over the wave cannot be moved into divergent control flow, so everything it reads is computed before the branch. +uint wgk_pin(uint v) { return WaveActiveBitOr(v & wgk_zero()); } + +// ---- 16-byte fragments ----------------------------------------------------------------------------------------------- +// Global: base pointer (arena byte offset) + offset in halfs; 16-byte aligned. LDS: kb_lds (the kernel file declares +// groupshared uint4 kb_lds[]) indexed in halfs (multiples of 8). +uint4 wgk_ld8(uint base, uint off_h) { return wg_ld128(base, off_h * 2u); } +void wgk_st8(uint base, uint off_h, uint4 v) { wg_st128(base, off_h * 2u, v); } +uint4 wgk_zero8() { return uint4(0u, 0u, 0u, 0u); } +v8f wgk_zero8f() { return wg_splat(0.0f); } + +// Value of v in lane (lane ^ 16): the other half of the wave (the other 8 channels of the same token). +float wgk_xor16(float v) { return wg_xor16(v); } +uint wgk_xor16_u(uint v) { return wg_xor16_u(v); } + +// FP32 accumulators -> FP16, round to nearest even (the driver's matrix conversion; HIP wgk_round8). +uint4 wgk_round8(v8f d) { return wg_to_f16_ags(d); } + +// FP16 fragment -> FP32 accumulator (exact), e.g. a bias that a matrix product starts from (HIP wgk_widen8). +v8f wgk_widen8(uint4 a) +{ + v8f r; + [unroll] for (uint i = 0; i < 8u; ++i) r.v[i] = wgk_elf(a, i); + return r; +} + +// D = A x B + C, 16x16x16, FP16 A / B, FP32 accumulation (v_wmma_f32_16x16x16_f16; HIP KAcc::mma). +v8f wgk_mma(uint4 a, uint4 b, v8f c) { return wg_wmma_f16(a, b, c); } + +// ---- exact FP16 elementwise operations (one IEEE FP16 operation each; result through the opaque zero) ---------------- +uint4 wgk_add8(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_opq(wgk_u(wgk_h2(a[i]) + wgk_h2(b[i]))); + return r; +} +uint4 wgk_sub8(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_opq(wgk_u(wgk_h2(a[i]) - wgk_h2(b[i]))); + return r; +} +uint4 wgk_mul8(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_opq(wgk_u(wgk_h2(a[i]) * wgk_h2(b[i]))); + return r; +} +uint4 wgk_splat8h(uint h) // h: FP16 bits +{ + const uint d = (h & 0xffffu) * 0x10001u; + return uint4(d, d, d, d); +} +// min(max(x, lo), hi) and max(x, lo) per element with FP16 constants (v_pk_max_num_f16 / v_pk_min_num_f16: IEEE +// maxNum / minNum, a NaN operand gives the other; HIP wgk_clamp8c / wgk_maxc8 with WGK_ASM_CLAMP) +uint4 wgk_clamp8c(uint4 x, uint lo, uint hi) // lo / hi: FP16 bits +{ + const half2 l = wgk_c2(lo), h = wgk_c2(hi); + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_opq(wgk_u(min(max(wgk_h2(x[i]), l), h))); + return r; +} +uint4 wgk_maxc8(uint4 x, uint lo) // lo: FP16 bits +{ + const half2 l = wgk_c2(lo); + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_opq(wgk_u(max(wgk_h2(x[i]), l))); + return r; +} + +// The same operations WITHOUT the opaque zero, for results whose only consumers cannot fuse with or regroup them: a +// matrix-product operand (WMMA), a store, a min / max. The FP16 operation itself is the same IEEE operation, so the +// bits are the same; the opaque zero only matters where a multiply feeds an add (fma contraction) or another multiply +// (regrouping). Every use is checked in the ISA: no v_pk_fma_f16 / v_fma_f16, every v_fma_mix_f32 is x * 1.0 + y. +uint4 wgk_add8n(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_u(wgk_h2(a[i]) + wgk_h2(b[i])); + return r; +} +uint4 wgk_sub8n(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_u(wgk_h2(a[i]) - wgk_h2(b[i])); + return r; +} +uint4 wgk_mul8n(uint4 a, uint4 b) +{ + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_u(wgk_h2(a[i]) * wgk_h2(b[i])); + return r; +} +uint4 wgk_clamp8cn(uint4 x, uint lo, uint hi) +{ + const half2 l = wgk_c2(lo), h = wgk_c2(hi); + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_u(min(max(wgk_h2(x[i]), l), h)); + return r; +} +uint4 wgk_maxc8n(uint4 x, uint lo) +{ + const half2 l = wgk_c2(lo); + uint4 r; + [unroll] for (uint i = 0; i < 4u; ++i) r[i] = wgk_u(max(wgk_h2(x[i]), l)); + return r; +} + +// ---- the activation of spec 40 (exact; HIP wgk_act2 / wgk_act8) ----------------------------------------------------- +// t = min(max(x, -2), 2), p = f16(a - f16(b |t|)), q = f16(0.5 + f16(t p)), act(x) = f16(x q) +// a = 0.412109375 (0x3698), b = 0.0810546875 (0x2d30); packed FP16, one rounding per operation. +#define WGK_ACT_A 0x3698u // 0.412109375 +#define WGK_ACT_B 0x2d30u // 0.0810546875 +#define WGK_H_M2 0xc000u // -2 +#define WGK_H_2 0x4000u // 2 +#define WGK_H_HALF 0x3800u // 0.5 +#define WGK_H_ONE 0x3c00u // 1 +uint wgk_act2(uint xu) +{ + const half2 x = wgk_h2(xu); + const half2 t = min(max(x, wgk_c2(WGK_H_M2)), wgk_c2(WGK_H_2)); + const half2 at = abs(t); + const half2 bt = wgk_h2(wgk_opq(wgk_u(wgk_c2(WGK_ACT_B) * at))); + const half2 p = wgk_h2(wgk_opq(wgk_u(wgk_c2(WGK_ACT_A) - bt))); + const half2 tp = wgk_h2(wgk_opq(wgk_u(t * p))); + const half2 q = wgk_h2(wgk_opq(wgk_u(wgk_c2(WGK_H_HALF) + tp))); + return wgk_opq(wgk_u(x * q)); +} +uint4 wgk_act8(uint4 x) +{ + return uint4(wgk_act2(x.x), wgk_act2(x.y), wgk_act2(x.z), wgk_act2(x.w)); +} +// act for a matrix-product operand (the MLP's hidden tile): the opaque zero only on the two products that feed an +// addition (b |t| -> a - b |t|, t p -> 0.5 + t p); a - b|t| feeds a multiply, 0.5 + t p the last multiply, whose result +// goes to the WMMA: nothing there to fuse or regroup. +uint wgk_act2n(uint xu) +{ + const half2 x = wgk_h2(xu); + const half2 t = min(max(x, wgk_c2(WGK_H_M2)), wgk_c2(WGK_H_2)); + const half2 at = abs(t); + const half2 bt = wgk_h2(wgk_opq(wgk_u(wgk_c2(WGK_ACT_B) * at))); + const half2 p = wgk_c2(WGK_ACT_A) - bt; + const half2 tp = wgk_h2(wgk_opq(wgk_u(t * p))); + const half2 q = wgk_c2(WGK_H_HALF) + tp; + return wgk_u(x * q); +} +uint4 wgk_act8n(uint4 x) +{ + return uint4(wgk_act2n(x.x), wgk_act2n(x.y), wgk_act2n(x.z), wgk_act2n(x.w)); +} + +// ---- softmax weights (HIP wgk_softmax_weights: FP32 weights, the network's default) --------------------------- +// gg: one query's 4 fragments of 8 keys (lane = query, the clamped polynomial g of spec 41 applied). w = 2^g in FP32 +// (v_exp_f32), the sum over 64 keys = 32 per lane in fragment / element order plus one lane exchange, inv = v_rcp_f32, +// A = f16(w inv) rounded ONCE from the exact product (HIP: v_fma_mixlo_f16 / v_fma_mixhi_f16 w * inv + 0). Here: the +// product's FP32 rounding p and its exact residual e = fma(w, inv, -p) give the round-to-odd FP32 value (13 bits more +// than FP16: rounding it to FP16 is the single rounding of the exact product), then the driver's RNE conversion. +// w >= 2^-14.43 and inv > 0 are finite and normal, so no zero / sign / FTZ case arises. +// Round to odd for a positive normal FP32 product p = RNE(x) with exact residual e = x - p (wg_math.hlsli +// wg_round_odd, specialised: no zero / infinity / sign cases): truncation of x is p, or p - 1 ulp when e < 0; the +// inexact bit e != 0 makes it odd. Gives the same bits as wg_round_odd for every p > 0 finite. +uint wgk_round_odd_pos(float p, float e) +{ + return (asuint(p) - (e < 0.0f ? 1u : 0u)) | (e != 0.0f ? 1u : 0u); +} +#ifndef KB_SMX +#define KB_SMX 1 +#endif +void wgk_softmax_weights(uint4 gg[4], out uint4 pa[4]) +{ + float w[4][8]; + float sum = 0.0f; +#if KB_SMX + // every 2^g first (independent), then the sum in the same element order + [unroll] for (uint st = 0; st < 4u; ++st) + [unroll] for (uint j = 0; j < 8u; ++j) w[st][j] = wg_exp2(wgk_elf(gg[st], j)); + [unroll] for (uint s1 = 0; s1 < 4u; ++s1) + [unroll] for (uint j = 0; j < 8u; ++j) sum = sum + w[s1][j]; +#else + [unroll] for (uint st = 0; st < 4u; ++st) + [unroll] for (uint j = 0; j < 8u; ++j) { + w[st][j] = wg_exp2(wgk_elf(gg[st], j)); + sum = sum + w[st][j]; + } +#endif + sum = sum + wgk_xor16(sum); + const float inv = wg_rcp(sum); + [unroll] for (uint s2 = 0; s2 < 4u; ++s2) { + v8f ro; + [unroll] for (uint j = 0; j < 8u; ++j) { + const float p = w[s2][j] * inv; + const float e = mad(w[s2][j], inv, -p); +#if KB_SMX + ro.v[j] = asfloat(wgk_round_odd_pos(p, e)); +#else + ro.v[j] = asfloat(wg_round_odd(p, e)); +#endif + } + pa[s2] = wgk_round8(ro); + } +} + +// ss + the sum of f16(x_c^2) over the 8 channels of x, in channel order, each addition in FP32 (HIP wgk_sumsq8: packed +// FP16 squares, then v_fma_mix_f32 x2_half * 1 + ss = the FP32 addition of the exact FP16 value). +float wgk_sumsq8(uint4 x, float ss) +{ + const uint4 q = wgk_mul8(x, x); + [unroll] for (uint j = 0; j < 8u; ++j) ss = ss + wgk_elf(q, j); + return ss; +} + +// Mirror about the edge token, edge not repeated (spec 41): -1 -> 1, n -> n - 2; repeated for far positions (HIP +// wgk_mirror; branchless: for 0 <= p < n the general formula gives p, and n = 1 gives 0). +int wgk_mirror(int p, int n) +{ + const int per = max(2 * (n - 1), 1); + int r = p % per; + r = select(r < 0, r + per, r); + return select(n == 1, 0, select(r < n, r, per - r)); +} diff --git a/clean/k/dx12/hlsl/k_norm.hlsli b/clean/k/dx12/hlsl/k_norm.hlsli new file mode 100644 index 00000000..8192cf42 --- /dev/null +++ b/clean/k/dx12/hlsl/k_norm.hlsli @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: MIT +// Model K block normalization reciprocal for the D3D12 ports: the HLSL counterpart of clean/k/kernels/k_norm.h +// (WrongGPU's own code). Needs k_common.hlsli. +// +// Spec 41: r = f16(1 / sqrt(s)) of the FP16 sum of squares s >= 0, as the HIP kernels compute it on gfx1201: +// kh_round(v_rsq_f32((float) s)) (round to nearest even), and r = 0 for s = 0 (s = inf gives rsq = 0 by itself). +// HIP's k_block.hip uses the SELECT form (a select after the rsq instead of a branch): the same bits; so does this one +// (branchless). rsqrt() is v_rsq_f32 on this driver (wg_math.hlsli); the FP32 -> FP16 rounding is wg_f2h (RNE). +#pragma once + +uint wgk_norm_reciprocal(uint s16) // FP16 bits in, FP16 bits out +{ + const uint r = wg_f2h(wg_rsq(f16tof32(s16))); + return select((s16 & 0x7fffu) == 0u, 0u, r); +} diff --git a/clean/k/dx12/k_plan_bridge.py b/clean/k/dx12/k_plan_bridge.py new file mode 100644 index 00000000..b07dc5f5 --- /dev/null +++ b/clean/k/dx12/k_plan_bridge.py @@ -0,0 +1,144 @@ +# SPDX-License-Identifier: MIT +"""Model K port checks: run a launch plan of the stage runner (clean/k/runner/k_stages.py prepare: buffers and launches +as the HIP kernels take them) on HIP in this process, and write the same launches as a kernel_replay chain, so that the +D3D12 ports run on the same input bytes (clean/k/dx12/k_port_check.py compares the raw bytes). + +PlanDev wraps the HIP device for the plan's allocations: every buffer gets SLACK bytes after its size and starts filled +with the HIP sentinel (an uploaded input overwrites its first bytes); the uploads are recorded, so the chain knows which +buffers are inputs. chain_lines turns every launch into a chain step: pointer arguments become @[+] (a +pointer into a buffer, as the weight blob sections are), scalars decimal values, structs by value B:, and structs +with pointers S::=@[+],... where the manifest's argument token (s:) says which +bytes are pointers (clean/runtime/struct_args.h). +""" +import ctypes +import json +import os +import struct + +HIP_SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64 + + +class PlanDev: + """The Hip device as the plan builder (k_stages._Plan) sees it, with slack and sentinels.""" + + def __init__(self, hip): + self.hip = hip + self.inputs = set() + self.sizes = {} + + def malloc(self, nbytes): + p = self.hip.malloc(nbytes + SLACK) + self.hip.memset(p, HIP_SENTINEL, nbytes + SLACK) + self.sizes[p] = nbytes + return p + + def upload(self, ptr, data): + self.inputs.add(ptr) + self.hip.upload(ptr, data) + + def free(self, ptr): + self.hip.free(ptr) + + def __getattr__(self, name): + return getattr(self.hip, name) + + +def manifest_args(build): + """kernel name -> its argument tokens, from /dx12/manifest.json.""" + with open(os.path.join(build, "dx12", "manifest.json")) as f: + return {k["name"]: k["args"].split(",") for k in json.load(f)["kernels"]} + + +def struct_pointer_offsets(token): + """Byte offsets of the pointer fields of an s: token.""" + offs, at = [], 0 + for f in token.split(":", 1)[1].split("."): + kind, n = f[0], int(f[1:]) + if kind == "p": + offs += [at + 8 * i for i in range(n)] + at += 8 * n + else: + at += n + return offs + + +def arg_bytes(arg): + if isinstance(arg, (bytes, bytearray)): + return bytes(arg) + return bytes(arg) # ctypes Structure / array + + +def scalar_word(arg): + """A 32-bit scalar argument (ctypes int / uint / float) as an unsigned decimal.""" + v = arg.value + if isinstance(arg, ctypes.c_float): + return struct.unpack("..d3d12). grid_div: launch with grid // grid_div (the + negative control uses 2). fills: {buffer name: bytes} initial contents that differ from the HIP run's (none).""" + bufs = {n: (p, pdev.sizes[p]) for n, p in plan["bufs"].items()} + lines = [f"chain {prefix}{name}"] + [f"module {m}" for m in modules] + for n, (p, size) in bufs.items(): + if p in pdev.inputs: + lines.append(f"buf b_{n} {size + SLACK} {name}.{n}.in {D3D12_SENTINEL}") + else: + lines.append(f"buf b_{n} {size + SLACK} - {D3D12_SENTINEL}") + for li, (kernel, grid, block, args) in enumerate(plan["launches"]): + lines.append(chain_step(kernel.split(":")[-1], max(1, grid // grid_div), block, args, tokens_of(kernel), bufs, cdir, + f"{prefix}{name}.l{li}")) + for n in outputs: + lines.append(f"save b_{n} {prefix}{name}.{n}.d3d12") + lines.append("") + return lines diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index c7e2b0a9..07568ad7 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -18,7 +18,9 @@ the half-grid control (the port again with half its workgroups) is used by the families whose grid has more than one workgroup; - Model M's scale_rgb (frame.co, its port already proven) runs through the same replay as a control. -The families: expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked +The families: narrow = the narrow network stages (k_block.co: k_stage_e2 .. k_stage2_d0_hidden; the stage runner's +launch plans, clean/k/dx12/k_plan_bridge.py), on random stage weights and inputs with special values (NaN, infinity, ++-65504, zero, -0, subnormal) in every other case; expo = k_expo_scalars against k_io.co (HIP); warm = k_warm (D3D12 only, no HIP kernel: it is checked against its own definition, the dummy word is written exactly when the key equals a lane's loaded value). One known limit, shared with Model M's ports: D3D12 flushes FP32 denormals (inputs and results) to zero, HIP keeps them. k_expo_scalars only meets one with a pre-exposure above about 8.5e37 (kappa = RN(1 / pi) * E), so the inputs here keep @@ -32,6 +34,7 @@ import struct import subprocess import sys +import time import numpy as np @@ -43,7 +46,10 @@ SLACK = 64 CONTROL = "scale_rgb" # Model M, frame.co # family -> (HIP code objects, D3D12 kernels) -FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"])} +NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1", + "k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"] +FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]), + "narrow": (["k_block.co"], NARROW_KERNELS)} CONTROL_CO = "frame.co" @@ -281,11 +287,157 @@ def run_warm(build, rocm, seeds, cdir, hip): return items, lines +# ---- family narrow: the narrow network stages (k_block.co) --------------------------------------------------------- + +# kernel -> (stage, stage-runner suffix, d0 form); the d0 kernels are the network's own forms (hidden history, compact +# pixel head) with the hidden history pointer appended to the stage runner's launch +NARROW = {"k_stage_e2": ("e2", "", None), "k_stage_e0": ("e0", "", None), "k_stage_e1": ("e1", "", None), + "k_stage_d2": ("d2", "", None), "k_stage_d1": ("d1", "", None), "k_stage2_e0": ("e0", "@w2", None), + "k_stage2_e1": ("e1", "@w2", None), "k_stage2_d1": ("d1", "@w2", None), + "k_stage2_d0_hp": ("d0", "@w2", "hp"), "k_stage2_d0_hidden": ("d0", "@w2", "hidden")} +# block-level sizes (W, H) and window offsets (spec 44: even): grids with fewer than a workgroup's windows among them +NARROW_SIZES = [(8, 8), (24, 16), (17, 13), (40, 24), (9, 33), (12, 20)] +NARROW_OFFSETS = [(0, 0), (2, 2), (4, 0), (0, 6), (6, 4)] +# special inputs: level 1 without NaN (infinity, +-65504, zero, -0, the smallest subnormal), level 2 with NaN as well +F16_SPECIALS = [np.array([np.inf, -np.inf, 65504.0, -65504.0, 0.0, -0.0, 6e-8, 1e-4, -3e4], np.float32), + np.array([np.nan, np.inf, -np.inf, 65504.0, -65504.0, 0.0, -0.0, 6e-8, 1e-4, -3e4], np.float32)] + + +def narrow_inputs(stage, size, rng, specials): + from clean.k.reference.layout import STAGE, cdiv + st = STAGE[stage] + W, H = size + if st["kind"] == "dec": + x = rng.standard_normal((cdiv(H, 2), cdiv(W, 2), st["C_prev"])).astype(np.float32) + skip = rng.standard_normal((H, W, st["C"])).astype(np.float32) + else: + x = rng.standard_normal((H, W, 16 if st.get("embed") else st["C"])).astype(np.float32) + skip = None + if specials: + for a in (x, skip): + if a is not None: + pick = rng.random(a.shape) < 0.01 + a[pick] = rng.choice(F16_SPECIALS[specials - 1], int(pick.sum())) + return x, skip + + +def itemsize(name): + return 4 if name == "out" else 2 + + +def run_narrow(build, rocm, seeds, cdir, hip, only=None): + from clean.k.reference.layout import STAGE + from clean.k.reference.network import random_stage_params + from clean.k.runner import k_launch as KL + from clean.k.runner import k_stages + from clean.k.dx12 import k_plan_bridge as PB + tokens = PB.manifest_args(build) + items, lines, neg = [], [], None + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for kernel, (stage, suffix, d0) in NARROW.items(): + if only and kernel not in only: + continue + st = STAGE[stage] + for ci in range(len(NARROW_SIZES)): + size, offset = NARROW_SIZES[ci], NARROW_OFFSETS[(ci + si) % len(NARROW_OFFSETS)] + special = ci % 3 # 0 none, 1 specials without NaN, 2 with NaN + name = f"nar_s{si}_{kernel}_{ci}" + blob = k_stages.pack(stage, random_stage_params(st, rng)) + x, skip = narrow_inputs(stage, size, rng, special) + pdev = PB.PlanDev(hip.h) + plan = k_stages.prepare(pdev, stage + suffix, blob, x, offset, skip) + try: + if d0: # the network's d0 forms: k_block.hip's k_stage2_d0_hidden / _hp take the hidden history + W, H = size + hid = pdev.malloc(W * H * 8) + plan["bufs"]["hidden"], plan["sizes"]["hidden"] = hid, W * H * 8 + k0, grid, block, args = plan["launches"][0] + assert k0 == "k_stage2_d0", k0 + plan["launches"] = [(kernel, grid, block, args + [ctypes.c_void_p(hid)])] + plan["outputs"] = [("hidden", (H, W, 4))] + ( + [("head", (2 * H, 2 * W, 4))] if d0 == "hp" else [("head", (H, W, 40))]) + outputs = [n for n, _ in plan["outputs"]] + for n, p in plan["bufs"].items(): + if p in pdev.inputs: + write(os.path.join(cdir, f"{name}.{n}.in"), bytes(hip.h.download(p, pdev.sizes[p]))) + for k, grid, block, a in plan["launches"]: + co = KL.KERNEL_CODE_OBJECTS[k] + hip.h.launch(hip.h.get_function(hip.mods[co], k), grid, block, a) + hip.h.sync() + sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs} + got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs} + defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]} + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, ["k_block.co"], + outputs) + if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels + neg = name + lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k.split(":")[-1]], cdir, + ["k_block.co"], outputs, prefix="negative_", grid_div=2) + items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size), + "offset": list(offset), "specials": special, "outputs": outputs, "hip": got, + "defined": defined}) + finally: + for p in plan["bufs"].values(): + pdev.free(p) + return items, lines, neg + + +def nan_equivalent_differ(got, want, defined, width=2): + """Differing bytes over the first `defined` bytes after ignoring the elements (2 bytes: FP16) that are NaN in both: + IEEE 754 does not define a NaN's sign or payload, and the two devices may generate or propagate a different one.""" + dt = np.uint16 if width == 2 else np.uint32 + a = np.frombuffer(got[:defined], dt) + b = np.frombuffer(want[:defined], dt) + exp, mant = (0x7C00, 0x03FF) if width == 2 else (0x7F800000, 0x007FFFFF) + nan = lambda v: ((v & exp) == exp) & ((v & mant) != 0) + return int(np.sum((a != b) & ~(nan(a) & nan(b)))), int(np.sum((a != b) & nan(a) & nan(b))) + + +def check_narrow(state, read): + items, _, neg = state + out = [] + for it in items: + per = {n: compare(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n], D3D12_SENTINEL, HIP_SENTINEL) + for n in it["outputs"]} + raw = all(v["identical"] for v in per.values()) + nan_only = 0 + if not raw and it["specials"] == 2: # NaN inputs: a NaN's sign / payload is not defined (see nan_equivalent_differ) + for n, v in per.items(): + if not v["identical"] and v["slack_bad"] == 0: + d, same = nan_equivalent_differ(read(f"{it['case']}.{n}.d3d12"), it["hip"][n], it["defined"][n]) + v["differ_outside_nan"], v["nan_elements_differing_in_sign"] = d, same + nan_only += same + ok_case = raw or all(v["identical"] or (v["slack_bad"] == 0 and v.get("differ_outside_nan", 1) == 0) + for v in per.values()) + out.append({"case": it["case"], "kernel": it["kernel"], "outputs": per, "raw_identical": raw, + "identical": ok_case, "nan_sign_only_elements": nan_only}) + caught, differ = None, 0 + if neg: + it = next(i for i in items if i["case"] == neg) + per = [compare(read(f"negative_{neg}.{n}.d3d12"), it["hip"][n], it["defined"][n], D3D12_SENTINEL, HIP_SENTINEL) + for n in it["outputs"]] + caught = all(p["differ"] >= 0 for p in per) and any(not p["identical"] for p in per) + differ = sum(max(p["differ"], 0) for p in per) + kernels = sorted({o["kernel"] for o in out}) + by_kernel = {k: [sum(o["identical"] for o in out if o["kernel"] == k), sum(1 for o in out if o["kernel"] == k)] + for k in kernels} + ok = bool(out) and all(o["identical"] for o in out) and caught is True + return {"cases": len(out), "identical": sum(o["identical"] for o in out), "by_kernel": by_kernel, + "raw_byte_identical": sum(o["raw_identical"] for o in out), + "cases_differing_only_in_nan_sign": sum(1 for o in out if not o["raw_identical"] and o["identical"]), + "nan_elements_differing_in_sign": sum(o["nan_sign_only_elements"] for o in out), + "special_value_cases": [sum(1 for i in items if i["specials"] == 1), sum(1 for i in items if i["specials"] == 2)], + "negative_control": {"half_grid": True, "case": neg, "caught": caught, "differ": differ}, + "failures": [o for o in out if not o["identical"]][:4]}, ok + + # ---- run ----------------------------------------------------------------------------------------------------------- def run(build, rocm, families, seeds, json_out): - cdir = os.path.join(build, "replay-k") - os.makedirs(cdir, exist_ok=True) + # a new folder per run: an output of an earlier run must never stand in for one this run did not write + cdir = os.path.join(build, "replay-k-" + time.strftime("%Y%m%d-%H%M%S")) + os.makedirs(cdir) res = {"families": families, "seeds": seeds, "results": {}} cos = sorted({co for f in families for co in FAMILY_KERNELS[f][0]} | {CONTROL_CO}) hip = Hip(build, rocm, cos) @@ -297,6 +449,9 @@ def run(build, rocm, families, seeds, json_out): if "warm" in families: state["warm"] = run_warm(build, rocm, seeds, cdir, hip) lines += state["warm"][1] + if "narrow" in families: + state["narrow"] = run_narrow(build, rocm, seeds, cdir, hip) + lines += state["narrow"][1] # the control: Model M's scale_rgb, same replay, against HIP rng = np.random.default_rng(seeds[0]) cn = 1000 @@ -349,6 +504,9 @@ def read(n): fails.append(it["case"]) res["results"]["warm"] = {"cases": len(state["warm"][0]), "as_defined": good, "failures": fails[:5]} ok = ok and good == len(state["warm"][0]) and good > 0 + if "narrow" in state: + res["results"]["narrow"], good = check_narrow(state["narrow"], read) + ok = ok and good ctl_got = read("ctl.out") ctl = compare(ctl_got, h[0], 1000 * 4, D3D12_SENTINEL, HIP_SENTINEL) res["control"] = {"kernel": CONTROL, "identical": bool(ctl["identical"]), "differ": ctl["differ"], diff --git a/clean/k/dx12/kernels/k_block.hlsl b/clean/k/dx12/kernels/k_block.hlsl new file mode 100644 index 00000000..158cdbc4 --- /dev/null +++ b/clean/k/dx12/kernels/k_block.hlsl @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT +// D3D12 ports of model K's narrow network stage kernels (clean/k/kernels/k_block.hip; HIP build: k_block.co). WrongGPU's +// own code. The k_block template (hlsl/k_block.hlsli) with the instantiations of k_block.hip, FP32 accumulation (the +// network's default "fused" plan, runtime_k.cpp KRuntime::network): +// k_stage_e2 k_block<64, 2, softmax, MAP, 64, 96> G = 1, merge 64 -> 96 +// k_stage2_e0 k_block<32, 2, fixed, EMBED, 16, 64, G = 2> embedding 16 -> 32 on load, merge +// k_stage2_e1 k_block<64, 2, fixed, MAP, 64, 64, G = 2> merge +// k_stage_d2 k_block<64, 2, softmax, EXPAND, 96> G = 1, expand + skip +// k_stage2_d1 k_block<64, 2, softmax, EXPAND, 64, G = 2, XG> x through P.y +// k_stage2_d0_hp k_block<32, 2, fixed, EXPAND, 64, HEAD, G = 2, HIDDEN, PIX> (k_head=compact, the default) +// k_stage2_d0_hidden k_block<32, 2, fixed, EXPAND, 64, HEAD, G = 2, HIDDEN> (k_head=full) +// and the one-window (G = 1) forms of e0 / e1 / d1 that HIP's k_block.co also has (K_STAGES: k_stage_e0 / _e1 / _d1, +// the same per-window maths and outputs as the G = 2 forms; d1 without XG: x stays in LDS): +// k_stage_e0 k_block<32, 2, fixed, EMBED, 16, 64> G = 1 +// k_stage_e1 k_block<64, 2, fixed, MAP, 64, 64> G = 1 +// k_stage_d1 k_block<64, 2, softmax, EXPAND, 64> G = 1 +// The FP16-accumulating forms (k_stageh_*) and the FP8 forms are not ported: the exact path only. +// Basis: our HIP kernels (spec 41 block, spec 44 windows, spec 43 6.8 hidden history, spec 40 table); exactness and the +// forms that differ (bit-identical) are listed in hlsl/k_block.hlsli and hlsl/k_common.hlsli. Arguments as HIP: +// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also the hidden history pointer (",p"). +// Launch as HIP: 128 lanes (4 waves of 32) per workgroup, one workgroup per window (G = 1) or per two windows (G = 2). +// +// @wg-kernel k_stage_e2 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage_d2 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=6144 +// @wg-kernel k_stage_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=4096 +// @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32 +// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +// @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1 +#include "wg_platform.hlsli" +#include "k_blkparams.hlsli" + +WG_ARGS_BEGIN + KBLK_PARAMS_DECL +#if KB_HIDDEN_ARG + uint hidden_p; // the hidden history Hh written by the d0 epilogue +#endif +WG_ARGS_END +#include "k_block.hlsli" + +#if KB_HIDDEN_ARG +WG_KERNEL(k_stage2_d0_hp) +{ + WG_SETUP(); + k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id()); +} +WG_KERNEL(k_stage2_d0_hidden) +{ + WG_SETUP(); + k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, false, false>(hidden_p, wg_block_id()); +} +#else +WG_KERNEL(k_stage_e2) +{ + WG_SETUP(); + k_block<64, 2, KMIX_SOFTMAX, KIN_MAP, 64, 96, false, 1, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage2_e0) +{ + WG_SETUP(); + k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage2_e1) +{ + WG_SETUP(); + k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage_d2) +{ + WG_SETUP(); + k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 96, 0, false, 1, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage2_d1) +{ + WG_SETUP(); + k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id()); +} +WG_KERNEL(k_stage_e0) +{ + WG_SETUP(); + k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 1, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage_e1) +{ + WG_SETUP(); + k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 1, false, false, false>(0u, wg_block_id()); +} +WG_KERNEL(k_stage_d1) +{ + WG_SETUP(); + k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 1, false, false, false>(0u, wg_block_id()); +} +#endif diff --git a/clean/k/tests/test_k_plan_bridge.py b/clean/k/tests/test_k_plan_bridge.py new file mode 100644 index 00000000..2873fbb0 --- /dev/null +++ b/clean/k/tests/test_k_plan_bridge.py @@ -0,0 +1,59 @@ +# SPDX-License-Identifier: MIT +"""The Model K port check's launch-plan bridge (clean/k/dx12/k_plan_bridge.py), CPU only: pointer fields of the manifest's +struct argument token, the pointers' buffer and offset, and the kernel_replay chain steps made from a stage launch. + +Run: python -m unittest clean.k.tests.test_k_plan_bridge +""" +import ctypes +import os +import tempfile +import unittest + +from clean.k.dx12 import k_plan_bridge as PB +from clean.k.runner.k_launch import KBlkParams + + +class PlanBridge(unittest.TestCase): + def test_struct_pointer_offsets(self): + self.assertEqual(PB.struct_pointer_offsets("s184:p19.b32"), [8 * i for i in range(19)]) + self.assertEqual(PB.struct_pointer_offsets("s40:p2.b8.x8.p1"), [0, 8, 32]) + self.assertEqual(PB.struct_pointer_offsets("s160:p13.b56"), [8 * i for i in range(13)]) + + def test_locate_and_ref(self): + bufs = {"in": (0x1000, 256), "weights": (0x4000, 1024)} + self.assertEqual(PB.locate(0x1000, bufs), ("in", 0)) + self.assertEqual(PB.locate(0x4000 + 96, bufs), ("weights", 96)) + self.assertEqual(PB.locate(0x1000 + 256 + PB.SLACK - 1, bufs), ("in", 256 + PB.SLACK - 1)) + with self.assertRaises(ValueError): + PB.locate(0x3000, bufs) + self.assertEqual(PB.ref("in", 0), "@b_in") + self.assertEqual(PB.ref("weights", 96), "@b_weights+96") + + def test_chain_step_of_a_block_launch(self): + assert ctypes.sizeof(KBlkParams) == 184 + bufs = {"in": (0x1000, 256), "weights": (0x4000, 1024), "block": (0x9000, 512)} + p = KBlkParams(x=0x1000, y=0x9000, wqkv=0x4000 + 96, w=24, h=16) + with tempfile.TemporaryDirectory() as d: + line = PB.chain_step("k_stage_e2", 12, 128, [p], ["s184:p19.b32"], bufs, d, "case.l0") + self.assertTrue(line.startswith("step k_stage_e2 12 128 S:case.l0.a0:")) + patch = dict(item.split("=") for item in line.split(":", 2)[2].split(",")) + self.assertEqual(patch["0"], "@b_in") # x + self.assertEqual(patch["16"], "@b_block") # y + self.assertEqual(patch["56"], "@b_weights+96") # wqkv + self.assertEqual(len(patch), 3) # null pointers are not patched + with open(os.path.join(d, "case.l0.a0"), "rb") as f: + self.assertEqual(f.read(), bytes(p)) + # scalars, a null pointer, a pointer into a buffer and a struct by value + line = PB.chain_step("k", 1, 32, [ctypes.c_void_p(0), ctypes.c_void_p(0x4000 + 8), ctypes.c_int(-3), + ctypes.c_float(1.0), (ctypes.c_ubyte * 4)(1, 2, 3, 4)], + ["p", "p", "u", "u", "b4"], bufs, d, "case.l1") + self.assertEqual(line, "step k 1 32 null @b_weights+8 4294967293 1065353216 B:case.l1.a4") + + def test_argument_count_mismatch_is_refused(self): + with tempfile.TemporaryDirectory() as d: + with self.assertRaises(ValueError): + PB.chain_step("k", 1, 32, [ctypes.c_int(1)], ["u", "u"], {}, d, "x") + + +if __name__ == "__main__": + unittest.main()