From f1ad5e285a2b564a17b2e45f3cd6cb9f3186cb43 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:17:33 -0500 Subject: [PATCH 1/3] Model L kernels: encoder and decoder blocks (FP16 halo input, token-grid mirror, FP16 output), input linears, both projections, coefficient step; GPU checks Model M's block template gets a gather-source hook (default unchanged; all Model M code objects byte-identical). Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/kernels/swin.hip | 27 +++- clean/l/FORMATS.md | 19 +-- clean/l/kernels/l_ops.hip | 244 ++++++++++++++++++++++++++++++++++ clean/l/kernels/l_swin.hip | 67 ++++++++++ clean/l/runner/l_c32_check.py | 100 +++++++++++++- clean/l/runner/l_ops_check.py | 185 ++++++++++++++++++++++++++ 6 files changed, 620 insertions(+), 22 deletions(-) create mode 100644 clean/l/kernels/l_ops.hip create mode 100644 clean/l/runner/l_ops_check.py diff --git a/clean/kernels/swin.hip b/clean/kernels/swin.hip index d237c407..9cbde7a7 100644 --- a/clean/kernels/swin.hip +++ b/clean/kernels/swin.hip @@ -452,8 +452,16 @@ struct SwinStoreEpi { static constexpr bool kFused = false; }; +// Gather source hook (SRC): the default reads the FP8 map x [H][W][C] at the mirrored member position. A custom SRC +// (kCustom; Model L, clean/l/kernels/l_swin.hip) provides SRC::load(x, pr, pc, H, W, hh, f): the FP32 values +// of the lane's channels hh * C/2 + 8 k + j (k < KC) of the member at grid position (pr, pc), which may lie outside +// the grid. Plain blocks only (no ADD, merge or expand). +struct SwinFp8Src { + static constexpr bool kCustom = false; +}; + template + class EPI = SwinStoreEpi, bool RESIDENT = false, class SRC = SwinFp8Src> WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* __restrict__ out, const u8* __restrict__ w, int H, int W, int ox, int oy, u8* __restrict__ aux WG_PROF_ARG_DECL, const void* epi = nullptr, int nwg = 0) { @@ -464,6 +472,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* // copy. Same arithmetic per window as the staged kernel (bit-identical). constexpr int RES = L::SIZE - L::STREAM; static_assert(!RESIDENT || (MCO == 0 && ECL == 0 && !EPI::kFused && WPW == 1), "resident: plain block kernels only"); + static_assert(!SRC::kCustom || (MCO == 0 && ECL == 0 && !ADD), "custom gather source: plain block kernels only"); WG_LDS __attribute__((aligned(16))) u8 lds[RESIDENT ? (RES + 15) / 16 * 16 : 2 * MAXU]; Stager stage; const u32 tid = wg_lane_id(); @@ -493,6 +502,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* int dst[WPW]; int eslot[WPW]; // fused expand: LDS record of this token v2i xa[WPW][KC], xb[WPW][ADD ? KC : 1]; + v8f xf[WPW][SRC::kCustom ? KC : 1]; if constexpr (ECL > 0) fused_expand(x, w + TAIL, lds + MAXU, H, W, nwx, nwin, ox, oy, tid, (int)wg_block_id()); #pragma unroll for (int i = 0; i < WPW; ++i) { @@ -518,8 +528,12 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* if (ADD) xb[i][k] = xa[i][k]; } #else - if constexpr (ECL == 0) load_frags(x + src, xa[i]); - if (ADD) load_frags(xadd + src, xb[i]); + if constexpr (SRC::kCustom) { + SRC::template load(x, pr, pc, H, W, hh, xf[i]); + } else { + if constexpr (ECL == 0) load_frags(x + src, xa[i]); + if (ADD) load_frags(xadd + src, xb[i]); + } #endif } @@ -559,11 +573,12 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* float ss = 0.0f; #pragma unroll for (int k = 0; k < KC; ++k) { - acc[i][k] = wg_unpack_fp8(xa[i][k]); + if constexpr (SRC::kCustom) acc[i][k] = xf[i][k]; + else acc[i][k] = wg_unpack_fp8(xa[i][k]); if (ADD) acc[i][k] += wg_unpack_fp8(xb[i][k]); - if (!(WG_NORM_DOT4 && !ADD)) ss += hsum8(acc[i][k] * acc[i][k]); + if (!(WG_NORM_DOT4 && !ADD && !SRC::kCustom)) ss += hsum8(acc[i][k] * acc[i][k]); } - if (WG_NORM_DOT4 && !ADD) { + if (WG_NORM_DOT4 && !ADD && !SRC::kCustom) { #pragma unroll for (int k = 0; k < KC; ++k) { ss = wg_dot4_fp8(xa[i][k][0], xa[i][k][0], ss); diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index 939c7039..33018059 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -1,7 +1,7 @@ # Model L: kernels, maps, blobs and launch order (issue #89) This is the interface the Model L runtime frame builds against. -- A row marked **exists** has a merged HIP kernel. +- A row marked **exists** has a HIP kernel that passed its GPU check (merged, or in the open Model L kernel PR). - A row marked **planned** is the interface the kernel is being written to. Its names, arguments, grids and maps are fixed here, and a change goes through this file first. - Each D3D12 port declares the same code object, kernel name and arguments as its HIP kernel (`clean/dx12/kernels/l_*.hlsl`). @@ -75,13 +75,13 @@ workgroup, in raster order of their map. | Kernel | Status | Arguments | Grid | Does | |---|---|---|---|---| | `lev/input_encoder` | planned (rework of the old values-only variant) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | -| `l_linear_32_32_relu` | planned | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | -| `l_linear_32_32_f16` | planned | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | -| `l_swin_block_c32_enc[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | -| `l_project_gather` | planned | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | -| `l_dec_project` | planned | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | -| `l_swin_block_c32_dec[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | -| `l_coef` | planned (new) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` | +| `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | +| `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | +| `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | +| `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | +| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | +| `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | +| `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` | ## Launch order of one frame @@ -105,6 +105,7 @@ The reconstruction (post) stage is still open. ## Checks (GPU, synthetic data) -- `clean/l/runner/l_c32_check.py`: the C = 32 block against the oracle. +- `clean/l/runner/l_c32_check.py`: the C = 32 blocks (`l_swin_block_c32`, `_enc`, `_dec`) against the oracle. +- `clean/l/runner/l_ops_check.py`: the linears, both projections and `l_coef` against the oracle steps. - `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. - One check per new kernel follows with that kernel. diff --git a/clean/l/kernels/l_ops.hip b/clean/l/kernels/l_ops.hip new file mode 100644 index 00000000..f5cee150 --- /dev/null +++ b/clean/l/kernels/l_ops.hip @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the per-pixel and per-token matrix kernels of Model L's encoder and decoder +// (clean/l/FORMATS.md: maps, blobs, arguments, grids). Maps are [rows][cols][C], channel fastest; FP8 E4M3 bytes or +// FP16. Launch: 1-D grid of ceil(n / 128) workgroups of 256 lanes (8 waves); wave i handles items 16 i .. 16 i + 15 +// (pixels or tokens in raster order); lane (t, h) works on item 16 i + t, and its accumulator element v of output tile +// mt is output channel h C/2 + 8 mt + v (layout L, clean/FORMATS.md). +#include "wg_platform.h" +#include "wg_mem.h" + +// Single-precision steps are rounded one operation at a time (no contraction), as in clean/kernels/decoder.hip. +#pragma clang fp contract(off) + +// Frame constants of the coefficient step: Model M's WgDecParams (clean/kernels/decoder.hip); l_coef reads rw, rh, fw. +struct LDecParams { + int rw, rh, fw, fh, reset, pad0; + float dw, dh, kx, ky, jx, jy, pjx, pjy; +}; +static_assert(sizeof(LDecParams) == 56, "LDecParams must match WgDecParams"); + +namespace { + +constexpr int kThreads = 256; +constexpr int kItemsPerWave = 16; + +WG_FN int item_of(u32 tid) { + return (int)(wg_block_id() * (kThreads / WG_WAVE) + (tid >> 5)) * kItemsPerWave + (int)(tid & 15u); +} + +WG_FN v8f rn16_8(v8f a) { +#pragma unroll + for (int i = 0; i < 8; ++i) a[i] = wg_rn16(a[i]); + return a; +} + +// Linear 32 -> 32 on a pixel map: acc = b + W x (bias as the accumulator start, one WMMA K block of 32 inputs), then +// RN16. RELU: y = q8(max(0, acc)) as FP8; otherwise y = acc as FP16. Blob: bias tile vector (rows R_L(32)), then +// fragments [mt < 2][kc < 2], rows R_L(32), K order K_L(32). +template +WG_FN void linear_32_32(const u8* __restrict__ x, u8* __restrict__ y, const u8* __restrict__ w, int npix) { + constexpr int C = 32, KC = 2, MT = 2; + const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int pix = item_of(tid); + const int pk = pix < npix ? pix : npix - 1; + v2i xf[KC]; +#pragma unroll + for (int kc = 0; kc < KC; ++kc) xf[kc] = *(const v2i*)(x + (size_t)pk * C + hh * (C / 2) + 8 * kc); + const float* bias = (const float*)w + hh * 8; + const u8* frag = w + 4 * C; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + v8f acc = wg_load8f(bias + mt * 16); +#pragma unroll + for (int kc = 0; kc < KC; ++kc) acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KC + kc) * 256))[lane], xf[kc], acc); + acc = rn16_8(acc); + if (pix >= npix) continue; + if constexpr (RELU) { +#pragma unroll + for (int i = 0; i < 8; ++i) acc[i] = wg_fmax(acc[i], 0.0f); + *(v2i*)(y + (size_t)pix * C + hh * (C / 2) + 8 * mt) = wg_pack_q8(acc); + } else { + *(v8h*)((_Float16*)y + (size_t)pix * C + hh * (C / 2) + 8 * mt) = wg_to_f16(acc); + } + } +} + +WG_FN float bits(u32 b) { return __builtin_bit_cast(float, b); } + +// sigma(x) = 1 / (1 + 2^(x * -log2 e)) with the hardware exponential and reciprocal. +WG_FN float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); } + +// tanh(t): for |t| >= 0.6, 1 - 2 / (2^(2 |t| log2 e) + 1) (1 from about 9.01 on), sign of t; below, an odd polynomial. +WG_FN float l_tanh(float t) { + const float a = __builtin_fabsf(t); + if (a >= bits(0x3F19999Au)) { + float r = __builtin_fmaf(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f); + if (a >= bits(0x41102CB4u)) r = 1.0f; + return __builtin_copysignf(r, t); + } + const float t2 = t * t; + const float p = __builtin_fmaf( + __builtin_fmaf(__builtin_fmaf(__builtin_fmaf(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), + t2, bits(0xBEAAA9EDu)), + t2, 0.0f); + return __builtin_fmaf(p, t, t); +} + +WG_FN u32 unorm8_code(float v) { return (u32)wg_rint(v * 255.0f); } // v in [0, 1] +WG_FN u32 snorm8_code(float v) { return (u32)(int)wg_rint(wg_fmin(wg_fmax(v, -1.0f), 1.0f) * 127.0f) & 0xffu; } + +struct LCodes { + u32 A, B, C0, C1, L; +}; + +// Model L's per-pixel functions on the 13 values a (clean/l/reference/table.py L_DEC_VALUES order). +WG_FN LCodes l_pixel(const float a[13], float S, float T) { + LCodes o; + o.A = unorm8_code(wg_rn16(l_sigma(a[0]))) | (unorm8_code(wg_rn16(l_sigma(a[1]))) << 8) | + (unorm8_code(wg_rn16(l_sigma(a[2]))) << 16); + const float m = wg_fmax(wg_fmax(a[3], a[4]), a[5]); + const float e0 = wg_exp2((a[3] - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a[4] - m) * bits(0x3FB8AA3Bu)), + e2 = wg_exp2((a[5] - m) * bits(0x3FB8AA3Bu)); + const float s = (e0 + e1) + e2; + const float r = wg_rcp(s); + o.B = unorm8_code(e0 * r) | (unorm8_code(e1 * r) << 8) | (unorm8_code(e2 * r) << 16); + float g[2]; +#pragma unroll + for (int i = 0; i < 2; ++i) { + const float v = a[6 + i]; + const float c = v >= 20.0f ? 20.0f : (v <= -20.0f ? -20.0f : v); + g[i] = wg_rn16(l_sigma(wg_rn16(c))); + } + const float sx = S * g[0], sy = S * g[1]; + const float sxy = sx * sy; + const float rho = l_tanh(a[8] * T); + o.C0 = wg_pack_h2(sx * sx, sy * sy); + o.C1 = wg_f2h_bits(sxy * rho); + o.L = 0; +#pragma unroll + for (int i = 0; i < 4; ++i) o.L |= snorm8_code(l_tanh(a[9 + i])) << (8 * i); + return o; +} + +// The 13 values of pixel (x, y) of the FP16 pixel map y2 [fh][fw][32]. +constexpr int kLValues[13] = {0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5}; + +WG_FN void l_values(const _Float16* __restrict__ y2, int fw, int x, int y, float a[13]) { + const _Float16* p = y2 + (size_t)(y * fw + x) * 32; + const v8h lo = *(const v8h*)p, hi = *(const v8h*)(p + 8); +#pragma unroll + for (int i = 0; i < 13; ++i) a[i] = (float)(kLValues[i] < 8 ? lo[kLValues[i]] : hi[kLValues[i] - 8]); +} + +// 8 FP8 bytes of input block kb for lane half hh in the K order of clean/l/runner/l_pack.py kord_groups32: block kb, +// slot (hh, j) = channel 32 (kb / 2) + 16 hh + 8 (kb % 2) + j. +WG_FN int group_channel(int kb, u32 hh) { return 32 * (kb >> 1) + 16 * (int)hh + 8 * (kb & 1); } + +} // namespace + +// x [npix][32] FP8 -> y [npix][32] FP8 = q8(max(0, RN16(b + W x))). w = blob enc0.Win1. +WG_KERNEL_N(l_linear_32_32_relu, kThreads)(const u8* x, u8* y, const u8* w, int npix) { + linear_32_32(x, y, w, npix); +} + +// x [npix][32] FP8 -> y [npix][32] FP16 = RN16(b + W x). w = blob enc0.Win2. +WG_KERNEL_N(l_linear_32_32_f16, kThreads)(const u8* x, u8* y, const u8* w, int npix) { + linear_32_32(x, y, w, npix); +} + +// Encoder output projection. x: pixel map [2 th][2 tw][32] FP8; token (r, c) reads pixel (2 r + p / 2, 2 c + p % 2) as +// channels 32 p + i. features [th][tw][64] = q8(clamp(s, +-6.28125)) with s = b, then s = RN16(s + W[:, g] z[g]) per +// group g = p of 32 channels. w = blob enc0.project (bias tile vector rows R_L(64), fragments [mt < 4][kb < 8]). +WG_KERNEL_N(l_project_gather, kThreads)(const u8* x, u8* feat, const u8* w, int tw, int th) { + constexpr int CO = 64, MT = 4, KB = 8; + const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int ntok = tw * th; + const int tok = item_of(tid); + const int tk = tok < ntok ? tok : ntok - 1; + const int r = tk / tw, c = tk - r * tw; + v2i zf[KB]; +#pragma unroll + for (int kb = 0; kb < KB; ++kb) { + const int p = kb >> 1; + const int pix = (2 * r + (p >> 1)) * (2 * tw) + 2 * c + (p & 1); + zf[kb] = *(const v2i*)(x + (size_t)pix * 32 + (group_channel(kb, hh) - 32 * p)); + } + const float* bias = (const float*)w + hh * 8; + const u8* frag = w + 4 * CO; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + v8f acc = wg_load8f(bias + mt * 16); +#pragma unroll + for (int g = 0; g < KB / 2; ++g) { + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g) * 256))[lane], zf[2 * g], acc); + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g + 1) * 256))[lane], zf[2 * g + 1], acc); + acc = rn16_8(acc); + } +#pragma unroll + for (int i = 0; i < 8; ++i) acc[i] = wg_fmin(wg_fmax(acc[i], -6.28125f), 6.28125f); + const v2i q = wg_pack_q8(acc); + if (tok < ntok) *(v2i*)(feat + (size_t)tok * CO + hh * (CO / 2) + 8 * mt) = q; + } +} + +// Decoder projection. z [th][tw][64] FP8 (dec1's output) -> y: pixel map [2 th][2 tw][32] FP8. Per token: 128 outputs +// n = 32 s + k, s = RN16(b + W[:, 0:32] z[0:32]), then RN16(s + W[:, 32:64] z[32:64]), then FP8; output n goes to pixel +// (2 r + s / 2, 2 c + s % 2), channel k. w = blob dec0.project (bias tile vector rows R_E(32), fragments [mt < 8][kb < +// 4]; tile mt = 2 s + g holds sub-pixel s, channels 16 hh + 8 g + v). +WG_KERNEL_N(l_dec_project, kThreads)(const u8* z, u8* y, const u8* w, int tw, int th) { + constexpr int MT = 8, KB = 4; + const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int ntok = tw * th; + const int tok = item_of(tid); + const int tk = tok < ntok ? tok : ntok - 1; + const int r = tk / tw, c = tk - r * tw; + v2i zf[KB]; +#pragma unroll + for (int kb = 0; kb < KB; ++kb) zf[kb] = *(const v2i*)(z + (size_t)tk * 64 + group_channel(kb, hh)); + const float* bias = (const float*)w + hh * 8; + const u8* frag = w + 4 * 128; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + v8f acc = wg_load8f(bias + mt * 16); +#pragma unroll + for (int g = 0; g < KB / 2; ++g) { + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g) * 256))[lane], zf[2 * g], acc); + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g + 1) * 256))[lane], zf[2 * g + 1], acc); + acc = rn16_8(acc); + } + const v2i q = wg_pack_q8(acc); + const int s = mt >> 1, g = mt & 1; + const int pix = (2 * r + (s >> 1)) * (2 * tw) + 2 * c + (s & 1); + if (tok < ntok) *(v2i*)(y + (size_t)pix * 32 + 16 * hh + 8 * g) = q; + } +} + +// Coefficient step. y2 [fh][fw][32] FP16 (the decoder block's output) -> A, B (RGBA8 unorm codes), C (4 x FP16: C0, C1, +// C2, 0) and the new latent (4 x s8 snorm), [rh][rw] each, pixels inside R only; Model M's coef_decoder output formats. +// Grid as coef_decoder: ceil((fh / 2)(fw / 2) / 128); lane (t, h) of wave i handles token t' = 16 i + t = (r, c), pixels +// (2 c, 2 r + h) and (2 c + 1, 2 r + h). w = blob dec0.coef (Model M's dec0 blob: S, T at byte 320). No motion, +// previous latent, jitter or reset is read. +WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) { + const u32 tid = wg_lane_id(), hh = (tid >> 4) & 1u; + const int tw = P.fw >> 1, ntok = (P.fh >> 1) * tw; + const int tok = item_of(tid); + if (tok >= ntok) return; + const int r = tok / tw, c = tok - r * tw; + const int y = 2 * r + (int)hh; + if (y >= P.rh) return; + const float S = ((const float*)w)[80], T = ((const float*)w)[81]; +#pragma unroll + for (int dx = 0; dx < 2; ++dx) { + const int x = 2 * c + dx; + if (x >= P.rw) continue; + float a[13]; + l_values((const _Float16*)y2, P.fw, x, y, a); + const LCodes o = l_pixel(a, S, T); + const u32 i = (u32)(y * P.rw + x); + A[i] = o.A; + B[i] = o.B; + C[2 * i] = o.C0; + C[2 * i + 1] = o.C1; + latent[i] = o.L; + } +} diff --git a/clean/l/kernels/l_swin.hip b/clean/l/kernels/l_swin.hip index ce2f73ce..baf2c2c2 100644 --- a/clean/l/kernels/l_swin.hip +++ b/clean/l/kernels/l_swin.hip @@ -23,6 +23,73 @@ WG_KERNEL_NW(l_swin_block_c32_nc, 256, WG_MINW_C32)(const u8* x, const u8* xadd, swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, true, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); } +// ---- Model L's two blocks (clean/l/FORMATS.md) --------------------------------------------------------------------- + +// Encoder block input: the FP16 halo map [H + 4][W + 4][C] (grid position (pc, pr) at map pixel (pr + 2, pc + 2)); +// window members outside the grid read the halo directly. Needs H, W multiples of 4 and offsets of 0 or 2. +struct LHaloF16Src { + static constexpr bool kCustom = true; + static constexpr int kHalo = 2; + template + static WG_FN void load(const u8* x, int pr, int pc, int H, int W, u32 hh, v8f* f) { + (void)H; + const _Float16* p = (const _Float16*)x + + ((size_t)((pr + kHalo) * (W + 2 * kHalo) + pc + kHalo) * C + hh * (C / 2)); +#pragma unroll + for (int k = 0; k < KC; ++k) { + const v8h h = *(const v8h*)(p + 8 * k); +#pragma unroll + for (int j = 0; j < 8; ++j) f[k][j] = (float)h[j]; + } + } +}; + +// Decoder block input: the FP8 pixel map [H][W][C] of 2 x 2 sub-pixel groups; a member outside the grid takes the +// mirrored 2 x 2 group (mirror of the group index, edge group not repeated) and keeps its sub-pixel. +struct LCoarseFp8Src { + static constexpr bool kCustom = true; + template + static WG_FN void load(const u8* x, int pr, int pc, int H, int W, u32 hh, v8f* f) { + const int r = 2 * wg_mirror(pr >> 1, H >> 1) + (pr & 1); + const int c = 2 * wg_mirror(pc >> 1, W >> 1) + (pc & 1); + v2i q[KC]; + load_frags(x + (size_t)(r * W + c) * C + hh * (C / 2), q); +#pragma unroll + for (int k = 0; k < KC; ++k) f[k] = wg_unpack_fp8(q[k]); + } +}; + +// Decoder block output: RN16(x2) as an FP16 map [H][W][C] (epi = the output pointer), real members only. +struct LStoreF16Epi { + static constexpr bool kFused = true; + template + static WG_FN void run(const v8f (&acc)[WPW][KC], const int (&dst)[WPW], int W, u8* lds, int lds_bytes, u32 tid, + const void* epi) { + (void)W, (void)lds, (void)lds_bytes; + const u32 hh = (tid & 31u) >> 4; +#pragma unroll + for (int i = 0; i < WPW; ++i) { + if (dst[i] < 0) continue; + _Float16* p = (_Float16*)epi + (size_t)dst[i] * C + hh * (C / 2); +#pragma unroll + for (int k = 0; k < KC; ++k) *(v8h*)(p + 8 * k) = wg_to_f16(acc[i][k]); + } + } +}; + +// x, xadd (unused), out, w, H, W, ox, oy as above. Encoder: x = FP16 halo map, out = FP8 map. Decoder: x = FP8 map, +// out = FP16 map. +#define L_BLOCK_KERNEL(name, NC, EPI, SRC, OUTARG) \ + WG_KERNEL_NW(name, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, int ox, \ + int oy) { \ + swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, NC, 256, 0, 0, EPI, false, SRC>(x, xadd, OUTARG, w, H, W, ox, oy, \ + nullptr, out); \ + } +L_BLOCK_KERNEL(l_swin_block_c32_enc, false, SwinStoreEpi, LHaloF16Src, out) +L_BLOCK_KERNEL(l_swin_block_c32_enc_nc, true, SwinStoreEpi, LHaloF16Src, out) +L_BLOCK_KERNEL(l_swin_block_c32_dec, false, LStoreF16Epi, LCoarseFp8Src, nullptr) +L_BLOCK_KERNEL(l_swin_block_c32_dec_nc, true, LStoreF16Epi, LCoarseFp8Src, nullptr) + // The same template at Model M's C = 64 (as swin_block_c64[_nc] of swin.co): the GPU check runs it next to Model M's // own kernel to show that this file builds the template exactly as swin.co does. WG_KERNEL_NW(l_swin_block_c64, 256, WG_MINW_C64)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 41af2eb6..07b7d27a 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -4,7 +4,8 @@ python clean/l/runner/l_c32_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--compile] [--with-swin-co] --json out.json ---compile builds /l_swin.co (and with --with-swin-co Model M's swin.co from clean/kernels/swin.hip) on the CPU. +--compile builds /l_swin.co and l_ops.co (and with --with-swin-co Model M's swin.co from clean/kernels/swin.hip) +on the CPU. The GPU part runs in a GPU slot. Output: counts only, no timing. Checks: @@ -14,6 +15,9 @@ differ at all. 2. l_swin_block_c32_nc against l_swin_block_c32 where the weights' flags allow it: byte-identical. 3. With swin.co: l_swin_block_c64 (this file's build of the template) against Model M's swin_block_c64: byte-identical. +4. Model L's two blocks (clean/l/FORMATS.md) against the oracle with their borders: l_swin_block_c32_enc (FP16 halo + map in, FP8 out) and l_swin_block_c32_dec (FP8 in, FP16 out, token-grid mirror; the rule of 1. on the FP8-rounded + values, plus FP16 differences reported); their `_nc` forms byte-identical to the clamped ones. """ import argparse import ctypes @@ -43,8 +47,9 @@ def compile_kernels(build, rocm, with_swin=False, clang=None): clean_process_environment(rocm) clang = clang or B.find_clang(rocm) kdir = os.path.join(ROOT, "clean", "kernels") - B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_swin.hip"), - os.path.join(build, "l_swin.co")) + ["-I", kdir]) + for name in ("l_swin", "l_ops"): + B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", name + ".hip"), + os.path.join(build, name + ".co")) + ["-I", kdir]) if with_swin: B.run(B.device_command(clang, rocm, os.path.join(kdir, "swin.hip"), os.path.join(build, "swin.co"))) @@ -83,6 +88,24 @@ def __init__(self, build, rocm): with open(path, "rb") as f: self.mods[co] = self.hip.load_module(f.read()) + def raw_block(self, co, kernel, xb, out_bytes, blob, H, W, offset): + """Launch a block kernel on input bytes xb; returns the output buffer's bytes.""" + h = self.hip + px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(out_bytes) + h.upload(px, xb) + h.upload(pw, blob) + h.memset(po, 0, out_bytes) + ox, oy = offset + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + args = [ctypes.c_void_p(px), ctypes.c_void_p(0), ctypes.c_void_p(po), ctypes.c_void_p(pw), + ctypes.c_int(H), ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)] + h.launch(h.get_function(self.mods[co], kernel), -(-nwin // 8), 256, args) + h.sync() + out = h.download(po, out_bytes) + for p in (px, pw, po): + h.free(p) + return out + def block(self, co, kernel, x, blob, offset, C): """x: FP8-exact [H, W, C] -> kernel output [H, W, C] (float32 values).""" h = self.hip @@ -113,6 +136,65 @@ def make_case(rng, H, W): return p, LP.pack_block(p, 32, 2, 128), q8(rng.normal(0, 1.5, (H, W, 32))) +L_CASES = [(8, 12), (16, 8), (32, 64), (48, 32)] # Model L's blocks: grid sides multiples of 4 + + +def f16_steps(a, b): + """Distance in FP16 units of last place (finite values).""" + def key(v): + i = np.asarray(v, np.float16).view(np.int16).astype(np.int64) + return np.where(i < 0, -(i & 0x7FFF), i) + return np.abs(key(a) - key(b)) + + +def l_block_cases(g, rngs): + """Model L's encoder block (FP16 halo map in, FP8 out, offsets "first") and decoder block (FP8 in, FP16 out, + token-grid mirror, offsets "second") against the oracle; the `_nc` forms against the clamped ones.""" + res = {"enc": [], "dec": [], "nc_identical": []} + for rng in rngs: + for H, W in L_CASES: + for parity in (0, 1): + p, blob, _ = make_case(rng, H, W) + safe = bool(np.frombuffer(blob[:4], np.uint32)[0] & 1) + # encoder block + off = T.block_offset(T.TABLE, "first", parity) + h = T.block_halo(T.TABLE, "first") + x = L.f16(rng.normal(0, 1.5, (H + 2 * h, W + 2 * h, 32))) + xb = x.astype(np.float16).tobytes() + raw = g.raw_block("l_swin.co", "l_swin_block_c32_enc", xb, H * W * 32, blob, H, W, off) + got = fp8_decode(np.frombuffer(raw, np.uint8).reshape(H, W, 32)) + ref = q8(L.swin_block(x, p, off, 2, CONSTS, "input", h)) + m = metrics(got, ref) + m.update({"case": [H, W, list(off)], "pass": passes(m)}) + res["enc"].append(m) + if safe: + res["nc_identical"].append( + g.raw_block("l_swin.co", "l_swin_block_c32_enc_nc", xb, H * W * 32, blob, H, W, off) == raw) + # decoder block + off = T.block_offset(T.TABLE, "second", parity) + x = q8(rng.normal(0, 1.5, (H, W, 32))) + xb = fp8_encode(x).tobytes() + raw = g.raw_block("l_swin.co", "l_swin_block_c32_dec", xb, H * W * 64, blob, H, W, off) + got = np.frombuffer(raw, np.float16).reshape(H, W, 32).astype(np.float64) + ref = L.f16(L.swin_block(x, p, off, 2, CONSTS, "coarse")) + m = metrics(q8(got), q8(ref)) # Model M's per-stage rule on the FP8-rounded values + s = f16_steps(got, ref) + m.update({"case": [H, W, list(off)], "f16_differ": int(np.sum(s > 0)), "f16_max_steps": int(s.max())}) + m["pass"] = passes(m) and bool(np.isfinite(got).all()) + res["dec"].append(m) + if safe: + res["nc_identical"].append( + g.raw_block("l_swin.co", "l_swin_block_c32_dec_nc", xb, H * W * 64, blob, H, W, off) == raw) + res["nc_cases"] = len(res["nc_identical"]) + res["nc_identical"] = all(res["nc_identical"]) if res["nc_identical"] else None + for k in ("enc", "dec"): + res[k + "_total"] = {"cases": len(res[k]), "values": sum(m["values"] for m in res[k]), + "differ": sum(m["differ"] for m in res[k]), + "max_steps": max(m["max_steps"] for m in res[k])} + res["pass"] = all(m["pass"] for m in res["enc"] + res["dec"]) and res["nc_identical"] in (True, None) + return res + + def run(build, rocm, seeds=(89,)): g = Gpu(build, rocm) rngs = [np.random.default_rng(s) for s in seeds] @@ -145,8 +227,9 @@ def run(build, rocm, seeds=(89,)): res["c32_total"] = {"cases": len(res["c32"]), "values": sum(m["values"] for m in res["c32"]), "differ": sum(m["differ"] for m in res["c32"]), "max_steps": max(m["max_steps"] for m in res["c32"])} + res["l_blocks"] = l_block_cases(g, [np.random.default_rng(1000 + s) for s in seeds]) res["pass"] = (all(m["pass"] for m in res["c32"]) and res["c32_nc_identical"] in (True, None) - and res["c64_template_identical"] in (True, None)) + and res["c64_template_identical"] in (True, None) and res["l_blocks"]["pass"]) return res @@ -169,9 +252,12 @@ def main(argv=None): res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) with open(a.json, "w") as f: json.dump(res, f, indent=1) - for m in res["c32"]: - print(json.dumps(m)) - print({k: v for k, v in res.items() if k != "c32"}) + print({k: v for k, v in res.items() if k not in ("c32", "l_blocks")}) + lb = res["l_blocks"] + print({k: lb[k] for k in ("enc_total", "dec_total", "nc_cases", "nc_identical", "pass")}) + for m in lb["enc"] + lb["dec"]: + if not m["pass"]: + print("FAIL", json.dumps(m)) return 0 if res["pass"] else 1 diff --git a/clean/l/runner/l_ops_check.py b/clean/l/runner/l_ops_check.py new file mode 100644 index 00000000..65401806 --- /dev/null +++ b/clean/l/runner/l_ops_check.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the matrix kernels of clean/l/kernels/l_ops.hip against the oracle steps +(clean/l/reference/oracle.py), synthetic data only. + + python clean/l/runner/l_ops_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--seeds N] + --json out.json + +/l_ops.co comes from clean/l/runner/l_c32_check.py --compile-only. The GPU part runs in a GPU slot. Output: +counts only, no timing. + +Per kernel and case: the values that differ from the oracle step, and Model M's per-stage rule (clean/l/runner/ +l_c32_check.py passes) on the FP8 values (on the FP8-rounded values for the FP16 output, whose FP16 differences are +reported too): +- l_linear_32_32_relu: q8(max(0, RN16(b + Win1 q))) on the value map (20 values, 12 zero channels); +- l_linear_32_32_f16: RN16(b + Win2 x), FP16; +- l_project_gather: the 2 x 2 gather and the 128 -> 64 projection (running FP16 sum per group of 32, clamp, FP8); +- l_dec_project: the 64 -> 4 x 32 projection scattered to the pixel map; +- l_coef: Model L's per-pixel functions on an FP16 map against oracle.coefficients_l: stored codes of A, B and the + latent within one step, C within 4 FP16 ulp (the oracle's exponential and tanh are exact; the kernel's use the + hardware operations), with the counts that differ at all. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 + +CASES = [(16, 24), (32, 64), (8, 8), (64, 96)] # pixel maps (rows, cols), even sides + + +class Ops: + def __init__(self, build, rocm): + from clean.runner.hip import Hip + self.hip = Hip(rocm) + with open(os.path.join(build, "l_ops.co"), "rb") as f: + self.mod = self.hip.load_module(f.read()) + + def run(self, kernel, xb, out_bytes, blob, ints, n): + h = self.hip + px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(out_bytes) + h.upload(px, xb) + h.upload(pw, blob) + h.memset(po, 0, out_bytes) + args = [ctypes.c_void_p(px), ctypes.c_void_p(po), ctypes.c_void_p(pw)] + [ctypes.c_int(v) for v in ints] + h.launch(h.get_function(self.mod, kernel), -(-n // 128), 256, args) + h.sync() + out = h.download(po, out_bytes) + for p in (px, pw, po): + h.free(p) + return out + + +COEF_CASES = [(32, 64, 30, 51), (16, 32, 16, 32), (64, 96, 61, 77)] # (fh, fw, rh, rw) + + +def coef_case(g, rng, dec, fh, fw, rh, rw): + """l_coef on a random FP16 map y2 [fh][fw][32] against oracle.coefficients_l: stored codes that differ.""" + from clean.runner.dec0 import DecParams + y2 = L.f16(rng.normal(0, 2.0, (fh, fw, 32))) + h = g.hip + yb = y2.astype(np.float16).tobytes() + blob = LP.pack_coef(dec) + npx = rh * rw + py, pw = h.malloc(len(yb)), h.malloc(len(blob)) + outs = [h.malloc(npx * n) for n in (4, 4, 8, 4)] + h.upload(py, yb) + h.upload(pw, blob) + for p, n in zip(outs, (4, 4, 8, 4)): + h.memset(p, 0, npx * n) + P = DecParams(rw=rw, rh=rh, fw=fw, fh=fh) + args = [ctypes.c_void_p(py)] + [ctypes.c_void_p(p) for p in outs] + [ctypes.c_void_p(pw), P] + h.launch(h.get_function(g.mod, "l_coef"), -(-((fh // 2) * (fw // 2)) // 128), 256, args) + h.sync() + A, B, C, Lt = (h.download(p, npx * n) for p, n in zip(outs, (4, 4, 8, 4))) + for p in [py, pw] + outs: + h.free(p) + a = y2[:rh, :rw][..., list(L._table.L_DEC_VALUES)].astype(np.float32) + ref = L.coefficients_l(a, dec["S"], dec["T"]) + got = {"codes_A": np.frombuffer(A, np.uint8).reshape(rh, rw, 4)[..., :3], + "codes_B": np.frombuffer(B, np.uint8).reshape(rh, rw, 4)[..., :3], + "C": np.frombuffer(C, np.float16).reshape(rh, rw, 4)[..., :3], + "codes_L": np.frombuffer(Lt, np.int8).reshape(rh, rw, 4)} + m = {"case": [fh, fw, rh, rw]} + for k in ("codes_A", "codes_B", "codes_L"): + d = np.abs(got[k].astype(int) - np.asarray(ref[k]).astype(int)) + m[k] = {"values": int(d.size), "differ": int(np.sum(d > 0)), "max": int(d.max())} + s = C32.f16_steps(got["C"], np.asarray(ref["C"]).astype(np.float16)) + m["C"] = {"values": int(s.size), "differ": int(np.sum(s > 0)), "max_f16_steps": int(s.max())} + # Codes within one step of the oracle and FP16 C within a few ulp (the oracle's exp / tanh are exact in float64). + m["pass"] = all(m[k]["max"] <= 1 for k in ("codes_A", "codes_B", "codes_L")) and m["C"]["max_f16_steps"] <= 4 + return m + + +def _fp8_metrics(got, ref, case): + m = C32.metrics(got, ref) + m.update({"case": case, "pass": C32.passes(m)}) + return m + + +def run(build, rocm, seeds=(89,)): + g = Ops(build, rocm) + res = {k: [] for k in ("relu", "f16", "project", "dec_project")} + for seed in seeds: + rng = np.random.default_rng(seed) + params = L.random_params(rng) + enc, dec = params["enc0"], params["dec0"] + b_w1 = LP.pack_linear(enc["Win1"], enc["bin1"], 32, 32) + b_w2 = LP.pack_linear(enc["Win2"], enc["bin2"], 32, 32) + b_pj = LP.pack_project(enc["W"], enc["b"], (32, 32, 32, 32)) + b_dp = LP.pack_dec_project(dec["W"], dec["b"]) + for H, W in CASES: + npix = H * W + q = np.zeros((H, W, 32)) + q[..., :20] = q8(rng.normal(0, 2.0, (H, W, 20))) + raw = g.run("l_linear_32_32_relu", fp8_encode(q).tobytes(), npix * 32, b_w1, [npix], npix) + got = fp8_decode(np.frombuffer(raw, np.uint8).reshape(H, W, 32)) + ref = L.linear(q[..., :20], enc["Win1"], enc["bin1"], "f16", "relu_q8") + res["relu"].append(_fp8_metrics(got, ref, [H, W])) + + x1 = q8(np.abs(rng.normal(0, 1.5, (H, W, 32)))) + raw = g.run("l_linear_32_32_f16", fp8_encode(x1).tobytes(), npix * 64, b_w2, [npix], npix) + got = np.frombuffer(raw, np.float16).reshape(H, W, 32).astype(np.float64) + ref = L.linear(x1, enc["Win2"], enc["bin2"], "f16", "none") + m = _fp8_metrics(q8(got), q8(ref), [H, W]) + s = C32.f16_steps(got, ref) + m.update({"f16_differ": int(np.sum(s > 0)), "f16_max_steps": int(s.max())}) + res["f16"].append(m) + + x3 = q8(rng.normal(0, 1.5, (H, W, 32))) + tw, th = W // 2, H // 2 + raw = g.run("l_project_gather", fp8_encode(x3).tobytes(), tw * th * 64, b_pj, [tw, th], tw * th) + got = fp8_decode(np.frombuffer(raw, np.uint8).reshape(th, tw, 64)) + ref = L.project_groups(L.space_to_depth(x3), enc["W"], enc["b"], (32, 32, 32, 32), 6.28125) + res["project"].append(_fp8_metrics(got, ref, [H, W])) + + z = q8(rng.normal(0, 1.5, (th, tw, 64))) + raw = g.run("l_dec_project", fp8_encode(z).tobytes(), npix * 32, b_dp, [tw, th], tw * th) + got = fp8_decode(np.frombuffer(raw, np.uint8).reshape(H, W, 32)) + ref = L.depth_to_space(L.dec_project(z, dec["W"], dec["b"])) + res["dec_project"].append(_fp8_metrics(got, ref, [H, W])) + for case in COEF_CASES: + res.setdefault("coef", []).append(coef_case(g, rng, dec, *case)) + res["coef_total"] = {k: {"values": sum(m[k]["values"] for m in res["coef"]), + "differ": sum(m[k]["differ"] for m in res["coef"])} + for k in ("codes_A", "codes_B", "C", "codes_L")} + for k in ("relu", "f16", "project", "dec_project"): + res[k + "_total"] = {"cases": len(res[k]), "values": sum(m["values"] for m in res[k]), + "differ": sum(m["differ"] for m in res[k]), + "max_steps": max(m["max_steps"] for m in res[k])} + res["f16_total"]["f16_differ"] = sum(m["f16_differ"] for m in res["f16"]) + res["pass"] = all(m["pass"] for k in ("relu", "f16", "project", "dec_project", "coef") for m in res[k]) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"}) + for k in ("relu", "f16", "project", "dec_project", "coef"): + for m in res[k]: + if not m["pass"]: + print("FAIL", k, json.dumps(m)) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) From feea19e6262dcdad566e3b7445f80b9d98fd5628 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:28:44 -0500 Subject: [PATCH 2/3] Model L: halo form of the input encoder's value kernel (encoder.hip WG_ENC_VALUES_OUT, off by default), its GPU check; map-copy test Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/kernels/encoder.hip | 41 +++++++++-- clean/l/FORMATS.md | 3 +- clean/l/runner/l_c32_check.py | 4 +- clean/l/runner/l_enc_check.py | 130 ++++++++++++++++++++++++++++++++++ clean/l/tests/test_l_map.py | 66 +++++++++++++++++ 5 files changed, 238 insertions(+), 6 deletions(-) create mode 100644 clean/l/runner/l_enc_check.py create mode 100644 clean/l/tests/test_l_map.py diff --git a/clean/kernels/encoder.hip b/clean/kernels/encoder.hip index c1327f1a..3254cbfa 100644 --- a/clean/kernels/encoder.hip +++ b/clean/kernels/encoder.hip @@ -83,6 +83,15 @@ #ifndef WG_ENC_AB_STORE #define WG_ENC_AB_STORE 0 // no motion-surface stores #endif +// Model L (clean/l/FORMATS.md, `lev/input_encoder`): no projection. The kernel runs on the padded grid plus a halo of +// WG_ENC_HALO pixels per side and stores each pixel's 20 dithered FP8 values into the halo map `features` +// [fh + 2 halo][fw + 2 halo][32] (bytes 20..31 zero). The values come from the mirrored render position, and the dither +// from the unmirrored one (index |y| mod 16, |x| mod 16). Motion surfaces are written only for pixels inside R. Built +// as its own code object; off by default: this file's kernel is unchanged. +#ifndef WG_ENC_VALUES_OUT +#define WG_ENC_VALUES_OUT 0 +#endif +#define WG_ENC_HALO (WG_ENC_VALUES_OUT ? 2 : 0) // Per-frame constants, passed by value (all 4-byte fields; must match EncParams in clean/runner/enc0.py). struct WgEncParams { @@ -390,12 +399,16 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri const u8* __restrict__ w, const WgEncParams& P, const MoT mo = MoT()) { const u32 tid = wg_lane_id(); const u32 lane = tid & 31u, t = lane & 15u, hh = lane >> 4; +#if WG_ENC_VALUES_OUT + const int Wt = (P.fw + 2 * WG_ENC_HALO) >> 1, ntok = ((P.fh + 2 * WG_ENC_HALO) >> 1) * Wt; +#else const int Wt = P.fw >> 1, ntok = (P.fh >> 1) * Wt; +#endif const int tok = (int)(wg_block_id() * (kEncThreads / WG_WAVE) + (tid >> 5)) * kTokensPerWave + (int)t; const int tk = tok < ntok ? tok : ntok - 1; const int r = tk / Wt, cc = tk - r * Wt; -#if !WG_ENC_FRAG_GLOBAL +#if !WG_ENC_FRAG_GLOBAL && !WG_ENC_VALUES_OUT // Stage the projection weights in LDS; the copy overlaps the per-pixel work, the barrier comes after it. WG_LDS __attribute__((aligned(16))) u8 lds[kFragBytes]; for (int i = (int)tid; i < kFragBytes / 16; i += kEncThreads) @@ -422,7 +435,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri } const int rw = P.rw, rh = P.rh; - const int X0 = 2 * cc, Y = 2 * r + (int)hh; + const int X0 = 2 * cc - WG_ENC_HALO, Y = 2 * r + (int)hh - WG_ENC_HALO; const int y = reflect(Y, rh); const int x0 = reflect(X0, rw), x1 = reflect(X0 + 1, rw); const bool rev = x0 > x1; // right padding: slot 0 is pixel B @@ -465,8 +478,16 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri #endif const float mcs[2][2] = {{rev ? mcab[2] : mcab[0], rev ? mcab[3] : mcab[1]}, {rev ? mcab[0] : mcab[2], rev ? mcab[1] : mcab[3]}}; +#if WG_ENC_VALUES_OUT + // Dither word per pixel at the unmirrored position: |y| mod 16, |x| mod 16 (X0 may be negative here). + const u8* trow = w + kOffDither + 256u * (u32)P.frame + (((u32)(Y < 0 ? -Y : Y) & 15u) << 4); + const u32 t2 = (u32)trow[(u32)(X0 < 0 ? -X0 : X0) & 15u] | + ((u32)trow[(u32)(X0 + 1 < 0 ? -(X0 + 1) : X0 + 1) & 15u] << 8); + const bool store = tok < ntok && Y >= 0 && Y < rh && X0 >= 0; +#else const u32 t2 = wg_ld16(w, (u32)kOffDither + 256u * (u32)P.frame + (((u32)Y & 15u) << 4) + ((u32)X0 & 15u)); const bool store = tok < ntok && Y < rh; +#endif const u32 so = (u32)(Y * rw + X0) * 4u; // Motions of both slots, and the motion surfaces of the real pixels: one 64-bit store per surface when both pixels @@ -524,7 +545,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri else pixel_values(f, P, U, (int)xs[dx], y, c, px); quantize(px.v, dx ? t2 >> 8 : t2 & 0xffu, q[dx]); -#if !WG_ENC_AB_PROJ +#if !WG_ENC_AB_PROJ && !WG_ENC_VALUES_OUT // Values 8..15 go to the partner lane (the other lane half of the same token). x8[dx][0] = wg_xor16_u(q[dx][2]); x8[dx][1] = wg_xor16_u(q[dx][3]); @@ -534,7 +555,19 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri #endif } -#if WG_ENC_AB_PROJ +#if WG_ENC_VALUES_OUT + // Pixel (X0 + dx, Y) of the halo map: 20 value bytes and 12 zero bytes. + if (tok < ntok) { + const u32 mw = (u32)(P.fw + 2 * WG_ENC_HALO); +#pragma unroll + for (int dx = 0; dx < 2; ++dx) { + const u32 o = ((u32)(Y + WG_ENC_HALO) * mw + (u32)(X0 + dx + WG_ENC_HALO)) * 32u; + wg_st128(features, o, (v4u){q[dx][0], q[dx][1], q[dx][2], q[dx][3]}); + wg_st128(features, o + 16u, (v4u){q[dx][4], 0u, 0u, 0u}); + } + } + (void)x8; +#elif WG_ENC_AB_PROJ if (tok < ntok) { v4u o0 = {q[0][0], q[0][1], q[0][2], q[0][3]}, o1 = {q[1][0], q[1][1], q[1][2], q[1][3] ^ q[0][4] ^ q[1][4]}; wg_st128(features, (u32)tok * kFeat + hh * 32u, o0); diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index 33018059..140d02d3 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -74,7 +74,7 @@ workgroup, in raster order of their map. | Kernel | Status | Arguments | Grid | Does | |---|---|---|---|---| -| `lev/input_encoder` | planned (rework of the old values-only variant) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | +| `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` | | `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` | | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | @@ -107,5 +107,6 @@ The reconstruction (post) stage is still open. - `clean/l/runner/l_c32_check.py`: the C = 32 blocks (`l_swin_block_c32`, `_enc`, `_dec`) against the oracle. - `clean/l/runner/l_ops_check.py`: the linears, both projections and `l_coef` against the oracle steps. +- `clean/l/runner/l_enc_check.py`: `lev/input_encoder` (halo map and motion surfaces) against the oracle. - `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte. - One check per new kernel follows with that kernel. diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 07b7d27a..3747fb97 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -4,7 +4,7 @@ python clean/l/runner/l_c32_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--compile] [--with-swin-co] --json out.json ---compile builds /l_swin.co and l_ops.co (and with --with-swin-co Model M's swin.co from clean/kernels/swin.hip) +--compile builds /l_swin.co, l_ops.co and l_enc_values.co (clean/kernels/encoder.hip with WG_ENC_VALUES_OUT=1; and with --with-swin-co Model M's swin.co from clean/kernels/swin.hip) on the CPU. The GPU part runs in a GPU slot. Output: counts only, no timing. @@ -50,6 +50,8 @@ def compile_kernels(build, rocm, with_swin=False, clang=None): for name in ("l_swin", "l_ops"): B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", name + ".hip"), os.path.join(build, name + ".co")) + ["-I", kdir]) + B.run(B.device_command(clang, rocm, os.path.join(kdir, "encoder.hip"), os.path.join(build, "l_enc_values.co"), + defines={"WG_ENC_VALUES_OUT": 1})) if with_swin: B.run(B.device_command(clang, rocm, os.path.join(kdir, "swin.hip"), os.path.join(build, "swin.co"))) diff --git a/clean/l/runner/l_enc_check.py b/clean/l/runner/l_enc_check.py new file mode 100644 index 00000000..a38826fa --- /dev/null +++ b/clean/l/runner/l_enc_check.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the input encoder's value kernel (`lev/input_encoder`: clean/kernels/encoder.hip +built with WG_ENC_VALUES_OUT=1 as l_enc_values.co) against the oracle, synthetic frames only. + + python clean/l/runner/l_enc_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--seeds N] + --json out.json + +/l_enc_values.co comes from clean/l/runner/l_c32_check.py --compile-only. The GPU part runs in a GPU slot. +Output: counts only, no timing. + +Per frame: the halo map q [fh + 4][fw + 4][32] against the oracle's dithered values (clean/l/reference/oracle.py +encode, map "q"): values that differ, the largest difference in FP8 steps, and Model M's per-stage rule +(clean/l/runner/l_c32_check.py passes). Bytes 20..31 of every pixel must be zero. The motion surfaces m_near, m_far, +m_center (inside R) are compared with the oracle's outputs, which are Model M's, under Model M's rule for them (at most 1 +FP16 ulp, at least 99.5 % exact; clean/tests/test_encoder_emu.py). +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_c32_check as C32 # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.reference import encoder as ME # noqa: E402 +from clean.reference.fp8 import fp8_decode # noqa: E402 +from clean.runner import enc0 as E0 # noqa: E402 + +# (rw, rh): render sizes with padding to multiples of 32, odd sizes and both reset states +CASES = [(40, 24, False), (37, 19, False), (64, 32, True), (90, 50, False)] + + +def run_frame(hip, mod, x, enc): + h = hip + prm = E0.frame_params(x) + data = E0.device_inputs(x) + halo = T.block_halo(T.TABLE, "first") + mw, mh = prm.fw + 2 * halo, prm.fh + 2 * halo + ntok = (mw // 2) * (mh // 2) + bufs = {} + for k, v in data.items(): + b = v if isinstance(v, bytes) else np.ascontiguousarray(v).tobytes() + bufs[k] = h.malloc(len(b)) + h.upload(bufs[k], b) + blob = LP.pack_values(enc) + pw = h.malloc(len(blob)) + h.upload(pw, blob) + eb = np.array([x["exposure"], x["exposure_prev"]], dtype=np.float32).tobytes() + pe = h.malloc(len(eb)) + h.upload(pe, eb) + pq = h.malloc(mw * mh * 32) + h.memset(pq, 0xEE, mw * mh * 32) # unwritten bytes would show + npx = prm.rw * prm.rh * 4 + pm = [h.malloc(npx) for _ in range(3)] + for p in pm: + h.memset(p, 0, npx) + args = [ctypes.c_void_p(bufs[k]) for k in ("color", "depth", "motion", "hist", "latent", "mhist")] + \ + [ctypes.c_void_p(pq)] + [ctypes.c_void_p(p) for p in pm] + [ctypes.c_void_p(pe), ctypes.c_void_p(pw), prm] + h.launch(h.get_function(mod, "input_encoder"), -(-ntok // 128), 256, args) + h.sync() + q = np.frombuffer(h.download(pq, mw * mh * 32), np.uint8).reshape(mh, mw, 32) + mot = [np.frombuffer(h.download(p, npx), np.float16).reshape(prm.rh, prm.rw, 2) for p in pm] + for p in list(bufs.values()) + [pw, pe, pq] + pm: + h.free(p) + return q, mot + + +def run(build, rocm, seeds=(89,)): + from clean.runner.hip import Hip + hip = Hip(rocm) + with open(os.path.join(build, "l_enc_values.co"), "rb") as f: + mod = hip.load_module(f.read()) + res = {"frames": []} + for seed in seeds: + rng = np.random.default_rng(seed) + params = L.random_params(rng) + for rw, rh, reset in CASES: + x = ME.random_inputs(rng, rw=rw, rh=rh, reset=reset) + q, mot = run_frame(hip, mod, x, params["enc0"]) + out = L.encode(x, params, parity=0, keep=True) + ref = out["maps"]["q"] + got = fp8_decode(q[..., :20]) + m = C32.metrics(got, ref) + m.update({"case": [rw, rh, reset], "pad_bytes_nonzero": int(np.count_nonzero(q[..., 20:]))}) + mdiff, mok = {}, True + for name, g in zip(("m_near", "m_far", "m_center"), mot): + r = np.asarray(out[name], np.float32) + fin = np.isfinite(r) + u = C32.f16_steps(g[fin], r[fin]) + same_nonfinite = bool(np.array_equal(g[~fin].astype(np.float32), r[~fin], equal_nan=True)) + mdiff[name] = {"differ": int(np.sum(u > 0)), "max_f16_steps": int(u.max()) if u.size else 0} + # Model M's rule for these surfaces (clean/tests/test_encoder_emu.py): at most 1 FP16 ulp, 99.5 % exact + mok &= same_nonfinite and mdiff[name]["max_f16_steps"] <= 1 and float(np.mean(u == 0)) >= 0.995 + m["motion"] = mdiff + m["pass"] = C32.passes(m) and m["pad_bytes_nonzero"] == 0 and mok + res["frames"].append(m) + res["total"] = {"frames": len(res["frames"]), "values": sum(m["values"] for m in res["frames"]), + "differ": sum(m["differ"] for m in res["frames"]), + "max_steps": max(m["max_steps"] for m in res["frames"])} + res["pass"] = all(m["pass"] for m in res["frames"]) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({"total": res["total"], "pass": res["pass"]}) + for m in res["frames"]: + if not m["pass"]: + print("FAIL", json.dumps(m)) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/tests/test_l_map.py b/clean/l/tests/test_l_map.py new file mode 100644 index 00000000..6589a8fe --- /dev/null +++ b/clean/l/tests/test_l_map.py @@ -0,0 +1,66 @@ +# SPDX-License-Identifier: MIT +"""Model L's copies of the converter map's facts agree with the map (data/preset-L.v*.map.json, spec-team output). + +Run: python -m unittest discover -s clean/l/tests -t . -p test_l_map.py + +The map is the source of truth. The table (clean/l/reference/table.py) and the coefficient kernel +(clean/l/kernels/l_ops.hip) hold copies of its value positions, and this test fails when they drift apart. The first +map (v1) still carried Model M's value positions, so the check needs v1.1 or later. +""" +import json +import os +import re +import unittest + +from clean.l.reference import table as T + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) +DATA = os.path.join(ROOT, "data") + + +def newest_l_map(): + """Path of the newest preset-L map at version 1.1 or later, else None.""" + best = None + for f in os.listdir(DATA): + m = re.fullmatch(r"preset-L\.v(\d+)(?:\.(\d+))?\.map\.json", f) + if m: + v = (int(m.group(1)), int(m.group(2) or 0)) + if v >= (1, 1) and (best is None or v > best[0]): + best = (v, os.path.join(DATA, f)) + return best[1] if best else None + + +def kernel_values(): + with open(os.path.join(ROOT, "clean", "l", "kernels", "l_ops.hip"), encoding="utf-8") as f: + m = re.search(r"kLValues\[13\]\s*=\s*\{([^}]*)\}", f.read()) + return tuple(int(v) for v in m.group(1).split(",")) + + +class MapCopiesTest(unittest.TestCase): + def test_kernel_values_are_the_table(self): + self.assertEqual(kernel_values(), T.L_DEC_VALUES) + + def test_table_values_are_the_map(self): + path = newest_l_map() + if path is None: + self.skipTest("no preset-L map v1.1 or later in data/") + with open(path, encoding="utf-8") as f: + m = json.load(f) + self.assertEqual(tuple(m["perms"]["dec0_values"]), T.L_DEC_VALUES) + self.assertEqual(m.get("preset", "L"), "L") + + def test_table_shapes_are_the_map(self): + path = newest_l_map() or os.path.join(DATA, "preset-L.v1.map.json") + with open(path, encoding="utf-8") as f: + m = json.load(f) + shapes = T.param_shapes() + for st in m["stages"]: + if st["stage"] not in shapes: + continue + got = {r["name"]: tuple(r["shape"]) for r in st["tensors"]} + want = {k: (v if v else (1,)) for k, v in shapes[st["stage"]].items()} # scalars: () here, [1] there + self.assertEqual(got, want, st["stage"]) + + +if __name__ == "__main__": + unittest.main() From b08a2ac8d0e0ac71e35b322ebcac8b7aa0064f6e Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:20:58 -0500 Subject: [PATCH 3/3] Model L review fixes: halo encoder slots that mirror onto one render column share its depth and motion (width-17 GPU case); FP16 criterion, rounding-boundary inputs and coefficient codes from the real block output in the GPU checks; Model M code-object comparison script Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/kernels/encoder.hip | 12 +++- clean/l/runner/l_c32_check.py | 66 +++++++++++++++++--- clean/l/runner/l_enc_check.py | 10 ++- clean/l/runner/l_ops_check.py | 77 ++++++++++++++++++++--- clean/runner/m_codeobjects_compare.py | 90 +++++++++++++++++++++++++++ 5 files changed, 235 insertions(+), 20 deletions(-) create mode 100644 clean/runner/m_codeobjects_compare.py diff --git a/clean/kernels/encoder.hip b/clean/kernels/encoder.hip index 3254cbfa..ef55102b 100644 --- a/clean/kernels/encoder.hip +++ b/clean/kernels/encoder.hip @@ -469,7 +469,15 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri // Per slot (sub-pixel dx of this lane): render column, near / far pixel indices, centre motion, colour, dither. const u32 xs[2] = {(u32)x0, (u32)x1}; - const u32 ins[2] = {rev ? inB : inA, rev ? inA : inB}, ifs[2] = {rev ? ifB : ifA, rev ? ifA : ifB}; +#if WG_ENC_VALUES_OUT + // Halo positions far right of a small render size can mirror (and clamp) onto one render column: x0 == x1 (then 0). + // Both slots are that pixel: slot 1 takes slot 0's depth neighbourhood and centre motion (it keeps its own dither). + const bool same = x0 == x1; +#else + constexpr bool same = false; +#endif + const u32 ins[2] = {rev ? inB : inA, same ? inA : (rev ? inA : inB)}; + const u32 ifs[2] = {rev ? ifB : ifA, same ? ifA : (rev ? ifA : ifB)}; #if WG_ENC_MOTION_F16 const v2u mh = wg_ld64(f.motion, (rc + (u32)xa) * 4u); const v4f mcab = {wg_h2f_lo(mh[0]), wg_h2f_hi(mh[0]), wg_h2f_lo(mh[1]), wg_h2f_hi(mh[1])}; @@ -477,7 +485,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri const v4f mcab = __builtin_bit_cast(v4f, wg_ld128(f.motion, (rc + (u32)xa) * 8u)); #endif const float mcs[2][2] = {{rev ? mcab[2] : mcab[0], rev ? mcab[3] : mcab[1]}, - {rev ? mcab[0] : mcab[2], rev ? mcab[1] : mcab[3]}}; + {same ? mcab[0] : (rev ? mcab[0] : mcab[2]), same ? mcab[1] : (rev ? mcab[1] : mcab[3])}}; #if WG_ENC_VALUES_OUT // Dither word per pixel at the unmirrored position: |y| mod 16, |x| mod 16 (X0 may be negative here). const u8* trow = w + kOffDither + 256u * (u32)P.frame + (((u32)(Y < 0 ? -Y : Y) & 15u) << 4); diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 3747fb97..118dd0da 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -16,8 +16,11 @@ 2. l_swin_block_c32_nc against l_swin_block_c32 where the weights' flags allow it: byte-identical. 3. With swin.co: l_swin_block_c64 (this file's build of the template) against Model M's swin_block_c64: byte-identical. 4. Model L's two blocks (clean/l/FORMATS.md) against the oracle with their borders: l_swin_block_c32_enc (FP16 halo - map in, FP8 out) and l_swin_block_c32_dec (FP8 in, FP16 out, token-grid mirror; the rule of 1. on the FP8-rounded - values, plus FP16 differences reported); their `_nc` forms byte-identical to the clamped ones. + map in, FP8 out) and l_swin_block_c32_dec (FP8 in, FP16 out, token-grid mirror); their `_nc` forms byte-identical + to the clamped ones. The decoder block's FP16 output must pass the rule of 1. on its FP8-rounded values and the + FP16 criterion (f16_passes). With /l_ops.co, the coefficient codes are computed from the real block output: + l_coef against the oracle's functions on that same output (the strict bound of l_ops_check.py), and end to end + against the oracle block + functions (at least 99.5 % of each code kind and of the C values identical). """ import argparse import ctypes @@ -149,7 +152,39 @@ def key(v): return np.abs(key(a) - key(b)) -def l_block_cases(g, rngs): +def f16_metrics(got, ref): + """FP16-output counterpart of metrics(): distances in FP16 ulps (finite reference values).""" + got = np.asarray(got, np.float64) + ref = np.asarray(ref, np.float64) + fin = np.isfinite(ref) + s = f16_steps(got[fin], ref[fin]) + d = np.abs(got[fin] - ref[fin]) + return {"f16_values": int(s.size), "f16_differ": int(np.sum(s > 0)), "f16_max_steps": int(s.max()) if s.size else 0, + "f16_frac_over_1": float(np.mean(s > 1)) if s.size else 0.0, + "f16_frac_exact": float(np.mean(s == 0)) if s.size else 1.0, + "f16_nonfinite": int(np.sum(~np.isfinite(got[fin]))) + int(np.sum(got[~fin] != ref[~fin])), + "f16_mean_abs_err": float(d.mean()) if d.size else 0.0, + "f16_ref_rms": float(np.sqrt(np.mean(ref[fin] ** 2))) if d.size else 0.0} + + +def f16_passes(m, max_steps=None): + """FP16 criterion: Model M's per-stage rule with FP16 steps in place of FP8 steps (at most 2 % of values more than + one FP16 ulp off, mean absolute error at most 1 % of the reference RMS), at least 99 % of values exact, no + non-finite value where the reference is finite, and, where given, a bound on the largest distance (1 ulp for an + output that is one rounding of an accumulator, as the second linear).""" + return (m["f16_nonfinite"] == 0 and m["f16_frac_over_1"] <= 0.02 and m["f16_frac_exact"] >= 0.99 + and m["f16_mean_abs_err"] <= 0.01 * m["f16_ref_rms"] + and (max_steps is None or m["f16_max_steps"] <= max_steps)) + + +class _OpsMod: + def __init__(self, hip, build): + self.hip = hip + with open(os.path.join(build, "l_ops.co"), "rb") as f: + self.mod = hip.load_module(f.read()) + + +def l_block_cases(g, rngs, ops=None): """Model L's encoder block (FP16 halo map in, FP8 out, offsets "first") and decoder block (FP8 in, FP16 out, token-grid mirror, offsets "second") against the oracle; the `_nc` forms against the clamped ones.""" res = {"enc": [], "dec": [], "nc_identical": []} @@ -180,9 +215,25 @@ def l_block_cases(g, rngs): got = np.frombuffer(raw, np.float16).reshape(H, W, 32).astype(np.float64) ref = L.f16(L.swin_block(x, p, off, 2, CONSTS, "coarse")) m = metrics(q8(got), q8(ref)) # Model M's per-stage rule on the FP8-rounded values - s = f16_steps(got, ref) - m.update({"case": [H, W, list(off)], "f16_differ": int(np.sum(s > 0)), "f16_max_steps": int(s.max())}) - m["pass"] = passes(m) and bool(np.isfinite(got).all()) + m.update(f16_metrics(got, ref)) # ... and the FP16 criterion on the FP16 values + m["case"] = [H, W, list(off)] + m["pass"] = passes(m) and f16_passes(m) + if ops is not None: # coefficient codes from the real block output + from clean.l.runner import l_ops_check as OC + dec = {"S": np.float32(rng.uniform(1.0, 4.0)), "T": np.float32(rng.uniform(0.2, 1.5))} + # (1) l_coef on the GPU block output against the oracle's functions on that same output: the + # strict bound of l_ops_check (codes within 1, C within 4 FP16 ulp). + m["coef_same_input"] = OC.coef_case(ops, rng, dec, H, W, H, W, y2=got, ref_y2=got) + # (2) end to end: codes from the GPU block + l_coef against codes from the oracle block + oracle + # functions. The block's FP16 differences (bounded by f16_passes) pass through steep functions + # (the latent code is 127 tanh), so a code may move by more than 1 where the block differs; the + # bound: at least 99.5 % of each code kind and of the C values identical (as Model M's motion + # surfaces, clean/tests/test_encoder_emu.py). + e2e = OC.coef_case(ops, rng, dec, H, W, H, W, y2=got, ref_y2=ref) + e2e["pass"] = all(e2e[k]["differ"] <= 0.005 * e2e[k]["values"] + for k in ("codes_A", "codes_B", "codes_L", "C")) + m["coef_end_to_end"] = e2e + m["pass"] = m["pass"] and m["coef_same_input"]["pass"] and e2e["pass"] res["dec"].append(m) if safe: res["nc_identical"].append( @@ -229,7 +280,8 @@ def run(build, rocm, seeds=(89,)): res["c32_total"] = {"cases": len(res["c32"]), "values": sum(m["values"] for m in res["c32"]), "differ": sum(m["differ"] for m in res["c32"]), "max_steps": max(m["max_steps"] for m in res["c32"])} - res["l_blocks"] = l_block_cases(g, [np.random.default_rng(1000 + s) for s in seeds]) + ops = _OpsMod(g.hip, build) if os.path.exists(os.path.join(build, "l_ops.co")) else None + res["l_blocks"] = l_block_cases(g, [np.random.default_rng(1000 + s) for s in seeds], ops) res["pass"] = (all(m["pass"] for m in res["c32"]) and res["c32_nc_identical"] in (True, None) and res["c64_template_identical"] in (True, None) and res["l_blocks"]["pass"]) return res diff --git a/clean/l/runner/l_enc_check.py b/clean/l/runner/l_enc_check.py index a38826fa..476f7ae6 100644 --- a/clean/l/runner/l_enc_check.py +++ b/clean/l/runner/l_enc_check.py @@ -35,7 +35,8 @@ from clean.runner import enc0 as E0 # noqa: E402 # (rw, rh): render sizes with padding to multiples of 32, odd sizes and both reset states -CASES = [(40, 24, False), (37, 19, False), (64, 32, True), (90, 50, False)] +CASES = [(40, 24, False), (37, 19, False), (64, 32, True), (90, 50, False), + (17, 17, False), (17, 24, True)] # width 17 (padded 32): halo columns 32, 33 both mirror to column 0 def run_frame(hip, mod, x, enc): @@ -100,12 +101,15 @@ def run(build, rocm, seeds=(89,)): # Model M's rule for these surfaces (clean/tests/test_encoder_emu.py): at most 1 FP16 ulp, 99.5 % exact mok &= same_nonfinite and mdiff[name]["max_f16_steps"] <= 1 and float(np.mean(u == 0)) >= 0.995 m["motion"] = mdiff - m["pass"] = C32.passes(m) and m["pad_bytes_nonzero"] == 0 and mok + # Model M's encoder criteria (clean/tests/test_encoder_emu.py): per frame the per-stage rule and at least + # 99 % of values exact; over all frames at most 0.01 % of values differ (checked below). + m["pass"] = (C32.passes(m) and m["frac_differ"] <= 0.01 and m["pad_bytes_nonzero"] == 0 and mok) res["frames"].append(m) res["total"] = {"frames": len(res["frames"]), "values": sum(m["values"] for m in res["frames"]), "differ": sum(m["differ"] for m in res["frames"]), "max_steps": max(m["max_steps"] for m in res["frames"])} - res["pass"] = all(m["pass"] for m in res["frames"]) + res["total"]["frac_differ"] = res["total"]["differ"] / res["total"]["values"] + res["pass"] = all(m["pass"] for m in res["frames"]) and res["total"]["frac_differ"] <= 1e-4 return res diff --git a/clean/l/runner/l_ops_check.py b/clean/l/runner/l_ops_check.py index 65401806..7b0d6b27 100644 --- a/clean/l/runner/l_ops_check.py +++ b/clean/l/runner/l_ops_check.py @@ -64,10 +64,13 @@ def run(self, kernel, xb, out_bytes, blob, ints, n): COEF_CASES = [(32, 64, 30, 51), (16, 32, 16, 32), (64, 96, 61, 77)] # (fh, fw, rh, rw) -def coef_case(g, rng, dec, fh, fw, rh, rw): - """l_coef on a random FP16 map y2 [fh][fw][32] against oracle.coefficients_l: stored codes that differ.""" +def coef_case(g, rng, dec, fh, fw, rh, rw, y2=None, ref_y2=None): + """l_coef on an FP16 map y2 [fh][fw][32] (default: random) against oracle.coefficients_l of ref_y2 (default: y2): + stored codes that differ. g: an object with .hip and .mod (l_ops.co).""" from clean.runner.dec0 import DecParams - y2 = L.f16(rng.normal(0, 2.0, (fh, fw, 32))) + if y2 is None: + y2 = L.f16(rng.normal(0, 2.0, (fh, fw, 32))) + ref_y2 = y2 if ref_y2 is None else ref_y2 h = g.hip yb = y2.astype(np.float16).tobytes() blob = LP.pack_coef(dec) @@ -85,7 +88,7 @@ def coef_case(g, rng, dec, fh, fw, rh, rw): A, B, C, Lt = (h.download(p, npx * n) for p, n in zip(outs, (4, 4, 8, 4))) for p in [py, pw] + outs: h.free(p) - a = y2[:rh, :rw][..., list(L._table.L_DEC_VALUES)].astype(np.float32) + a = np.asarray(ref_y2)[:rh, :rw][..., list(L._table.L_DEC_VALUES)].astype(np.float32) ref = L.coefficients_l(a, dec["S"], dec["T"]) got = {"codes_A": np.frombuffer(A, np.uint8).reshape(rh, rw, 4)[..., :3], "codes_B": np.frombuffer(B, np.uint8).reshape(rh, rw, 4)[..., :3], @@ -102,6 +105,63 @@ def coef_case(g, rng, dec, fh, fw, rh, rw): return m +def _fp16_ties(rng, n): + """Values exactly halfway between neighbouring FP16 numbers over the whole exponent range, both signs.""" + e = rng.integers(-14, 16, n) + m = rng.integers(0, 1024, n) + v = (1.0 + (m + 0.5) / 1024.0) * np.exp2(e.astype(np.float64)) + return np.where(rng.random(n) < 0.5, -v, v) + + +def _fp8_ties_and_saturation(): + """Midpoints between neighbouring positive FP8 E4M3 values, values above 448, negatives and zero.""" + pos = np.unique(fp8_decode(np.arange(0, 127, dtype=np.uint8))) + mids = (pos[:-1] + pos[1:]) / 2.0 + extra = np.array([448.0, 449.0, 464.0, 480.0, 1000.0, 65504.0, -0.25, -448.0, -1000.0, 0.0]) + return np.concatenate([mids, extra]) + + +def boundary_cases(g, rng, enc): + """Rounding boundaries of the two input linears, driven through the bias with zero input (acc = b exactly): FP16 + ties and overflow (l_linear_32_32_f16), FP8 ties and saturation after RN16 (l_linear_32_32_relu); then the FP16 + linear on extreme inputs (+-448) with large weights. Output bits must equal the oracle's (one rounding each).""" + out = [] + H, W = 4, 32 + npix = H * W + zero = bytes(npix * 32) + W0 = np.zeros((32, 32)) + for kernel, values, mode, nbytes in ( + ("l_linear_32_32_f16", np.concatenate([_fp16_ties(rng, 256 - 3), [65520.0, 70000.0, -1.0e5]]), "none", 64), + ("l_linear_32_32_relu", _fp8_ties_and_saturation(), "relu_q8", 32)): + bad = 0 + vals = np.concatenate([values, np.zeros((-len(values)) % 32)]) + for k in range(0, len(vals), 32): + b = vals[k:k + 32].astype(np.float32) + raw = g.run(kernel, zero, npix * nbytes, LP.pack_linear(W0, b, 32, 32), [npix], npix) + ref = L.linear(np.zeros((32,)), W0, b.astype(np.float64), "f16", mode) + if mode == "none": + got = np.frombuffer(raw, np.float16).reshape(npix, 32) + want = np.broadcast_to(ref.astype(np.float16), got.shape) + else: + got = np.frombuffer(raw, np.uint8).reshape(npix, 32) + want = np.broadcast_to(fp8_encode(ref), got.shape) + bad += int(np.sum(got.view(np.uint8 if mode != "none" else np.uint16) + != want.view(np.uint8 if mode != "none" else np.uint16))) + out.append({"case": kernel + " bias boundaries", "values": int(len(values)) * npix, "differ_bits": bad, + "pass": bad == 0}) + x = np.where(rng.random((H, W, 32)) < 0.5, -448.0, 448.0) + Wx = q8(rng.normal(0, 4.0, (32, 32))) + bx = L.f16(rng.normal(0, 100.0, 32)) + raw = g.run("l_linear_32_32_f16", fp8_encode(x).tobytes(), npix * 64, LP.pack_linear(Wx, bx, 32, 32), [npix], npix) + got = np.frombuffer(raw, np.float16).reshape(H, W, 32).astype(np.float64) + ref = L.linear(x, Wx, bx, "f16", "none") + m = C32.f16_metrics(got, ref) + m.update({"case": "l_linear_32_32_f16 extreme inputs", "overflow_values": int(np.sum(~np.isfinite(ref)))}) + m["pass"] = C32.f16_passes(m, max_steps=1) + out.append(m) + return out + + def _fp8_metrics(got, ref, case): m = C32.metrics(got, ref) m.update({"case": case, "pass": C32.passes(m)}) @@ -133,8 +193,8 @@ def run(build, rocm, seeds=(89,)): got = np.frombuffer(raw, np.float16).reshape(H, W, 32).astype(np.float64) ref = L.linear(x1, enc["Win2"], enc["bin2"], "f16", "none") m = _fp8_metrics(q8(got), q8(ref), [H, W]) - s = C32.f16_steps(got, ref) - m.update({"f16_differ": int(np.sum(s > 0)), "f16_max_steps": int(s.max())}) + m.update(C32.f16_metrics(got, ref)) + m["pass"] = m["pass"] and C32.f16_passes(m, max_steps=1) # one rounding of the accumulator res["f16"].append(m) x3 = q8(rng.normal(0, 1.5, (H, W, 32))) @@ -149,6 +209,7 @@ def run(build, rocm, seeds=(89,)): got = fp8_decode(np.frombuffer(raw, np.uint8).reshape(H, W, 32)) ref = L.depth_to_space(L.dec_project(z, dec["W"], dec["b"])) res["dec_project"].append(_fp8_metrics(got, ref, [H, W])) + res.setdefault("boundary", []).extend(boundary_cases(g, rng, enc)) for case in COEF_CASES: res.setdefault("coef", []).append(coef_case(g, rng, dec, *case)) res["coef_total"] = {k: {"values": sum(m[k]["values"] for m in res["coef"]), @@ -159,7 +220,7 @@ def run(build, rocm, seeds=(89,)): "differ": sum(m["differ"] for m in res[k]), "max_steps": max(m["max_steps"] for m in res[k])} res["f16_total"]["f16_differ"] = sum(m["f16_differ"] for m in res["f16"]) - res["pass"] = all(m["pass"] for k in ("relu", "f16", "project", "dec_project", "coef") for m in res[k]) + res["pass"] = all(m["pass"] for k in ("relu", "f16", "project", "dec_project", "coef", "boundary") for m in res[k]) return res @@ -174,7 +235,7 @@ def main(argv=None): with open(a.json, "w") as f: json.dump(res, f, indent=1) print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"}) - for k in ("relu", "f16", "project", "dec_project", "coef"): + for k in ("relu", "f16", "project", "dec_project", "coef", "boundary"): for m in res[k]: if not m["pass"]: print("FAIL", k, json.dumps(m)) diff --git a/clean/runner/m_codeobjects_compare.py b/clean/runner/m_codeobjects_compare.py new file mode 100644 index 00000000..954c289f --- /dev/null +++ b/clean/runner/m_codeobjects_compare.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: MIT +"""Model M's code objects, base versus head: proof that a change to a shared kernel source (clean/kernels/*.hip, e.g. +a template another model reuses) leaves every Model M code object byte for byte unchanged. + + python clean/runner/m_codeobjects_compare.py --base --work [--rocm ] [--json out.json] + python clean/runner/m_codeobjects_compare.py --base-tree --work ... + +Both trees are built with clean/runner/build.py --no-runtime --variants (the default code objects and every variant) +into /base and /head, each under the same output names: the build's -cuid comes from the output file name, +so only objects built under the same name are comparable. Every .co is then compared by SHA-256. + +--base checks the ref out as a detached git worktree at /base-tree (created once and reused; this script +never deletes it). --base-tree uses an existing checkout instead. The head is this script's own tree. Run it for every +PR that touches a kernel source Model M builds, and put the result line in the PR description. Exit code 0: all +identical. +""" +import argparse +import hashlib +import json +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, ROOT) + +from clean.runner.hip import DEFAULT_ROCM, hip_child_environment # noqa: E402 + +NO_WINDOW = 0x08000000 if os.name == "nt" else 0 + + +def hashes(d): + out = {} + for root, _, files in os.walk(d): + for f in files: + if f.endswith(".co"): + p = os.path.join(root, f) + with open(p, "rb") as fh: + out[os.path.relpath(p, d).replace(os.sep, "/")] = hashlib.sha256(fh.read()).hexdigest() + return out + + +def build(tree, out, rocm, env): + r = subprocess.run([sys.executable, "-B", os.path.join("clean", "runner", "build.py"), "--rocm", rocm, "--out", out, + "--no-runtime", "--variants"], cwd=tree, env=env, capture_output=True, text=True, + creationflags=NO_WINDOW) + if r.returncode: + sys.stderr.write((r.stdout + r.stderr)[-4000:]) + raise SystemExit(f"build failed in {tree} (exit {r.returncode})") + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--base", help="git ref of the base (checked out as a worktree under --work)") + g.add_argument("--base-tree", help="an existing checkout of the base") + ap.add_argument("--work", required=True, help="folder for the base worktree and both build outputs") + ap.add_argument("--rocm", default=DEFAULT_ROCM) + ap.add_argument("--json") + a = ap.parse_args(argv) + os.makedirs(a.work, exist_ok=True) + base_tree = a.base_tree + if a.base: + base_tree = os.path.join(a.work, "base-tree") + if not os.path.isdir(base_tree): + subprocess.run(["git", "-C", ROOT, "worktree", "add", "--detach", base_tree, a.base], check=True, + creationflags=NO_WINDOW) + else: + subprocess.run(["git", "-C", base_tree, "checkout", "--detach", a.base], check=True, + creationflags=NO_WINDOW) + env, _ = hip_child_environment(None, a.rocm) + env["TMP"] = env["TEMP"] = os.path.join(a.work, "tmp") + os.makedirs(env["TMP"], exist_ok=True) + outs = {"base": os.path.join(a.work, "base"), "head": os.path.join(a.work, "head")} + build(base_tree, outs["base"], a.rocm, env) + build(ROOT, outs["head"], a.rocm, env) + hb, hh = hashes(outs["base"]), hashes(outs["head"]) + differ = sorted(k for k in set(hb) | set(hh) if hb.get(k) != hh.get(k)) + res = {"base": a.base or base_tree, "base_objects": len(hb), "head_objects": len(hh), "differ": differ} + print("Model M code objects: %d base / %d head; %s" % (len(hb), len(hh), + "all identical" if not differ else "DIFFER: %s" % differ)) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 if not differ and hb else 1 + + +if __name__ == "__main__": + sys.exit(main())