diff --git a/clean/kernels/encoder.hip b/clean/kernels/encoder.hip index c1327f1a..e9b735e7 100644 --- a/clean/kernels/encoder.hip +++ b/clean/kernels/encoder.hip @@ -83,6 +83,11 @@ #ifndef WG_ENC_AB_STORE #define WG_ENC_AB_STORE 0 // no motion-surface stores #endif +// Model L (clean/l, #89): instead of the projection, store the quantized values as a token map [fh/2][fw/2][80] +// (sub-pixel p = dx + 2 dy, value i at byte 20 p + i). Built as its own code object; off: this file's kernel unchanged. +#ifndef WG_ENC_VALUES_OUT +#define WG_ENC_VALUES_OUT 0 +#endif // Per-frame constants, passed by value (all 4-byte fields; must match EncParams in clean/runner/enc0.py). struct WgEncParams { @@ -524,7 +529,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri else pixel_values(f, P, U, (int)xs[dx], y, c, px); quantize(px.v, dx ? t2 >> 8 : t2 & 0xffu, q[dx]); -#if !WG_ENC_AB_PROJ +#if !WG_ENC_AB_PROJ && !WG_ENC_VALUES_OUT // Values 8..15 go to the partner lane (the other lane half of the same token). x8[dx][0] = wg_xor16_u(q[dx][2]); x8[dx][1] = wg_xor16_u(q[dx][3]); @@ -534,7 +539,18 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri #endif } -#if WG_ENC_AB_PROJ +#if WG_ENC_VALUES_OUT + // Lane half hh holds sub-pixels 2 hh and 2 hh + 1 (row dy = hh): 40 contiguous bytes of the token at 40 hh. + if (tok < ntok) { + const u32 o = (u32)tok * 80u + hh * 40u; + wg_st64(features, o, (v2u){q[0][0], q[0][1]}); + wg_st64(features, o + 8u, (v2u){q[0][2], q[0][3]}); + wg_st64(features, o + 16u, (v2u){q[0][4], q[1][0]}); + wg_st64(features, o + 24u, (v2u){q[1][1], q[1][2]}); + wg_st64(features, o + 32u, (v2u){q[1][3], q[1][4]}); + } + (void)w; (void)x8; +#elif WG_ENC_AB_PROJ if (tok < ntok) { v4u o0 = {q[0][0], q[0][1], q[0][2], q[0][3]}, o1 = {q[1][0], q[1][1], q[1][2], q[1][3] ^ q[0][4] ^ q[1][4]}; wg_st128(features, (u32)tok * kFeat + hh * 32u, o0); diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md new file mode 100644 index 00000000..fbe1756f --- /dev/null +++ b/clean/l/FORMATS.md @@ -0,0 +1,57 @@ +# Model L: kernels, blobs and launch order (issue #89) + +Status: HIP kernels for Model L's encoder chain, decoder front and C = 32 block; no runtime integration yet. Feature +maps are `[tokens][C]` FP8 E4M3 bytes, channel fastest, as in Model M (`clean/FORMATS.md`, "Feature maps"); a token +map at F/2 has `ntok = (fh / 2) * (fw / 2)` tokens in raster order. Blobs: `clean/l/runner/l_pack.py` +(`pack_model_l`); the ten shared Swin stages use Model M's blobs, kernels and launch plan unchanged. + +## Code objects + +| File | Source | Kernels | +|---|---|---| +| `l_enc_values.co` | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` (values only) | +| `l_ops.co` | `clean/l/kernels/l_ops.hip` (`-I clean/kernels`) | `l_linear_80_32`, `l_linear_32_32`, `l_project_80_32`, `l_copy_channels` | +| `l_swin.co` | `clean/l/kernels/l_swin.hip` (`-I clean/kernels`) | `l_swin_block_c32`, `l_swin_block_c32_nc`, `l_swin_block_c64` (check only) | + +The runtime loads `l_enc_values.co` under the tag `lev` (its kernel is launched as `lev/input_encoder`, next to Model +M's own `input_encoder`); `l_ops.co` and `l_swin.co` are untagged. These are also the names the D3D12 ports will +declare. + +All are built with `clean/runner/build.py`'s `device_command` (gfx1201). The build's `-cuid` comes from the output file +name, so compare code objects only between outputs of the same name. + +## Kernels + +| Kernel | Arguments | Grid (workgroups of 256 lanes) | Does | +|---|---|---|---| +| `input_encoder` of `l_enc_values.co` | as Model M's `input_encoder` (color, depth, motion, hist, latent, mhist, **raw**, m_near, m_far, m_center, expo, w, `WgEncParams`) | ceil(ntok / 128) | the per-pixel values and their dithered FP8 quantization as Model M's enc0; writes `raw` [ntok][80] (sub-pixel p = dx + 2 dy, value i at byte 20 p + i) and the three motion surfaces; w = blob `enc0.values` | +| `l_linear_80_32` | x, y, w, ntok | ceil(ntok / 128) | y [ntok][32] = q8(W x + b), x [ntok][80]; w = blob `enc0.Wa` | +| `l_linear_32_32` | x, y, w, ntok | ceil(ntok / 128) | y [ntok][32] = q8(W x + b); w = blob `enc0.Wb` | +| `l_swin_block_c32[_nc]` | x, xadd (null), out, w, H, W, ox, oy | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | one Swin block at C = 32 (2 heads, MLP 128), as Model M's block kernels; `_nc` when bit 0 of the blob's first word is set; w = blob `enc0.blk` or `dec0.blk` | +| `l_project_80_32` | raw, x, features, w, ntok | ceil(ntok / 128) | features [ntok][64] = q8(clamp(s, +-6.28125)) with s = b, then s = RN16(s + W[:, g] z[g]) for each group g of 32 channels of z = [raw (80) | x (32) | 16 zeros]; w = blob `enc0.project` | +| `l_copy_channels` | src, sstride, soff, dst, dstride, doff, n, ntok | ceil(ntok n / 16 / 256) | dst[t][doff .. doff + n) = src[t][soff .. soff + n) (bytes; strides, offsets and n multiples of 16) | + +## Launch order of one frame (default table) + +Model M's frame (`clean/FORMATS.md`, "Stage order per frame") with steps 2 and 4 replaced: + +2. Encoder (offsets of the block: table entry `window_offsets["first"]` for the frame parity): + 1. `input_encoder` (`l_enc_values.co`) -> `raw` [F/2][80], m_near, m_far, m_center + 2. `l_linear_80_32` raw -> `x1` [F/2][32] + 3. `l_linear_32_32` x1 -> `x2` [F/2][32] + 4. `l_swin_block_c32[_nc]` x2 -> `x3` [F/2][32] (H = fh / 2, W = fw / 2) + 5. `l_project_80_32` raw, x3 -> `enc0.features` [F/2][64] (what enc1 reads) +3. enc1 ... dec1: Model M's stages with Model L's weights, unchanged. +4. Decoder (offsets: `window_offsets["second"]`): + 1. `l_copy_channels` dec1.out (stride 64, offset 0) -> `y` [F/2][32] + 2. `l_swin_block_c32[_nc]` y -> `y2` [F/2][32] + 3. `l_copy_channels` y2 (32, 0) -> `z` (64, 0), n = 32; `l_copy_channels` dec1.out (64, 32) -> `z` (64, 32), n = 32 + 4. Model M's `coef_decoder` with features = `z`, w = blob `dec0` + +Extra buffers per frame: `raw` 80 ntok bytes, `x1` / `x2` / `x3` 32 ntok each (two can be shared), `y` / `y2` 32 ntok, +`z` 64 ntok. The launch list is derived from the table by `clean/l/runner/l_pack.py` `encoder_launches`. + +## Checks (GPU, synthetic data) + +`clean/l/runner/l_c32_check.py` and `clean/l/runner/l_chain_check.py`: every kernel in isolation against the oracle +step on the same input, the whole encoder chain, and the decoder front (see their docstrings for the rules). diff --git a/clean/l/README.md b/clean/l/README.md index 29e626af..fb5d6bfb 100644 --- a/clean/l/README.md +++ b/clean/l/README.md @@ -1,7 +1,7 @@ # Model L (issue #89) Work in progress for a new model choice, milestone 0.7.0. Nothing here is used by the runtime, the front end or the -launcher yet. +launcher yet. Kernels, blobs and the launch order of a frame: [FORMATS.md](FORMATS.md). | Path | Content | |---|---| @@ -10,6 +10,8 @@ launcher yet. | `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged | | `kernels/l_swin.hip` | the C = 32 Swin block kernels (`l_swin_block_c32[_nc]`) from Model M's block template (`clean/kernels/swin.hip`, included for its templates only), same blob format and launch geometry as Model M's block kernels | | `runner/l_c32_check.py` | GPU check (synthetic): the C = 32 kernel against the oracle, `_nc` against the clamping kernel, and this file's C = 64 build of the template against Model M's `swin_block_c64` | +| `kernels/l_ops.hip` | the encoder chain's pointwise linears and output projection, and the channel copy of the decoder front | +| `runner/l_chain_check.py` | GPU check (synthetic): each kernel of the encoder chain and the decoder front against the oracle step on the same input, the whole encoder chain, the decoder front; Model M's encoder.co against a build of main | | `tests/test_l_pack.py` | packing round trips, the block packer against Model M's stage blobs, and the unpacked weights applied the kernels' way against the oracle's maps, byte for byte | | `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | diff --git a/clean/l/kernels/l_ops.hip b/clean/l/kernels/l_ops.hip new file mode 100644 index 00000000..a00380a4 --- /dev/null +++ b/clean/l/kernels/l_ops.hip @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): the small FP8-map kernels of Model L's encoder and decoder chains (clean/l/runner/l_pack.py +// encoder_launches). Maps are [tokens][C] FP8 E4M3 bytes, channel fastest (clean/FORMATS.md "Feature maps"). Blobs: +// clean/l/runner/l_pack.py. Launch: 1-D grid of ceil(tokens / 128) workgroups of 256 lanes (8 waves); wave i handles +// tokens 16 i .. 16 i + 15; lane (t, h) works on token 16 i + t, lane half h on channels [h C/2, (h + 1) C/2) (layout L) +// or, for the projection, on the 16 channels of each group that its K slots name. +#include "wg_platform.h" +#include "wg_mem.h" + +namespace { + +constexpr int kThreads = 256; +constexpr int kTokensPerWave = 16; + +WG_FN int token_of(u32 tid) { + return (int)(wg_block_id() * (kThreads / WG_WAVE) + (tid >> 5)) * kTokensPerWave + (int)(tid & 15u); +} + +// y = q8(W x + b), FP32 accumulation over the K groups in order. Blob: bias tile vector (rows R_L(CO)), then fragments +// [mt < CO/16][kc < CI/16] with rows R_L(CO) and K order K_L(CI) (merge / expand section format). +template +WG_FN void linear_map(const u8* __restrict__ x, u8* __restrict__ y, const u8* __restrict__ w, int ntok) { + constexpr int KC = CI / 16, MT = CO / 16; + static_assert(CI % 16 == 0 && CO % 16 == 0, "widths in multiples of 16"); + const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int tok = token_of(tid); + const int tk = tok < ntok ? tok : ntok - 1; + v2i xf[KC]; +#pragma unroll + for (int kc = 0; kc < KC; ++kc) xf[kc] = *(const v2i*)(x + (size_t)tk * CI + hh * (CI / 2) + 8 * kc); + const float* bias = (const float*)w + hh * 8; + const u8* frag = w + 4 * CO; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + v8f acc = wg_load8f(bias + mt * 16); +#pragma unroll + for (int kc = 0; kc < KC; ++kc) acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KC + kc) * 256))[lane], xf[kc], acc); + const v2i q = wg_pack_q8(acc); + if (tok < ntok) *(v2i*)(y + (size_t)tok * CO + hh * (CO / 2) + 8 * mt) = q; + } +} + +// 8 bytes of the concatenated input z = [a (CA channels) | b (CB channels) | zeros] at channel c (c a multiple of 8; +// CA and CB are multiples of 8, so the 8 bytes never straddle two parts). +template +WG_FN v2i concat8(const u8* a, const u8* b, int tk, int c) { + if (c < CA) return *(const v2i*)(a + (size_t)tk * CA + c); + if (c < CA + CB) return *(const v2i*)(b + (size_t)tk * CB + (c - CA)); + v2i z = {0, 0}; + return z; +} + +// features = q8(clamp(s, +-6.28125)), s = b; for each group g of 32 channels of z: s = RN16(s + W[:, g] z[g]). +// Blob: bias tile vector (rows R_L(64)), then fragments [mt < 4][kb < NZ/16], rows R_L(64), K order: block kb slot +// (h, j) = channel 32 (kb / 2) + 16 h + 8 (kb % 2) + j (clean/l/runner/l_pack.py kord_groups32). +template +WG_FN void project_map(const u8* __restrict__ a, const u8* __restrict__ b, u8* __restrict__ feat, + const u8* __restrict__ w, int ntok) { + constexpr int CO = 64, MT = CO / 16, KB = NZ / 16; + static_assert(NZ % 32 == 0 && CA + CB <= NZ && CA % 8 == 0 && CB % 8 == 0, "groups of 32, parts of 8 bytes"); + const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u; + const int tok = token_of(tid); + const int tk = tok < ntok ? tok : ntok - 1; + v2i zf[KB]; +#pragma unroll + for (int kb = 0; kb < KB; ++kb) zf[kb] = concat8(a, b, tk, 32 * (kb >> 1) + 16 * (int)hh + 8 * (kb & 1)); + const float* bias = (const float*)w + hh * 8; + const u8* frag = w + 4 * CO; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + v8f acc = wg_load8f(bias + mt * 16); +#pragma unroll + for (int g = 0; g < KB / 2; ++g) { + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g) * 256))[lane], zf[2 * g], acc); + acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g + 1) * 256))[lane], zf[2 * g + 1], acc); +#pragma unroll + for (int i = 0; i < 8; ++i) acc[i] = wg_rn16(acc[i]); + } +#pragma unroll + for (int i = 0; i < 8; ++i) acc[i] = wg_fmin(wg_fmax(acc[i], -6.28125f), 6.28125f); + const v2i q = wg_pack_q8(acc); + if (tok < ntok) *(v2i*)(feat + (size_t)tok * CO + hh * (CO / 2) + 8 * mt) = q; + } +} + +} // namespace + +// Pointwise linears of Model L's encoder: 80 -> 32 (the raw value map; the select is zero weights) and 32 -> 32. +WG_KERNEL_N(l_linear_80_32, kThreads)(const u8* x, u8* y, const u8* w, int ntok) { linear_map<80, 32>(x, y, w, ntok); } +WG_KERNEL_N(l_linear_32_32, kThreads)(const u8* x, u8* y, const u8* w, int ntok) { linear_map<32, 32>(x, y, w, ntok); } + +// The encoder's output projection over [raw (80) | x (32) | 16 zeros] -> the 64 FP8 features enc1 reads. +WG_KERNEL_N(l_project_80_32, kThreads)(const u8* raw, const u8* x, u8* feat, const u8* w, int ntok) { + project_map<80, 32, 128>(raw, x, feat, w, ntok); +} + +// dst[t][doff .. doff + n) = src[t][soff .. soff + n) for every token t < ntok, n a multiple of 16 bytes; one lane per +// 16 bytes. Splits dec1's map for the decoder block and joins the block output back (clean/l/runner). +WG_KERNEL_N(l_copy_channels, kThreads)(const u8* src, int sstride, int soff, u8* dst, int dstride, int doff, int n, + int ntok) { + const int chunks = n >> 4; + const long long i = (long long)wg_block_id() * kThreads + wg_lane_id(); + const long long t = i / chunks; + if (t >= ntok) return; + const int k = (int)(i - t * chunks); + *(v4u*)(dst + (size_t)t * dstride + doff + 16 * k) = *(const v4u*)(src + (size_t)t * sstride + soff + 16 * k); +} diff --git a/clean/l/runner/l_chain_check.py b/clean/l/runner/l_chain_check.py new file mode 100644 index 00000000..f50aa494 --- /dev/null +++ b/clean/l/runner/l_chain_check.py @@ -0,0 +1,255 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of Model L's encoder chain and decoder front, synthetic data only. + + python clean/l/runner/l_chain_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--compile] + [--main ] --json out.json + +--compile (CPU) builds into : l_ops.co (clean/l/kernels/l_ops.hip), l_swin.co, l_enc_values.co +(clean/kernels/encoder.hip with WG_ENC_VALUES_OUT=1) and encoder.co (the same file without it); with --main also +encoder.co of that checkout as encoder_main.co, and reports whether the two encoder.co are byte-identical (the new +switch leaves Model M's kernel unchanged). The GPU part runs in a GPU slot. Output: counts only, no timing. + +Every kernel is checked in isolation: its input is the GPU's own output of the step before, and the reference is the +oracle's step on that same input (clean/l/reference/oracle.py, the block with the kernels' sigmoid GELU). Reported per +step: values that differ at all, values more than one FP8 step off, non-finite values, mean absolute error against the +reference RMS. Also the whole encoder chain (GPU features against the oracle's encode) and the decoder front. +""" +import argparse +import ctypes +import filecmp +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.l.runner.l_c32_check import metrics # noqa: E402 +from clean.reference import encoder as ME # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 +from clean.runner import enc0 as E0 # noqa: E402 + +FRAMES = [(40, 24), (64, 36), (97, 55)] + + +def compile_kernels(build, rocm, main_tree=None, clang=None): + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + clang = clang or B.find_clang(rocm) + kdir = os.path.join(ROOT, "clean", "kernels") + ldir = os.path.join(ROOT, "clean", "l", "kernels") + B.run(B.device_command(clang, rocm, os.path.join(ldir, "l_ops.hip"), os.path.join(build, "l_ops.co")) + + ["-I", kdir]) + B.run(B.device_command(clang, rocm, os.path.join(ldir, "l_swin.hip"), os.path.join(build, "l_swin.co")) + + ["-I", kdir]) + B.run(B.device_command(clang, rocm, os.path.join(kdir, "encoder.hip"), os.path.join(build, "l_enc_values.co"), + defines={"WG_ENC_VALUES_OUT": 1})) + B.run(B.device_command(clang, rocm, os.path.join(kdir, "encoder.hip"), os.path.join(build, "encoder.co"))) + res = {} + if main_tree: + # same output file name: the build's -cuid is derived from it, so differently named outputs always differ + os.makedirs(os.path.join(build, "main"), exist_ok=True) + B.run(B.device_command(clang, rocm, os.path.join(main_tree, "clean", "kernels", "encoder.hip"), + os.path.join(build, "main", "encoder.co"))) + res["encoder_co_identical_to_main"] = filecmp.cmp(os.path.join(build, "encoder.co"), + os.path.join(build, "main", "encoder.co"), shallow=False) + return res + + +class Gpu: + def __init__(self, build, rocm): + from clean.runner.hip import Hip + self.h = Hip(rocm) + self.mods = {} + for co in ("l_ops.co", "l_swin.co", "l_enc_values.co"): + with open(os.path.join(build, co), "rb") as f: + self.mods[co] = self.h.load_module(f.read()) + self.live = [] + + def buf(self, data=None, nbytes=None): + n = len(data) if data is not None else nbytes + p = self.h.malloc(n) + if data is not None: + self.h.upload(p, data) + else: + self.h.memset(p, 0, n) + self.live.append(p) + return p + + def free_all(self): + for p in self.live: + self.h.free(p) + self.live = [] + + def launch(self, co, kernel, grid, args): + self.h.launch(self.h.get_function(self.mods[co], kernel), grid, 256, args) + self.h.sync() + + def get_fp8(self, p, shape): + return fp8_decode(np.frombuffer(self.h.download(p, int(np.prod(shape))), dtype=np.uint8)).reshape(shape) + + +def V(p): + return ctypes.c_void_p(p) + + +def run_values(g, x, enc): + """The values kernel on frame x -> raw map [fh/2, fw/2, 80].""" + prm = E0.frame_params(x) + data = E0.device_inputs(x) + ntok = (prm.fh // 2) * (prm.fw // 2) + bufs = {k: g.buf(v) for k, v in data.items()} + w = g.buf(LP.pack_values(enc)) + expo = g.buf(np.array([x["exposure"], x["exposure_prev"]], dtype=np.float32).tobytes()) + raw = g.buf(nbytes=ntok * 80) + rw, rh = prm.rw, prm.rh + mn, mf, mc = (g.buf(nbytes=rw * rh * 4) for _ in range(3)) + args = [V(bufs[k]) for k in ("color", "depth", "motion", "hist", "latent", "mhist")] + \ + [V(raw), V(mn), V(mf), V(mc), V(expo), V(w), prm] + g.launch("l_enc_values.co", "input_encoder", -(-ntok // 128), args) + return g.get_fp8(raw, (prm.fh // 2, prm.fw // 2, 80)), raw, ntok + + +def run_linear(g, kernel, x_ptr, ntok, blob, C_out, shape_hw): + y = g.buf(nbytes=ntok * C_out) + w = g.buf(blob) + g.launch("l_ops.co", kernel, -(-ntok // 128), [V(x_ptr), V(y), V(w), ctypes.c_int(ntok)]) + return g.get_fp8(y, shape_hw + (C_out,)), y + + +def run_block(g, x, x_ptr, blob, offset): + H, W, C = x.shape + y = g.buf(nbytes=H * W * C) + w = g.buf(blob) + ox, oy = offset + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + g.launch("l_swin.co", "l_swin_block_c32", -(-nwin // 8), + [V(x_ptr), V(0), V(y), V(w), ctypes.c_int(H), ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)]) + return g.get_fp8(y, (H, W, C)), y + + +def copy_channels(g, src, sstride, soff, dst, dstride, doff, n, ntok): + chunks = n // 16 + g.launch("l_ops.co", "l_copy_channels", -(-(ntok * chunks) // 256), + [V(src), ctypes.c_int(sstride), ctypes.c_int(soff), V(dst), ctypes.c_int(dstride), ctypes.c_int(doff), + ctypes.c_int(n), ctypes.c_int(ntok)]) + + +def stage(name, got, ref): + m = metrics(got, ref) + m["step"] = name + return m + + +def run(build, rocm, seeds=(89,)): + g = Gpu(build, rocm) + consts = dict(T.M_BLOCK_CONSTS, gelu="sigmoid") + t = T.TABLE + out = [] + for seed, (rw, rh), parity in ((s, f, p) for s in seeds for f in FRAMES for p in (0, 1)): + rng = np.random.default_rng([seed, rw, rh, parity]) + params = L.random_params(rng) + enc = params["enc0"] + blobs = LP.pack_model_l(params) + x = ME.random_inputs(rng, rw=rw, rh=rh, frame=parity) + fw, fh = x["feature_size"] + hw = (fh // 2, fw // 2) + steps = [] + raw, raw_p, ntok = run_values(g, x, enc) + ref = L.encode(x, enc, parity, t, keep=True) + steps.append(stage("values", raw, ref["maps"]["raw"])) + h1, h1_p = run_linear(g, "l_linear_80_32", raw_p, ntok, blobs["enc0.Wa"], 32, hw) + steps.append(stage("linear1", h1, q8(raw[..., :20] @ np.asarray(enc["Wa"], np.float64).T + enc["ba"]))) + h2, h2_p = run_linear(g, "l_linear_32_32", h1_p, ntok, blobs["enc0.Wb"], 32, hw) + steps.append(stage("linear2", h2, q8(h1 @ np.asarray(enc["Wb"], np.float64).T + enc["bb"]))) + off = T.block_offset(t, "first", parity) + bp = {k[4:]: v for k, v in enc.items() if k.startswith("blk.")} + h3, h3_p = run_block(g, h2, h2_p, blobs["enc0.blk"], off) + steps.append(stage("block", h3, q8(L.swin_block(h2, bp, off, 2, consts)))) + feat_p = g.buf(nbytes=ntok * 64) + wproj = g.buf(blobs["enc0.project"]) + g.launch("l_ops.co", "l_project_80_32", -(-ntok // 128), + [V(raw_p), V(h3_p), V(feat_p), V(wproj), ctypes.c_int(ntok)]) + feat = g.get_fp8(feat_p, hw + (64,)) + z = np.concatenate([raw, h3, np.zeros(hw + (16,))], axis=-1) + steps.append(stage("project", feat, L.project_groups(z, enc["W"], enc["b"], (32, 32, 32, 32)))) + ref_sig = L.encode(x, enc, parity, T.table(consts={"default": consts})) + steps.append(stage("encoder_chain", feat, ref_sig["features"])) + # decoder front: split dec1's map, block on channels 0..31, join + dec = params["dec0"] + Z = q8(rng.normal(0, 1.5, hw + (64,))) + Z_p = g.buf(fp8_encode(Z).tobytes()) + y_p = g.buf(nbytes=ntok * 32) + copy_channels(g, Z_p, 64, 0, y_p, 32, 0, 32, ntok) + y = g.get_fp8(y_p, hw + (32,)) + steps.append(stage("split", y, Z[..., :32])) + offd = T.block_offset(t, "second", parity) + dbp = {k[4:]: v for k, v in dec.items() if k.startswith("blk.")} + yb, yb_p = run_block(g, y, y_p, blobs["dec0.blk"], offd) + front_p = g.buf(nbytes=ntok * 64) + copy_channels(g, yb_p, 32, 0, front_p, 64, 0, 32, ntok) + copy_channels(g, Z_p, 64, 32, front_p, 64, 32, 32, ntok) + front = g.get_fp8(front_p, hw + (64,)) + want = L.decoder_front(Z, dec, parity, T.table(consts={"default": consts})) + steps.append(stage("decoder_front", front, want)) + steps.append({"step": "join_exact", "identical": bool(np.array_equal(front[..., :32], yb) and + np.array_equal(front[..., 32:], Z[..., 32:]))}) + out.append({"frame": [rw, rh], "parity": parity, "steps": steps}) + g.free_all() + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--clang") + ap.add_argument("--compile", action="store_true") + ap.add_argument("--compile-only", action="store_true") + ap.add_argument("--main", help="a checkout of main: compare its encoder.co with this tree's") + ap.add_argument("--seeds", type=int, default=1, help="run the frames with this many seeds (89, 90, ...)") + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + res = {} + if a.compile or a.compile_only: + res["compile"] = compile_kernels(a.build, a.rocm, a.main, a.clang) + print(res["compile"], flush=True) + if a.compile_only: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + return 0 + res["frames"] = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) + tot = {} + for fr in res["frames"]: + for st in fr["steps"]: + if "values" in st: + t = tot.setdefault(st["step"], {"values": 0, "differ": 0, "max_steps": 0, "nonfinite": 0, + "max_frac_over_1_step": 0.0, "spec09_mean_ok": True}) + t["values"] += st["values"] + t["differ"] += st["differ"] + t["max_steps"] = max(t["max_steps"], st["max_steps"]) + t["nonfinite"] += st["nonfinite"] + t["max_frac_over_1_step"] = max(t["max_frac_over_1_step"], st["frac_over_1_step"]) + t["spec09_mean_ok"] &= st["mean_abs_err"] <= 0.01 * st["ref_rms"] + elif "identical" in st: + t = tot.setdefault(st["step"], {"cases": 0, "identical": 0}) + t["cases"] += 1 + t["identical"] += int(st["identical"]) + res["totals"] = tot + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + for k, v in res["totals"].items(): + print(k, v) + return 0 + + +if __name__ == "__main__": + sys.exit(main())