From b184c901d8420f30f51524416d2ae8df55b66e97 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:18:33 -0500 Subject: [PATCH 1/2] Model L: C = 32 Swin block kernel from Model M's block template, and its GPU check clean/l/kernels/l_swin.hip includes clean/kernels/swin.hip for its templates only (WG_SWIN_NO_EXPORTS, as dec1_dec0.hip does) and exports l_swin_block_c32[_nc] (2 heads, MLP 128) with Model M's block arithmetic, packed weights and launch geometry, plus l_swin_block_c64 (the template at Model M's width) for the check that this file builds it as swin.co does. clean/l/runner/l_c32_check.py (GPU, synthetic): the C = 32 kernel against the oracle, _nc against the clamping kernel, and l_swin_block_c64 against Model M's swin_block_c64. Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/l/kernels/l_swin.hip | 31 +++++++ clean/l/runner/l_c32_check.py | 169 ++++++++++++++++++++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 clean/l/kernels/l_swin.hip create mode 100644 clean/l/runner/l_c32_check.py diff --git a/clean/l/kernels/l_swin.hip b/clean/l/kernels/l_swin.hip new file mode 100644 index 00000000..ce2f73ce --- /dev/null +++ b/clean/l/kernels/l_swin.hip @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: MIT +// Model L (issue #89): Swin block kernels at the widths Model L adds, from Model M's block template +// (clean/kernels/swin.hip, included for its templates only). Same arithmetic, packed weights +// (clean/runner/packing.py pack_block) and launch geometry as Model M's block kernels (clean/FORMATS.md, "Kernels and +// launch geometry"): 1-D grid of ceil(nwin / 8) workgroups of 256 lanes, one window per wave. +// +// Build: clang -x hip ... -I clean/kernels clean/l/kernels/l_swin.hip -o l_swin.co (clean/l/runner/l_build.py). +#define WG_SWIN_NO_EXPORTS 1 +#include "swin.hip" + +#ifndef WG_MINW_C32 +#define WG_MINW_C32 1 +#endif + +// x, xadd (unused, may be null), out, w, H, W, ox, oy: as swin_block_c64. `_nc` when bit 0 of the blob's flags word is +// set (the weights prove every inner FP8 pack in range), with identical results. +WG_KERNEL_NW(l_swin_block_c32, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, + int ox, int oy) { + swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, false, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); +} +WG_KERNEL_NW(l_swin_block_c32_nc, 256, WG_MINW_C32)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, + int ox, int oy) { + swin_block<32, 2, 128, 1, false, WG_MLP_GROUP, true, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); +} + +// The same template at Model M's C = 64 (as swin_block_c64[_nc] of swin.co): the GPU check runs it next to Model M's +// own kernel to show that this file builds the template exactly as swin.co does. +WG_KERNEL_NW(l_swin_block_c64, 256, WG_MINW_C64)(const u8* x, const u8* xadd, u8* out, const u8* w, int H, int W, + int ox, int oy) { + swin_block<64, 2, 256, 1, false, WG_MLP_GROUP, false, 256>(x, xadd, out, w, H, W, ox, oy, nullptr); +} diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py new file mode 100644 index 00000000..2ef07d1f --- /dev/null +++ b/clean/l/runner/l_c32_check.py @@ -0,0 +1,169 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): GPU check of the C = 32 Swin block kernel (clean/l/kernels/l_swin.hip), synthetic data only. + + python clean/l/runner/l_c32_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" [--compile] + [--with-swin-co] --json out.json + +--compile builds /l_swin.co (and with --with-swin-co Model M's swin.co from clean/kernels/swin.hip) on the CPU. +The GPU part runs in a GPU slot. Output: counts only, no timing. + +Checks: +1. l_swin_block_c32 against the oracle (clean/l/reference/oracle.py swin_block with the kernels' default sigmoid + GELU), per case: no non-finite value, at most 2 % of values more than one FP8 step off, mean absolute error at most + 1 % of the reference RMS (Model M's per-stage rule, clean/tests/test_kernels_emu.py), and the share of values that + differ at all. +2. l_swin_block_c32_nc against l_swin_block_c32 where the weights' flags allow it: byte-identical. +3. With swin.co: l_swin_block_c64 (this file's build of the template) against Model M's swin_block_c64: byte-identical. +""" +import argparse +import ctypes +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 +from clean.l.runner import l_pack as LP # noqa: E402 +from clean.reference import swin as MS # noqa: E402 +from clean.reference.fp8 import fp8_decode, fp8_encode, q8 # noqa: E402 +from clean.runner import packing as P # noqa: E402 + +CASES = [(9, 13, (2, 2)), (16, 8, (0, 2)), (30, 54, (0, 0)), (31, 17, (2, 0)), (48, 64, (2, 2)), (5, 7, (0, 0))] + + +def compile_kernels(build, rocm, with_swin=False, clang=None): + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + clang = clang or B.find_clang(rocm) + kdir = os.path.join(ROOT, "clean", "kernels") + B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "l", "kernels", "l_swin.hip"), + os.path.join(build, "l_swin.co")) + ["-I", kdir]) + if with_swin: + B.run(B.device_command(clang, rocm, os.path.join(kdir, "swin.hip"), os.path.join(build, "swin.co"))) + + +def ulp_distance(a, b): + ca, cb = fp8_encode(a).astype(int), fp8_encode(b).astype(int) + oa = np.where(ca & 0x80, -(ca & 0x7F), ca) + ob = np.where(cb & 0x80, -(cb & 0x7F), cb) + return np.abs(oa - ob) + + +def metrics(got, ref): + got = np.asarray(got, np.float64) + ref = np.asarray(ref, np.float64) + finite = np.isfinite(got) + d = np.abs(np.where(finite, got, 0.0) - ref) + u = ulp_distance(np.where(finite, got, 0.0), ref) + return {"nonfinite": int(np.sum(~finite)), "frac_over_1_step": float(np.mean(u > 1)), + "frac_differ": float(np.mean(u > 0)), "mean_abs_err": float(d.mean()), + "ref_rms": float(np.sqrt(np.mean(ref * ref)))} + + +def passes(m): + return m["nonfinite"] == 0 and m["frac_over_1_step"] <= 0.02 and m["mean_abs_err"] <= 0.01 * m["ref_rms"] + + +class Gpu: + def __init__(self, build, rocm): + from clean.runner.hip import Hip + self.hip = Hip(rocm) + self.mods = {} + for co in ("l_swin.co", "swin.co"): + path = os.path.join(build, co) + if os.path.exists(path): + with open(path, "rb") as f: + self.mods[co] = self.hip.load_module(f.read()) + + def block(self, co, kernel, x, blob, offset, C): + """x: FP8-exact [H, W, C] -> kernel output [H, W, C] (float32 values).""" + h = self.hip + H, W = x.shape[:2] + xb = fp8_encode(x).tobytes() + px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(len(xb)) + h.upload(px, xb) + h.upload(pw, blob) + h.memset(po, 0, len(xb)) + ox, oy = offset + nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) + args = [ctypes.c_void_p(px), ctypes.c_void_p(0), ctypes.c_void_p(po), ctypes.c_void_p(pw), + ctypes.c_int(H), ctypes.c_int(W), ctypes.c_int(ox), ctypes.c_int(oy)] + h.launch(h.get_function(self.mods[co], kernel), -(-nwin // 8), 256, args) + h.sync() + out = np.frombuffer(h.download(po, len(xb)), dtype=np.uint8).reshape(H, W, C) + for p in (px, pw, po): + h.free(p) + return fp8_decode(out) + + +def run(build, rocm, seed=89): + g = Gpu(build, rocm) + rng = np.random.default_rng(seed) + consts = dict(T.M_BLOCK_CONSTS, gelu="sigmoid") + res = {"c32": [], "c32_nc_identical": None, "c64_template_identical": None} + nc_same = [] + for H, W, off in CASES: + p = L.random_block_params(rng, 32, 2, 128, "blk.") + blob = LP.pack_block(p, "blk.", 32, 2, 128) + x = q8(rng.normal(0, 1.5, (H, W, 32))) + got = g.block("l_swin.co", "l_swin_block_c32", x, blob, off, 32) + bp = {k[4:]: v for k, v in p.items()} + ref = q8(L.swin_block(x, bp, off, 2, consts)) + m = metrics(got, ref) + m.update({"case": [H, W, list(off)], "pass": passes(m)}) + res["c32"].append(m) + safe = bool(np.frombuffer(blob[:4], np.uint32)[0] & 1) + if safe: + nc = g.block("l_swin.co", "l_swin_block_c32_nc", x, blob, off, 32) + nc_same.append(nc.tobytes() == got.tobytes()) + res["c32_nc_cases"] = len(nc_same) + res["c32_nc_identical"] = all(nc_same) if nc_same else None + if "swin.co" in g.mods: + same = [] + for H, W, off in CASES: + p = MS.random_params("enc1", rng) + blob = P.pack_stage("enc1", p) + x = q8(rng.normal(0, 1.5, (H, W, 64))) + a = g.block("swin.co", "swin_block_c64", x, blob, off, 64) + b = g.block("l_swin.co", "l_swin_block_c64", x, blob, off, 64) + same.append(a.tobytes() == b.tobytes()) + res["c64_template_identical"] = all(same) + res["c64_cases"] = len(same) + res["pass"] = (all(m["pass"] for m in res["c32"]) and res["c32_nc_identical"] in (True, None) + and res["c64_template_identical"] in (True, None)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--clang") + ap.add_argument("--compile", action="store_true") + ap.add_argument("--compile-only", action="store_true") + ap.add_argument("--with-swin-co", action="store_true") + ap.add_argument("--json", required=True) + a = ap.parse_args(argv) + os.makedirs(a.build, exist_ok=True) + if a.compile or a.compile_only: + compile_kernels(a.build, a.rocm, a.with_swin_co, a.clang) + if a.compile_only: + return 0 + res = run(a.build, a.rocm) + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + for m in res["c32"]: + print(json.dumps(m)) + print({k: v for k, v in res.items() if k != "c32"}) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) From 5a1b9a281f6286bbd43a7c41a8a410a68f72121a Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:39:51 -0500 Subject: [PATCH 2/2] Model L C = 32 check: several seeds, total and largest FP8-step difference; README rows Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/l/README.md | 2 ++ clean/l/runner/l_c32_check.py | 17 +++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/clean/l/README.md b/clean/l/README.md index 536e6858..29e626af 100644 --- a/clean/l/README.md +++ b/clean/l/README.md @@ -8,6 +8,8 @@ launcher yet. | `reference/table.py` | Model L's structure table: the encoder's op list, the decoder's channel segments and block, window offsets, block constants, map names; parameter shapes from the table; matrix work per frame (computed, not measured). Entries in `PROVISIONAL` are pending confirmation and are changed here only | | `reference/oracle.py` | NumPy oracle (float64, only the prescribed FP8 / FP16 roundings): `encode`, `decode`, `network`, synthetic `random_params`. The Swin stages it shares with Model M call Model M's oracle (`clean/reference/swin.py`); the output reconstruction is not written yet | | `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged | +| `kernels/l_swin.hip` | the C = 32 Swin block kernels (`l_swin_block_c32[_nc]`) from Model M's block template (`clean/kernels/swin.hip`, included for its templates only), same blob format and launch geometry as Model M's block kernels | +| `runner/l_c32_check.py` | GPU check (synthetic): the C = 32 kernel against the oracle, `_nc` against the clamping kernel, and this file's C = 64 build of the template against Model M's `swin_block_c64` | | `tests/test_l_pack.py` | packing round trips, the block packer against Model M's stage blobs, and the unpacked weights applied the kernels' way against the oracle's maps, byte for byte | | `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte | diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 2ef07d1f..2084ade7 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -63,7 +63,8 @@ def metrics(got, ref): d = np.abs(np.where(finite, got, 0.0) - ref) u = ulp_distance(np.where(finite, got, 0.0), ref) return {"nonfinite": int(np.sum(~finite)), "frac_over_1_step": float(np.mean(u > 1)), - "frac_differ": float(np.mean(u > 0)), "mean_abs_err": float(d.mean()), + "frac_differ": float(np.mean(u > 0)), "values": int(u.size), "differ": int(np.sum(u > 0)), + "max_steps": int(u.max()) if u.size else 0, "mean_abs_err": float(d.mean()), "ref_rms": float(np.sqrt(np.mean(ref * ref)))} @@ -103,13 +104,13 @@ def block(self, co, kernel, x, blob, offset, C): return fp8_decode(out) -def run(build, rocm, seed=89): +def run(build, rocm, seeds=(89,)): g = Gpu(build, rocm) - rng = np.random.default_rng(seed) + rngs = [np.random.default_rng(s) for s in seeds] consts = dict(T.M_BLOCK_CONSTS, gelu="sigmoid") res = {"c32": [], "c32_nc_identical": None, "c64_template_identical": None} nc_same = [] - for H, W, off in CASES: + for rng, (H, W, off) in ((r, c) for r in rngs for c in CASES): p = L.random_block_params(rng, 32, 2, 128, "blk.") blob = LP.pack_block(p, "blk.", 32, 2, 128) x = q8(rng.normal(0, 1.5, (H, W, 32))) @@ -127,7 +128,7 @@ def run(build, rocm, seed=89): res["c32_nc_identical"] = all(nc_same) if nc_same else None if "swin.co" in g.mods: same = [] - for H, W, off in CASES: + for rng, (H, W, off) in ((r, c) for r in rngs for c in CASES): p = MS.random_params("enc1", rng) blob = P.pack_stage("enc1", p) x = q8(rng.normal(0, 1.5, (H, W, 64))) @@ -136,6 +137,9 @@ def run(build, rocm, seed=89): same.append(a.tobytes() == b.tobytes()) res["c64_template_identical"] = all(same) res["c64_cases"] = len(same) + res["c32_total"] = {"cases": len(res["c32"]), "values": sum(m["values"] for m in res["c32"]), + "differ": sum(m["differ"] for m in res["c32"]), + "max_steps": max(m["max_steps"] for m in res["c32"])} res["pass"] = (all(m["pass"] for m in res["c32"]) and res["c32_nc_identical"] in (True, None) and res["c64_template_identical"] in (True, None)) return res @@ -149,6 +153,7 @@ def main(argv=None): ap.add_argument("--compile", action="store_true") ap.add_argument("--compile-only", action="store_true") ap.add_argument("--with-swin-co", action="store_true") + ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--json", required=True) a = ap.parse_args(argv) os.makedirs(a.build, exist_ok=True) @@ -156,7 +161,7 @@ def main(argv=None): compile_kernels(a.build, a.rocm, a.with_swin_co, a.clang) if a.compile_only: return 0 - res = run(a.build, a.rocm) + res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) with open(a.json, "w") as f: json.dump(res, f, indent=1) for m in res["c32"]: