From fa2d003896a2ab272252f99804e1245e4b456394 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 13:27:27 -0500 Subject: [PATCH] Frame generation: net1 convolution kernel for HIP and D3D12, bit for bit against the CPU reference fg_conv_f16 (clean/fg/kernels/fg_net1.hip, code object fg_net1.co) and its D3D12 port (clean/dx12/kernels/fg_conv.hlsl): one net1 convolution, 3x3 or 1x1, zero border, float16 accumulator from the bias in exact 16-term chunks in the layer's chunk order, leaky ReLU (slope 0x211E) or none. The arithmetic is integer code shared by both back ends (clean/fg/kernels/fg_exact16.h, the method of fp16acc.py). build_dx12.py builds fg_ kernels only when --only names them, so the Model M and Model L kernel sets (kernel_sets.json) and their DXILs stay as they are; test_kernel_sets checks it. clean/fg/runner/fg_d3d12_check.py: HIP against D3D12 (raw bytes, sentinels 0xA5 / 0x5A, slack, half-grid negative control) and HIP against the CPU reference, on every net1 layer shape and special values. Refs #96 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/README.md | 1 + clean/dx12/build_dx12.py | 17 +- clean/dx12/kernels/fg_conv.hlsl | 68 +++++++ clean/dx12/tests/test_kernel_sets.py | 13 ++ clean/fg/FORMATS.md | 14 ++ clean/fg/README.md | 3 + clean/fg/kernels/fg_exact16.h | 142 ++++++++++++++ clean/fg/kernels/fg_net1.hip | 60 ++++++ clean/fg/runner/__init__.py | 1 + clean/fg/runner/fg_d3d12_check.py | 275 +++++++++++++++++++++++++++ 10 files changed, 592 insertions(+), 2 deletions(-) create mode 100644 clean/dx12/kernels/fg_conv.hlsl create mode 100644 clean/fg/kernels/fg_exact16.h create mode 100644 clean/fg/kernels/fg_net1.hip create mode 100644 clean/fg/runner/__init__.py create mode 100644 clean/fg/runner/fg_d3d12_check.py diff --git a/clean/dx12/README.md b/clean/dx12/README.md index 4a70db2b..79213c48 100644 --- a/clean/dx12/README.md +++ b/clean/dx12/README.md @@ -20,6 +20,7 @@ as its HIP kernel (`clean/dx12/compare.py` proves it per kernel on synthetic fra | `dxil_asm.cpp` | tool used by the build (assembles the rewritten DXIL listing) | | `compare.py` | HIP vs hybrid D3D12, every stage output / state buffer byte for byte | | `kernels/exposure.hlsl` | the reference port (`exposure_auto_diet`): copy its structure | +| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; check: `clean/fg/runner/fg_d3d12_check.py` | ## Arithmetic mode (why build_dx12.py and nothing else) diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 6b951950..1327395d 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -5,7 +5,8 @@ python clean/dx12/build_dx12.py --out build/dx12-m/hip [--isa] [--only exposure_auto_diet,...] [-j N] [--model-l] Model L's kernels (#89: those that reproduce one of Model L's code objects, co=l_*.co, direct-route kernels named l_*, and MODEL_L_ONLY) are built only with --model-l -or when --only names them, so that a build without the option is Model M's kernel set unchanged. +or when --only names them, so that a build without the option is Model M's kernel set unchanged. Frame generation's +kernels (#96: co=fg_*.co or named fg_*) are built only when --only names them, never by a full build. Every kernel source declares its kernels in header lines (clean/dx12/README.md): @@ -186,13 +187,23 @@ def is_model_l(decl): return decl["co"].startswith("l_") or decl["name"].startswith("l_") or decl["name"] in MODEL_L_ONLY +def is_frame_gen(decl): + """A port of frame generation's kernels (#96: code objects fg_*.co, or named fg_*). They are groundwork that no model + build launches yet: built only when --only names them, never by a full build (with or without --model-l), so the + Model M and Model L kernel sets and their DXILs stay as they are.""" + return decl["co"].startswith("fg_") or decl["name"].startswith("fg_") + + def kernel_set(model_l=False): """(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's - with model_l. From the sources' declarations alone (nothing is compiled).""" + with model_l. From the sources' declarations alone (nothing is compiled). Frame generation's kernels are never in + it (is_frame_gen).""" keys = set() for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: if fn.endswith(".hlsl"): for d in kernel_decls(os.path.join(KERNEL_DIR, fn)): + if is_frame_gen(d): + continue if model_l or not is_model_l(d): keys.add((d["name"], d["co"])) return sorted(keys) @@ -321,6 +332,8 @@ def main(argv=None): continue if not only and not a.model_l and is_model_l(d): continue + if not only and is_frame_gen(d): + continue key = (d["name"], d["co"]) if key in kernels and key not in old: raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") diff --git a/clean/dx12/kernels/fg_conv.hlsl b/clean/dx12/kernels/fg_conv.hlsl new file mode 100644 index 00000000..30cc25aa --- /dev/null +++ b/clean/dx12/kernels/fg_conv.hlsl @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 convolution (issue #96): fg_conv_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip), +// byte for byte. The arithmetic is the shared integer code of clean/fg/kernels/fg_exact16.h (exact float16 chunk +// accumulation, one nearest-even rounding per chunk, the leaky ReLU), so both back ends run the same operations; only +// memory access differs (byte offsets into the arena here). +// +// One output element per lane: idx = (o * H + y) * W + x of [cout][H][W] float16; x [cin][H][W] float16; w natural +// [cout][cin][k][k] float16; b [cout] float16. Stride 1, zero border; chunk order: 0 ascending (1 x 1), 1 channel block, +// kx, ky; 2 kx, channel block, ky. act: 0 none, 1 leaky ReLU. Not part of Model M's or Model L's kernel set (fg_ kernels +// are built only when --only names them). +// +// @wg-kernel fg_conv_f16 co=fg_net1.co block=256 wave=32 args=p,p,p,p,u,u,u,u,u,u,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint x, w, b, y; // pointers (arena byte offsets) + int cin, cout, H, W, k, order, act; +WG_ARGS_END + +WG_KERNEL(fg_conv_f16) +{ + WG_SETUP(); + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= cout * H * W) return; + const int o = idx / (H * W); + const int rem = idx - o * (H * W); + const int yy = rem / W; + const int xx = rem - yy * W; + const int p = k / 2; + const int nb = cin / 16; + const int taps = k * k; + uint acc = wg_ld16(b, (uint)o * 2u); + [loop] for (int n = 0; n < nb * taps; ++n) { + int cb, kx, ky; + if (order == 0) { + cb = n; kx = 0; ky = 0; + } else if (order == 1) { + cb = n / taps; + const int t = n - cb * taps; + kx = t / k; + ky = t - kx * k; + } else { + kx = n / (nb * k); + const int r = n - kx * (nb * k); + cb = r / k; + ky = r - cb * k; + } + const int sy = yy + ky - p; + const int sx = xx + kx - p; + const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W; + FgAcc s = fg_acc_begin(acc); + [loop] for (int i = 0; i < 16; ++i) { + const int c = cb * 16 + i; + const uint wv = wg_ld16(w, (uint)(((o * cin + c) * k + ky) * k + kx) * 2u); + uint xv = 0u; // outside the grid: 0 (0 * inf is NaN) + if (inside) xv = wg_ld16(x, (uint)((c * H + sy) * W + sx) * 2u); + s = fg_acc_add(s, wv, xv); + } + acc = fg_acc_end(s); + } + if (act == 1) acc = fg_leaky16(acc); + wg_st16(y, (uint)idx * 2u, acc); +} diff --git a/clean/dx12/tests/test_kernel_sets.py b/clean/dx12/tests/test_kernel_sets.py index bdb87aa8..a9925bf1 100644 --- a/clean/dx12/tests/test_kernel_sets.py +++ b/clean/dx12/tests/test_kernel_sets.py @@ -48,6 +48,19 @@ def test_model_l_adds_only_model_l_kernels(self): self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)]) + def test_frame_gen_kernels_are_in_no_set(self): + """Frame generation's ports (#96, co=fg_*.co or named fg_*) are declared but enter neither set: only --only + builds them, so the Model M and Model L builds stay as they are.""" + decls = [] + for fn in sorted(os.listdir(B.KERNEL_DIR)): + if fn.endswith(".hlsl"): + decls += B.kernel_decls(os.path.join(B.KERNEL_DIR, fn)) + fg = [(d["name"], d["co"]) for d in decls if B.is_frame_gen(d)] + self.assertTrue(fg, "no frame-generation kernel declared") + full = set(B.kernel_set(True)) + self.assertFalse([k for k in fg if k in full]) + + if __name__ == "__main__": if "--write" in sys.argv: with open(SETS, "w", newline="\n") as f: diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index 1af55c55..e2cf3f3c 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -95,6 +95,20 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c - Not part of net1 and still open: what the 10 input values are made of (images A-D, the warps) and the input grid size relative to the output. +## GPU kernels (`kernels/`, code object `fg_net1.co`; D3D12 ports `clean/dx12/kernels/fg_*.hlsl`) + +- Groundwork: no runtime launches them yet. Tensors float16 channel-first `[c][h][w]`, weights natural + `[out][in][k][k]` float16 (the map's `W`), biases `[out]` float16; a 1-D grid of 256-lane workgroups, one output + element per lane. +- `fg_conv_f16(x, w, b, y, cin, cout, H, W, k, order, act)`: one net1 convolution exactly as `net1.conv16`: stride 1, + zero border, float16 accumulator from the bias, 16-term chunks each summed exactly and rounded once, chunk order + `order` (0 ascending, 1 channel block / kx / ky, 2 kx / channel block / ky), `act` 0 none or 1 leaky ReLU. cin is a + multiple of 16. +- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN + results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's + and Model L's kernel sets and DXILs do not change. +- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference. + ## Configuration (`reference/config.py`) - No configuration value changes the networks or their weights. diff --git a/clean/fg/README.md b/clean/fg/README.md index 871febfd..5441ced2 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -17,6 +17,9 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `reference/fp16acc.py` | exact float16 chunk accumulation: RN16(acc + 16 products) with integer arithmetic | | `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals | | `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only | +| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction | +| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none); D3D12 port `clean/dx12/kernels/fg_conv.hlsl` | +| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | | `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` | diff --git a/clean/fg/kernels/fg_exact16.h b/clean/fg/kernels/fg_exact16.h new file mode 100644 index 00000000..0e7af661 --- /dev/null +++ b/clean/fg/kernels/fg_exact16.h @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT +// Frame generation (issue #96): exact float16 arithmetic in integers, shared by the HIP kernels (clean/fg/kernels/*.hip) +// and their D3D12 ports (clean/dx12/kernels/fg_*.hlsl), so both back ends compute the same bits by construction. +// +// The same method as the CPU reference clean/fg/reference/fp16acc.py: a finite float16 value is m * 2^-24 with an integer +// |m| < 2^40; m splits into a high and a low 20-bit part, so a 16-term dot product is exact in three 64-bit sums +// (high * high, high * low + low * high, low * low). The accumulator acc' = RN16(acc + sum of a_i * b_i) is rounded once, +// to nearest with ties to even, by integer arithmetic only. inf and NaN follow IEEE (any NaN, 0 * inf or inf - inf gives +// NaN, written as 0x7E00; otherwise an infinite term gives that infinity). +// +// The including file defines: +// FG_FN function qualifier (HIP: WG_FN; HLSL: nothing) +// FG_I64 signed 64-bit integer (HIP: long long; HLSL: int64_t) +// FG_FBH32(v) index of the highest set bit of a non-zero 32-bit unsigned value +#ifndef FG_EXACT16_H +#define FG_EXACT16_H + +#define FG_NAN16 0x7E00u +#define FG_LOW20 1048575 + +struct FgAcc { + FG_I64 hh; // sum of high * high (units 2^-8) + FG_I64 mid; // sum of high * low + low * high, plus 16 * m(acc) (units 2^-28) + FG_I64 ll; // sum of low * low (units 2^-48), never negative + uint nan_; // 1: the result is NaN + uint pinf; // 1: a +inf term + uint ninf; // 1: a -inf term +}; + +FG_FN uint fg_is_nan16(uint h) { return ((h & 0x7C00u) == 0x7C00u && (h & 0x3FFu) != 0u) ? 1u : 0u; } +FG_FN uint fg_is_inf16(uint h) { return ((h & 0x7FFFu) == 0x7C00u) ? 1u : 0u; } + +// floor(log2(v)) of a 64-bit value v > 0 +FG_FN uint fg_log2_64(FG_I64 v) { + const uint hi = (uint)(v >> 32); + const uint lo = (uint)v; // the low 32 bits + return hi != 0u ? 32u + FG_FBH32(hi) : FG_FBH32(lo); +} + +// a finite float16 (bits) -> m with value = m * 2^-24 +FG_FN FG_I64 fg_m16(uint h) { + const uint e = (h >> 10) & 31u; + const uint f = h & 1023u; + FG_I64 m = (FG_I64)f; + if (e != 0u) m = (FG_I64)(f | 1024u) * ((FG_I64)1 << (e - 1u)); + return (h & 0x8000u) != 0u ? -m : m; +} + +FG_FN FgAcc fg_acc_begin(uint acc) { + FgAcc s; + s.hh = 0; + s.mid = 0; + s.ll = 0; + s.nan_ = fg_is_nan16(acc); + s.pinf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) == 0u) ? 1u : 0u; + s.ninf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) != 0u) ? 1u : 0u; + if ((acc & 0x7C00u) != 0x7C00u) s.mid = fg_m16(acc) * 16; + return s; +} + +// s + a * b (a, b float16 bits) +FG_FN FgAcc fg_acc_add(FgAcc s, uint a, uint b) { + if (fg_is_nan16(a) != 0u || fg_is_nan16(b) != 0u) { + s.nan_ = 1u; + } else if (fg_is_inf16(a) != 0u || fg_is_inf16(b) != 0u) { + if ((a & 0x7FFFu) == 0u || (b & 0x7FFFu) == 0u) { + s.nan_ = 1u; // 0 * inf + } else if (((a ^ b) & 0x8000u) != 0u) { + s.ninf = 1u; + } else { + s.pinf = 1u; + } + } else { + const FG_I64 ma = fg_m16(a); + const FG_I64 mb = fg_m16(b); + const FG_I64 ah = ma >> 20, al = ma & FG_LOW20; + const FG_I64 bh = mb >> 20, bl = mb & FG_LOW20; + s.hh += ah * bh; + s.mid += ah * bl + al * bh; + s.ll += al * bl; + } + return s; +} + +// RN16 of the exact value (hh * 2^40 + mid * 2^20 + ll) * 2^-48, ll >= 0, as float16 bits +FG_FN uint fg_round16(FG_I64 hh, FG_I64 mid, FG_I64 ll) { + const FG_I64 lim = (FG_I64)1 << 30; + const uint big = (hh > lim || hh < -lim) ? 1u : 0u; // |value| >= 2^22: infinite + const FG_I64 hhc = hh > lim ? lim : (hh < -lim ? -lim : hh); + // value * 2^25 = hh * 2^17 + mid2 / 8 + llr / 2^23, with mid2 = mid + (ll >> 20) and 0 <= llr < 2^20 + const FG_I64 l1 = ll >> 20; + const FG_I64 llr = ll & FG_LOW20; + const FG_I64 mid2 = mid + l1; + const FG_I64 m1 = mid2 >> 3; // floor + const FG_I64 midr = mid2 & 7; + const FG_I64 q = hhc * 131072 + m1; // floor of value * 2^25 + const uint sticky = (midr != 0 || llr != 0) ? 1u : 0u; + const uint neg = q < 0 ? 1u : 0u; + FG_I64 mag = q; + if (neg != 0u) mag = sticky != 0u ? -q - 1 : -q; + // quantum in units of 2^-25: 2 below 2^-14 (subnormal), else 2^(e - 10) in the binade [2^e, 2^(e + 1)) units + const uint e = fg_log2_64(mag > 0 ? mag : 1); + const uint qexp = mag < 2048 ? 1u : e - 10u; + const FG_I64 quantum = (FG_I64)1 << qexp; + const FG_I64 r = mag & (quantum - 1); + const FG_I64 down = mag - r; + const FG_I64 half = quantum >> 1; + const uint odd = (uint)((down >> qexp) & 1); + const uint up = (r > half || (r == half && (sticky != 0u || odd != 0u))) ? 1u : 0u; + const FG_I64 res = down + (up != 0u ? quantum : 0); // the rounded magnitude, units 2^-25 + const uint sign = big != 0u ? (hh < 0 ? 1u : 0u) : neg; + uint bits; + if (big != 0u || res >= ((FG_I64)65536 << 25)) { + bits = 0x7C00u; + } else if (res < 2048) { + bits = (uint)(res >> 1); // subnormal (or zero) + } else { + const uint e2 = fg_log2_64(res); + bits = ((e2 - 10u) << 10) | ((uint)(res >> (e2 - 10u)) & 1023u); + } + return bits | (sign << 15); +} + +FG_FN uint fg_acc_end(FgAcc s) { + if (s.nan_ != 0u || (s.pinf != 0u && s.ninf != 0u)) return FG_NAN16; + if (s.pinf != 0u) return 0x7C00u; + if (s.ninf != 0u) return 0xFC00u; + return fg_round16(s.hh, s.mid, s.ll); +} + +// the order of a float16 value for max (-0 and +0 are equal); not for NaN +FG_FN int fg_key16(uint h) { return (h & 0x8000u) != 0u ? -(int)(h & 0x7FFFu) : (int)(h & 0x7FFFu); } + +// net1's activation: y = max(x, h(s * x)) with the delivered slope s (float16 0x211E); NaN stays NaN +#define FG_LEAKY_SLOPE16 0x211Eu +FG_FN uint fg_leaky16(uint x) { + if (fg_is_nan16(x) != 0u) return x; + const uint t = fg_acc_end(fg_acc_add(fg_acc_begin(0u), FG_LEAKY_SLOPE16, x)); + return fg_key16(t) > fg_key16(x) ? t : x; +} + +#endif diff --git a/clean/fg/kernels/fg_net1.hip b/clean/fg/kernels/fg_net1.hip new file mode 100644 index 00000000..8701536b --- /dev/null +++ b/clean/fg/kernels/fg_net1.hip @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// Frame generation (issue #96): HIP kernels of net1's operations (clean/fg/reference/net1.py), bit for bit against the +// CPU reference. D3D12 ports: clean/dx12/kernels/fg_*.hlsl. Tensors are float16, channel-first [c][h][w]; weights are +// natural [out][in][k][k] float16; biases [out] float16. 1-D grid of 256-lane workgroups, one output element per lane. +#include "wg_platform.h" +#include "wg_mem.h" + +typedef unsigned int uint; +typedef unsigned short u16; +#define FG_FN WG_FN +#define FG_I64 long long +#define FG_FBH32(v) (31u - (uint)__builtin_clz(v)) +#include "fg_exact16.h" + +// fg_conv_f16: one convolution of net1. Stride 1, zero border (k = 3) or none (k = 1), cross-correlation; the float16 +// accumulator starts at the bias and takes 16-term chunks (16 input channels of one tap), each summed exactly and added +// with one float16 rounding, in the chunk order `order`: 0 ascending (1 x 1: channel blocks in order), 1 channel block +// outer, then kx, then ky; 2 kx outer, then channel block, then ky. act: 0 none, 1 leaky ReLU (slope 0x211E). +// cin must be a multiple of 16. Output element idx = (o * H + y) * W + x. +WG_KERNEL_N(fg_conv_f16, 256)(const u16* __restrict__ x, const u16* __restrict__ w, const u16* __restrict__ b, + u16* __restrict__ y, int cin, int cout, int H, int W, int k, int order, int act) { + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= cout * H * W) return; + const int o = idx / (H * W); + const int rem = idx - o * (H * W); + const int yy = rem / W; + const int xx = rem - yy * W; + const int p = k / 2; + const int nb = cin / 16; + const int taps = k * k; + uint acc = b[o]; + for (int n = 0; n < nb * taps; ++n) { + int cb, kx, ky; + if (order == 0) { + cb = n; kx = 0; ky = 0; + } else if (order == 1) { + cb = n / taps; + const int t = n - cb * taps; + kx = t / k; + ky = t - kx * k; + } else { + kx = n / (nb * k); + const int r = n - kx * (nb * k); + cb = r / k; + ky = r - cb * k; + } + const int sy = yy + ky - p, sx = xx + kx - p; + const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W; + FgAcc s = fg_acc_begin(acc); + for (int i = 0; i < 16; ++i) { + const int c = cb * 16 + i; + const uint wv = w[((o * cin + c) * k + ky) * k + kx]; + const uint xv = inside ? (uint)x[(c * H + sy) * W + sx] : 0u; // outside the grid: 0 (0 * inf is NaN) + s = fg_acc_add(s, wv, xv); + } + acc = fg_acc_end(s); + } + if (act == 1) acc = fg_leaky16(acc); + y[idx] = (u16)acc; +} diff --git a/clean/fg/runner/__init__.py b/clean/fg/runner/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/fg/runner/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py new file mode 100644 index 00000000..76cf3c59 --- /dev/null +++ b/clean/fg/runner/fg_d3d12_check.py @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (issue #96): net1's GPU kernels (clean/fg/kernels/fg_net1.hip) against their D3D12 ports +(clean/dx12/kernels/fg_*.hlsl) byte for byte, and against the CPU reference (clean/fg/reference/net1.py) bit for bit. + + python clean/fg/runner/fg_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/fg/runner/fg_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] [--kernels fg_conv_f16] [--json out.json] + +--prepare (CPU) builds into : fg_net1.co (ROCm clang), the D3D12 ports (clean/dx12/build_dx12.py --only; the +pinned DXC through WG_TOOLCHAINS, BUILDING.md) and kernel_replay.exe (clean/dx12/tests/kernel_replay.cpp). +--run (GPU, inside one GPU slot) runs every case on HIP in this process and the same inputs through kernel_replay on +the native D3D12 device in a child process, then: +- compares the RAW downloaded bytes of both back ends; each output buffer starts filled with its back end's own + sentinel (HIP 0xA5, D3D12 0x5A) and has 64 slack bytes after it that neither may write, so an unwritten byte or a + write past the end differs; +- a negative control (the first case with 4 or more workgroups, again on D3D12 with half of them) must be caught; +- compares the HIP output with the CPU reference, value for value (float16 bits; NaN equals NaN, the reference's NaN + payload is not specified). +Cases: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several sizes, and +special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights. Output: counts only, no timing. +""" +import argparse +import ctypes +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.fg.reference import net1 as N # noqa: E402 + +HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A +SLACK = 64 +CO = "fg_net1.co" +ORDER_CODE = {"ascending": 0, "block_kx_ky": 1, "kx_block_ky": 2} +GRIDS = [(5, 7), (1, 1), (8, 8), (3, 17), (16, 12)] + + +def _layers(): + """(layer, cin, cout, k, order, act) of net1's 17 convolutions, from the wiring and the layout.""" + from clean.fg import layout as L + by = {l.name: l for l in L.NET1} + out = [] + for s in N.NET1_WIRING: + if s["op"] == "conv": + l = by[s["layer"]] + out.append((l.name, l.cin, l.out, 3 if l.taps == 9 else 1, s["chunks"], s["act"])) + return out + + +def _f16(a): + return np.asarray(a, dtype=np.float16) + + +SPECIALS = np.array([np.nan, np.inf, -np.inf, 65504.0, -65504.0, 2.0 ** -24, -2.0 ** -24, 2.0 ** -20, -0.0, 0.0], + dtype=np.float16) + + +def conv_cases(kernel, rng, specials=False): + cases = [] + for li, (name, cin, cout, k, order, act) in enumerate(_layers()): + for gi, (H, W) in enumerate(GRIDS): + if cin * cout > 32768 and H * W > 64: # the widest layers on the smaller grids only + continue + x = _f16(rng.uniform(-2, 2, (cin, H, W))) + w = _f16(rng.uniform(-0.2, 0.2, (cout, cin, k, k))) + b = _f16(rng.uniform(-0.5, 0.5, cout)) + cases.append(_conv_case(kernel, x, w, b, k, order, act, f"{name} {H}x{W}")) + if specials: + for k, order in ((3, "block_kx_ky"), (3, "kx_block_ky"), (1, "ascending")): + for act in ("leaky", "none"): + cin, cout, H, W = 32, 8, 6, 9 + x = _f16(rng.uniform(-2, 2, (cin, H, W))) + w = _f16(rng.uniform(-1, 1, (cout, cin, k, k))) + b = _f16(rng.uniform(-1, 1, cout)) + fx = rng.random(x.shape) < 0.04 + x[fx] = rng.choice(SPECIALS, int(fx.sum())) + fw = rng.random(w.shape) < 0.02 + w[fw] = rng.choice(SPECIALS, int(fw.sum())) + b[0], b[1], b[2] = np.inf, -np.inf, np.nan + big = _f16(rng.uniform(100, 255, (cout, cin, k, k))) # sums that overflow to +-inf + w[5] = big[5] + w[6] = -big[6] + cases.append(_conv_case(kernel, x, w, b, k, order, act, f"specials k{k} {order} {act}")) + return cases + + +def _conv_case(kernel, x, w, b, k, order, act, label): + cin, H, W = x.shape + cout = w.shape[0] + n = cout * H * W + return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label, + "args": [("buf", x.tobytes()), ("buf", w.tobytes()), ("buf", b.tobytes()), ("out", 2 * n + SLACK), + ("u32", cin), ("u32", cout), ("u32", H), ("u32", W), ("u32", k), ("u32", ORDER_CODE[order]), + ("u32", 1 if act == "leaky" else 0)], + "defined": 2 * n, + "cpu": lambda: N.conv16(x.astype(np.float64), w.astype(np.float64).transpose(0, 2, 3, 1), + b.astype(np.float64), chunks=order, act=act)} + + +KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True)} + + +# ---- prepare -------------------------------------------------------------------------------------------------------- + +def prepare(build, rocm, kernels): + from clean.l.runner.l_d3d12_check import prepare_replay + from clean.runner import build as B + from clean.runner.hip import clean_process_environment + os.makedirs(build, exist_ok=True) + clean_process_environment(rocm) + clang = B.find_clang(rocm) + B.run(B.device_command(clang, rocm, os.path.join(ROOT, "clean", "fg", "kernels", "fg_net1.hip"), + os.path.join(build, CO)) + ["-I", os.path.join(ROOT, "clean", "kernels")]) + return prepare_replay(build, rocm, kernels) + + +# ---- run ------------------------------------------------------------------------------------------------------------ + +def hip_run(h, mod, case): + args, owned, outs = [], [], [] + for a in case["args"]: + if a[0] == "buf": + p = h.malloc(max(len(a[1]), 4)) + h.upload(p, a[1]) + owned.append(p) + args.append(ctypes.c_void_p(p)) + elif a[0] == "out": + p = h.malloc(a[1]) + h.memset(p, HIP_SENTINEL, a[1]) + owned.append(p) + outs.append((p, a[1])) + args.append(ctypes.c_void_p(p)) + elif a[0] == "u32": + args.append(ctypes.c_int(a[1])) + try: + h.launch(h.get_function(mod, case["kernel"]), case["grid"], case["block"], args) + h.sync() + return [bytes(h.download(p, n)) for p, n in outs] + finally: + for p in owned: + h.free(p) + + +def replay_lines(name, case, cdir, grid=None, prefix=""): + lines = [f"case {prefix}{name}", f"module {CO}", f"kernel {case['kernel']} {grid or case['grid']} {case['block']}"] + oi = 0 + for i, a in enumerate(case["args"]): + if a[0] == "buf": + f = f"{name}.a{i}" + if not prefix: + with open(os.path.join(cdir, f), "wb") as fh: + fh.write(a[1]) + lines.append(f"arg buf {f}") + elif a[0] == "out": + lines.append(f"arg out {a[1]} {prefix}{name}.o{oi} {D3D12_SENTINEL}") + oi += 1 + elif a[0] == "u32": + lines.append(f"arg u32 {a[1]}") + return lines + [""] + + +def compare_raw(got, want, defined): + """Raw bytes: identical everywhere; in the slack after `defined` bytes each back end keeps its own sentinel.""" + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want, np.uint8) + if a.size != b.size: + return {"bytes": int(b.size), "differ": -1, "slack_ok": False} + return {"bytes": int(b.size), "differ": int(np.sum(a[:defined] != b[:defined])), + "slack_ok": bool(np.all(a[defined:] == D3D12_SENTINEL) and np.all(b[defined:] == HIP_SENTINEL))} + + +def compare_cpu(hip_bytes, ref): + """HIP float16 output against the CPU reference: bits equal, or both NaN.""" + g = np.frombuffer(hip_bytes, np.uint16) + r = np.asarray(ref, np.float64).astype(np.float16).reshape(-1).view(np.uint16) + gn = ((g & 0x7C00) == 0x7C00) & ((g & 0x3FF) != 0) + rn = ((r & 0x7C00) == 0x7C00) & ((r & 0x3FF) != 0) + same = (g == r) | (gn & rn) + return {"values": int(g.size), "differ": int(np.sum(~same)), "nan": int(np.sum(gn)), + "inf": int(np.sum((g & 0x7FFF) == 0x7C00))} + + +def run(build, rocm, seeds, kernels): + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + h = Hip(rocm) + with open(os.path.join(build, CO), "rb") as f: + mod = h.load_module(f.read()) + cdir = os.path.join(build, "replay-fg") + os.makedirs(cdir, exist_ok=True) + lines, hip_out, neg = [], {}, None + for kernel in kernels: + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, case in enumerate(KINDS[kernel](kernel, rng)): + name = f"{kernel}_s{si}_{ci}" + outs = hip_run(h, mod, case) + hip_out[name] = (kernel, case, outs) + lines += replay_lines(name, case, cdir) + if neg is None and case["grid"] >= 4: + neg = name + lines += replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_") + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() == "HIP_PLATFORM"} + env["WG_D3D12_ARENA_MB"] = "512" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=150, creationflags=0x08000000) + res = {"replay_exit": r.returncode, "replay_tail": r.stdout[-400:] + r.stderr[-300:], "cases": []} + + def read(path): + return open(path, "rb").read() if os.path.exists(path) else b"" + + ok = r.returncode == 0 + for name, (kernel, case, outs) in hip_out.items(): + raw = compare_raw(read(os.path.join(cdir, f"{name}.o0")), outs[0], case["defined"]) + cpu = compare_cpu(outs[0][:case["defined"]], case["cpu"]()) + # a byte either back end leaves unwritten differs (0xA5 against 0x5A), so differ == 0 also proves every byte + # was written by both; equal bytes that happen to be 0xA5 are ordinary data + good = raw["differ"] == 0 and raw["slack_ok"] and cpu["differ"] == 0 + ok = ok and good + res["cases"].append({"case": name, "kernel": kernel, "label": case["label"], "raw": raw, "cpu": cpu, + "ok": good}) + if neg: + kernel, case, outs = hip_out[neg] + nraw = compare_raw(read(os.path.join(cdir, f"negative_{neg}.o0")), outs[0], case["defined"]) + caught = nraw["differ"] != 0 or not nraw["slack_ok"] + res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": nraw["differ"]} + ok = ok and caught + else: + res["negative_control"] = None + ok = False + res["summary"] = {"cases": len(res["cases"]), "cases_ok": sum(c["ok"] for c in res["cases"]), + "values": sum(c["cpu"]["values"] for c in res["cases"]), + "nan_values": sum(c["cpu"]["nan"] for c in res["cases"]), + "inf_values": sum(c["cpu"]["inf"] for c in res["cases"]), "pass": bool(ok)} + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--kernels", default=",".join(KINDS)) + ap.add_argument("--json", default=None) + a = ap.parse_args(argv) + kernels = a.kernels.split(",") + if a.prepare: + prepare(a.build, a.rocm, kernels) + if a.run: + res = run(a.build, a.rocm, list(range(1, a.seeds + 1)), kernels) + text = json.dumps(res, indent=1) + if a.json: + with open(a.json, "w", encoding="utf-8", newline="\n") as f: + f.write(text + "\n") + print(json.dumps(res["summary"]), "negative control:", json.dumps(res["negative_control"])) + for c in res["cases"]: + if not c["ok"]: + print("FAIL", c["case"], c["label"], json.dumps(c["raw"]), json.dumps(c["cpu"])) + print("replay exit", res["replay_exit"]) + return 0 if res["summary"]["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main())