From da9e18d978b24c3fe70d9155d881c9ee0453e0d3 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:39:57 -0500 Subject: [PATCH] Frame generation: net1 fast path, tier B on the matrix instructions (HIP and D3D12 byte-identical) - kernels/fg_net1_mma.hip, clean/dx12/kernels/fg_mconv.hlsl: fg_mconv1/2/4, one v_wmma_f32_16x16x16_f16 per spec chunk in the layer's chunk order, the float16 accumulator from the bias as C, rounded to float16 after every chunk; exact leaky ReLU and fused 2 x 2 pool in the epilogue; NHWC float16 - fg_upscale_hwc_f16 (HIP and fg_upscale_hwc.hlsl): the exact upscale on NHWC tensors - runner/fg_net1_mma.py: net1 in 27 launches; HIP == D3D12 for every buffer (sentinels, slack, half-grid negative control per kernel, special values); report against the CPU reference; timing (information only) - records/net1-tierb-2026-11-14.json: gate results, report (counts only) and timing - Exact kernels unchanged (validation mode); fg_ kernels still built only through --only Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/kernels/fg_mconv.hlsl | 104 +++++ clean/dx12/kernels/fg_upscale_hwc.hlsl | 59 +++ clean/fg/FORMATS.md | 20 +- clean/fg/README.md | 4 + clean/fg/kernels/fg_net1_mma.hip | 175 ++++++++ clean/fg/records/net1-tierb-2026-11-14.json | 1 + clean/fg/runner/fg_net1_mma.py | 421 ++++++++++++++++++++ clean/fg/tests/test_fg_net1_mma_plan.py | 72 ++++ 8 files changed, 855 insertions(+), 1 deletion(-) create mode 100644 clean/dx12/kernels/fg_mconv.hlsl create mode 100644 clean/dx12/kernels/fg_upscale_hwc.hlsl create mode 100644 clean/fg/kernels/fg_net1_mma.hip create mode 100644 clean/fg/records/net1-tierb-2026-11-14.json create mode 100644 clean/fg/runner/fg_net1_mma.py create mode 100644 clean/fg/tests/test_fg_net1_mma_plan.py diff --git a/clean/dx12/kernels/fg_mconv.hlsl b/clean/dx12/kernels/fg_mconv.hlsl new file mode 100644 index 00000000..3e60ae08 --- /dev/null +++ b/clean/dx12/kernels/fg_mconv.hlsl @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's matrix convolution (issue #96; tier B of the net1 fast path): fg_mconv of +// fg_net1_mma.co (clean/fg/kernels/fg_net1_mma.hip), byte-identical to HIP: the same fragments and chunk order, +// v_wmma_f32_16x16x16_f16 through the driver channel (wg_wmma_f16) with the float16 accumulator as C, its result rounded +// to float16 after every chunk (wg_f2h = v_cvt_f16_f32, nearest even), the exact integer leaky ReLU and pool +// (fg_exact16.h). Driver rules (clean/dx12/README.md): every load of a chunk before its first matrix call; branchless +// selects; the chunk loop bound through an opaque zero (no forced unrolling); store pointers through wg_uniform(). +// Built only when --only names the kernels (fg_ kernels are in no model's set). +// +// @wg-kernel fg_mconv1 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=1 +// @wg-kernel fg_mconv2 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=2 +// @wg-kernel fg_mconv4 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=4 +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +#ifndef FG_MT +#define FG_MT 4 +#endif + +WG_ARGS_BEGIN + uint x, wf, b, y, p; // pointers (arena byte offsets) + int cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y; +WG_ARGS_END + +uint fg_opq0() { return wg_grid_ >> 31; } // an opaque zero (the grid size is below 2^31) + +uint4 fg_pack8(uint r[8]) { return uint4(r[0] | (r[1] << 16), r[2] | (r[3] << 16), r[4] | (r[5] << 16), r[6] | (r[7] << 16)); } + +WG_KERNEL(fg_mconv) +{ + WG_SETUP(); + const uint lane = wg_lane_id() & 31u, wave = wg_lane_id() >> 5; + const int blk = (int)wg_block_id(); + const int tx = blk % tiles_x, rest = blk / tiles_x; + const int ty = rest % tiles_y, mg = rest / tiles_y; + const int n = (int)(lane & 15u), hh = (int)(lane >> 4); + const int px = tx * 16 + n; + const int py0 = ty * 16 + 2 * (int)wave; + const int pad = k / 2, nb = cin / 16, taps = k * k, nq = nb * taps; + const uint py = (uint)wg_uniform((int)y), pp = (uint)wg_uniform((int)p); + + v8f a32[FG_MT][2]; + [unroll] for (int mt = 0; mt < FG_MT; ++mt) { + const int o0 = (mg * FG_MT + mt) * 16 + 8 * hh; + [unroll] for (uint v = 0; v < 8; ++v) { + const float bv = wg_h2f(wg_ld16(b, (uint)(o0 + (int)v) * 2u)); + a32[mt][0].v[v] = bv; + a32[mt][1].v[v] = bv; + } + } + const int nq_loop = nq + (int)fg_opq0(); + for (int q = 0; q < nq_loop; ++q) { + // the chunk's (channel block, ky, kx) in the layer's order, branchless + const int t1 = q % taps, cb1 = q / taps; + const int kx2 = q / (nb * k), r2 = q % (nb * k); + const int cb = select(order == 0, q, select(order == 1, cb1, r2 / k)); + const int kx = select(order == 0, 0, select(order == 1, t1 / k, kx2)); + const int ky = select(order == 0, 0, select(order == 1, t1 % k, r2 % k)); + uint4 fb[2], fa[FG_MT]; + const int sx = px + kx - pad; + [unroll] for (int nt = 0; nt < 2; ++nt) { + const int sy = py0 + nt + ky - pad; + const bool inb = and(and(sx >= 0, sx < W), and(sy >= 0, sy < H)); + const uint e = select(inb, (uint)((sy * W + sx) * cin + cb * 16 + 8 * hh), 0u); + const uint4 u = wg_ld128(x, e * 2u); + fb[nt] = uint4(select(inb, u.x, 0u), select(inb, u.y, 0u), select(inb, u.z, 0u), select(inb, u.w, 0u)); + } + [unroll] for (int mt2 = 0; mt2 < FG_MT; ++mt2) + fa[mt2] = wg_ld128(wf, (uint)((((mg * FG_MT + mt2) * nq + q) * 32 + (int)lane) * 16)); + [unroll] for (int mt3 = 0; mt3 < FG_MT; ++mt3) + [unroll] for (int nt2 = 0; nt2 < 2; ++nt2) { + const v8f d = wg_wmma_f16(fa[mt3], fb[nt2], a32[mt3][nt2]); + [unroll] for (uint v3 = 0; v3 < 8; ++v3) a32[mt3][nt2].v[v3] = wg_h2f(wg_f2h(d.v[v3])); + } + } + [unroll] for (int mt4 = 0; mt4 < FG_MT; ++mt4) { + const int o0 = (mg * FG_MT + mt4) * 16 + 8 * hh; + uint r0[8], r1[8]; + [unroll] for (uint v4 = 0; v4 < 8; ++v4) { + const uint h0 = wg_f2h(a32[mt4][0].v[v4]), h1 = wg_f2h(a32[mt4][1].v[v4]); + r0[v4] = select(act == 1, fg_leaky16(h0), h0); + r1[v4] = select(act == 1, fg_leaky16(h1), h1); + } + if (px < W && py0 < H) wg_st128(py, (uint)((py0 * W + px) * cout + o0) * 2u, fg_pack8(r0)); + if (px < W && py0 + 1 < H) wg_st128(py, (uint)(((py0 + 1) * W + px) * cout + o0) * 2u, fg_pack8(r1)); + if (pool == 1) { + uint m[8]; + [unroll] for (uint v5 = 0; v5 < 8; ++v5) { + const uint x01 = wg_xor_u(r0[v5], 1u), x11 = wg_xor_u(r1[v5], 1u); + uint t = r0[v5]; + t = fg_pool_step16(t, x01); + t = fg_pool_step16(t, r1[v5]); + t = fg_pool_step16(t, x11); + m[v5] = t; + } + const int Ho = H / 2, Wo = W / 2, ox = px / 2, oy = py0 / 2; + if ((n & 1) == 0 && ox < Wo && oy < Ho) wg_st128(pp, (uint)((oy * Wo + ox) * cout + o0) * 2u, fg_pack8(m)); + } + } +} diff --git a/clean/dx12/kernels/fg_upscale_hwc.hlsl b/clean/dx12/kernels/fg_upscale_hwc.hlsl new file mode 100644 index 00000000..a2fbdc88 --- /dev/null +++ b/clean/dx12/kernels/fg_upscale_hwc.hlsl @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 upscale on NHWC tensors (issue #96): fg_upscale_hwc_f16 of fg_net1_mma.co +// (clean/fg/kernels/fg_net1_mma.hip), byte for byte; the arithmetic of fg_upscale.hlsl (align corners, float16 steps, +// NaN written as 0x7E00), element (y, x, c) at (y * W + x) * C + c. Built only when --only names it. +// +// @wg-kernel fg_upscale_hwc_f16 co=fg_net1_mma.co block=256 wave=32 args=p,p,u,u,u,u,u,u,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint x, y; // pointers (arena byte offsets) + int C, hi, wi, ho, wo; + uint rx_bits, ry_bits; +WG_ARGS_END + +struct FgTap { int i0; int i1; float w0; float w1; }; + +FgTap fg_axis(float r, int o, int n_in) +{ + FgTap t; + const float s = r * (float)o; + t.i0 = (int)s; + t.i1 = (int)(s + 1.0f); + if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; } + t.w1 = wg_h2f(wg_f2h(s - (float)t.i0)); + t.w0 = wg_h2f(wg_f2h(((float)t.i0 - s) + 1.0f)); + return t; +} + +float fg_lerp16(float wa, float a, float wb, float b) +{ + return wg_h2f(wg_f2h(wg_h2f(wg_f2h(wa * a)) + wg_h2f(wg_f2h(wb * b)))); +} + +WG_KERNEL(fg_upscale_hwc_f16) +{ + WG_SETUP(); + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * ho * wo) return; + const int c = idx % C; + const int pix = idx / C; + const int oy = pix / wo; + const int ox = pix - oy * wo; + const FgTap tx = fg_axis(asfloat(rx_bits), ox, wi); + const FgTap ty = fg_axis(asfloat(ry_bits), oy, hi); + const float p00 = wg_h2f(wg_ld16(x, (uint)((ty.i0 * wi + tx.i0) * C + c) * 2u)); + const float p01 = wg_h2f(wg_ld16(x, (uint)((ty.i0 * wi + tx.i1) * C + c) * 2u)); + const float p10 = wg_h2f(wg_ld16(x, (uint)((ty.i1 * wi + tx.i0) * C + c) * 2u)); + const float p11 = wg_h2f(wg_ld16(x, (uint)((ty.i1 * wi + tx.i1) * C + c) * 2u)); + const float top = fg_lerp16(tx.w0, p00, tx.w1, p01); + const float bot = fg_lerp16(tx.w0, p10, tx.w1, p11); + uint outv = wg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot)); + if (fg_is_nan16(outv) != 0u) outv = FG_NAN16; + wg_st16(y, (uint)idx * 2u, outv); +} diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index d2f7c878..19ea0f0b 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -120,7 +120,25 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's and Model L's kernel sets and DXILs do not change. - `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference; - `runner/fg_net1_gpu.py` the whole network, every step output. + `runner/fg_net1_gpu.py` the whole network, every step output. These exact kernels stay as the validation mode. + +## Fast path, tier B (`kernels/fg_net1_mma.hip`, code object `fg_net1_mma.co`; D3D12 `fg_mconv.hlsl`, `fg_upscale_hwc.hlsl`) + +- Tensors NHWC float16 (element (y, x, c) at (y W + x) C + c), so 8 consecutive channels of a pixel are one 16-byte + fragment load or store. Weights as WMMA A fragments in chunk order: `[cout / 16][chunks][32 lanes][8]` float16, lane + l holding output channel 16 mt + l % 16 and input channels 16 cb + 8 (l / 16) + 0..7 of the chunk's tap + (`runner/fg_net1_mma.py` `swizzle`). +- `fg_mconv{1,2,4}(x, wf, b, y, p, cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y)`: the spec's chunk structure + on `v_wmma_f32_16x16x16_f16`: the accumulator starts at the bias; each chunk (16 input channels of one tap, in the + layer's chunk order) is one instruction with the float16 accumulator as C, and its result is rounded to float16 + (nearest even) before the next chunk. Not bit-exact: the instruction's sum is not the exact one + (`records/wmma-probe-2026-11-14.json`). An exact-zero sum gives +0, the reference's convention. Leaky ReLU and, with + `pool` 1, the 2 x 2 max-pool are the exact helpers of `fg_exact16.h`, in the epilogue. 1 / 2 / 4 output-channel tiles + of 16 per wave; a workgroup is 8 waves over 16 x 16 pixels. +- `fg_upscale_hwc_f16`: `fg_upscale_f16`'s arithmetic on NHWC tensors. The adds are `fg_add_f16`. +- net1 is 27 launches: five encoder convolutions with the pool fused, the bottleneck's two, and per decoder level an + upscale, an add and two convolutions. HIP and D3D12 are byte-identical; against the CPU reference the output is a + report (net1's output is not gated; the gate is the generated frame against the oracle). ## Configuration (`reference/config.py`) diff --git a/clean/fg/README.md b/clean/fg/README.md index d3282b56..975a7391 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -25,6 +25,10 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `records/wmma-probe-2026-11-14.json` | the probe's record (HIP): neither form equals the exact chunk. On ordinary data (random, net1-like, the layer chains; 62.4 million outputs) float32 accumulate rounded to float16 per chunk is more than one step off in 1,216 outputs and more than four in 211; float16 accumulate in 6.05 million and 1.86 million | | `tests/test_fg_wmma_probe.py` | the probe's CPU parts: the float16-midpoint test against brute force, the RN32 reference against exact fractions, the case classes | | `tests/test_fg_net1_gpu_plan.py` | the chain as planned on the CPU: every step output's shape against the reference's buffers, launch arguments, natural weight order, chain lines | +| `kernels/fg_net1_mma.hip`, `clean/dx12/kernels/fg_mconv.hlsl`, `fg_upscale_hwc.hlsl` | net1's fast path, tier B (code object `fg_net1_mma.co`): `fg_mconv1/2/4` (one `v_wmma_f32_16x16x16_f16` per spec chunk in the layer's chunk order, the float16 accumulator from the bias as C, the result rounded to float16 after every chunk; leaky ReLU and the fused 2 x 2 pool in the epilogue; NHWC float16) and `fg_upscale_hwc_f16`. Not bit-exact to the CPU reference (the matrix unit's chunk sum is not exact); HIP and D3D12 byte-identical | +| `runner/fg_net1_mma.py` | net1 on the fast path in 27 launches, HIP and D3D12: every buffer byte for byte between the back ends (sentinels, slack, a half-grid negative control per kernel; synthetic weights, odd sizes, special values, `--model` execute-only); against the CPU reference as a report (largest step distance, values over 1 and over 4 steps, per buffer and per output value); `--timing` at 1080p- and 1440p-sized inputs (information only, after a launch-length probe) | +| `records/net1-tierb-2026-11-14.json` | the fast path's record: the gate results, the report against the CPU reference (synthetic and converted weights, counts only) and the timing | +| `tests/test_fg_net1_mma_plan.py` | the fast path as planned on the CPU: every weight fragment element against the weights, chunk order and tap; buffers and launches of the fused plan | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | | `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` | diff --git a/clean/fg/kernels/fg_net1_mma.hip b/clean/fg/kernels/fg_net1_mma.hip new file mode 100644 index 00000000..20b55090 --- /dev/null +++ b/clean/fg/kernels/fg_net1_mma.hip @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: MIT +// Frame generation (issue #96): net1's convolutions on gfx12's 16 x 16 x 16 matrix instructions (the fast path of +// handoff note NET1-FAST-PATH, step 2). Not bit-exact to the CPU reference: the matrix unit's chunk sum is not the exact +// one (clean/fg/records/wmma-probe-2026-11-14.json); net1's output against the CPU reference is reported, not gated. +// HIP and the D3D12 port (clean/dx12/kernels/fg_mconv.hlsl) are byte-identical to each other. +// +// Layout (fast path only): tensors NHWC float16, element (y, x, c) at (y * W + x) * C + c, so the 8 consecutive +// channels of one pixel are one 16-byte fragment load or store. +// +// fg_mconv(x, wf, b, y, p, cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y), tier B of the note: +// one net1 convolution (stride 1, zero border for k = 3), the spec's chunk structure: the accumulator starts at the +// bias and takes one WMMA per chunk (16 input channels of one tap; K = 16), in the layer's chunk order (order 0 +// ascending, 1 channel block / kx / ky, 2 kx / channel block / ky): v_wmma_f32_16x16x16_f16 with C = the float16 +// accumulator (widened exactly), its float32 result rounded to float16 after every chunk (v_cvt_f16_f32, nearest +// even), so the accumulator stays float16 as in the spec. An exact-zero sum gives +0, the reference's convention +// (the probe found a -0 only where the matrix unit dropped a tiny term). +// Then leaky ReLU (act 1; fg_leaky16, exact) and the output y; with pool 1 also net1's 2 x 2 max-pool of the +// activated output into p (floor sizes; fg_pool_step16 from tap (0, 0), (0, 1), (1, 0), (1, 1)). +// Weights wf: A fragments in chunk order, [cout / 16][chunks][32 lanes][8] float16: lane l, row o = 16 mt + l % 16, +// element j = input channel 16 cb + 8 (l / 16) + j of the chunk's tap (built by the runner, fg_net1_mma.py). +// Work: a workgroup is 8 waves over a 16 x 16 pixel tile; wave w takes rows 2w and 2w + 1 (two 16-pixel N tiles) and +// MT output-channel tiles (16 MT channels). Grid: tiles_x * tiles_y * (cout / (16 MT)) workgroups of 256 lanes; +// block id = (mg * tiles_y + ty) * tiles_x + tx. +// fg_upscale_hwc_f16: net1's bilinear upscale (fg_upscale_f16 of fg_net1.hip, the same arithmetic) on NHWC tensors. +#include "wg_platform.h" +#include "wg_mem.h" + +typedef unsigned int uint; +typedef unsigned short u16; +#define FG_FN WG_FN +#define FG_I64 long long +#define FG_FBH32(v) (31u - (uint)__builtin_clz(v)) +#include "fg_exact16.h" + +#pragma clang fp contract(off) + +typedef uint v4u32 __attribute__((ext_vector_type(4))); + +WG_FN uint fg_f2h(float f) { return (uint)__builtin_bit_cast(u16, (_Float16)f); } // round to nearest even +WG_FN float fg_h2f(uint h) { return (float)__builtin_bit_cast(_Float16, (u16)h); } // exact +WG_FN uint fg_xor1(uint v) { // v of lane (lane ^ 1) + return (uint)__builtin_amdgcn_ds_bpermute((int)(((wg_lane_id() & 31u) ^ 1u) << 2), (int)v); +} + +WG_FN v8h fg_as_v8h(v4u32 u) { return __builtin_bit_cast(v8h, u); } +WG_FN v4u32 fg_ld128(const u16* p, uint elem) { return *(const v4u32*)(p + elem); } + +// one chunk's (channel block, ky, kx) in the layer's order +WG_FN void fg_chunk(int q, int order, int k, int nb, int* cb, int* ky, int* kx) { + const int taps = k * k; + if (order == 0) { *cb = q; *ky = 0; *kx = 0; } + else if (order == 1) { *cb = q / taps; const int t = q - *cb * taps; *kx = t / k; *ky = t - *kx * k; } + else { *kx = q / (nb * k); const int r = q - *kx * (nb * k); *cb = r / k; *ky = r - *cb * k; } +} + +template +WG_FN void fg_mconv(const u16* __restrict__ x, const u16* __restrict__ wf, const u16* __restrict__ b, + u16* __restrict__ y, u16* __restrict__ p, int cin, int cout, int H, int W, int k, int order, + int act, int pool, int tiles_x, int tiles_y) { + const uint lane = wg_lane_id() & 31u, wave = wg_lane_id() >> 5; + const int blk = (int)wg_block_id(); + const int tx = blk % tiles_x, rest = blk / tiles_x; + const int ty = rest % tiles_y, mg = rest / tiles_y; + const int n = (int)(lane & 15u), hh = (int)(lane >> 4); + const int px = tx * 16 + n; // this lane's pixel column + const int py0 = ty * 16 + 2 * (int)wave; // rows py0 (N tile 0) and py0 + 1 (N tile 1) + const int pad = k / 2, nb = cin / 16, nq = nb * k * k; + // accumulators: [mt][nt], element v = output channel 16 (mg MT + mt) + 8 hh + v of pixel (px, py0 + nt) + v8f a32[MT][2]; + for (int mt = 0; mt < MT; ++mt) { + const int o0 = (mg * MT + mt) * 16 + 8 * hh; + for (int v = 0; v < 8; ++v) { + const float bv = fg_h2f(b[o0 + v]); + for (int nt = 0; nt < 2; ++nt) a32[mt][nt][v] = bv; + } + } + for (int q = 0; q < nq; ++q) { + int cb, ky, kx; + fg_chunk(q, order, k, nb, &cb, &ky, &kx); + v8h fb[2], fa[MT]; + const int sx = px + kx - pad; + for (int nt = 0; nt < 2; ++nt) { + const int sy = py0 + nt + ky - pad; + const bool in = sx >= 0 && sx < W && sy >= 0 && sy < H; + const uint e = in ? (uint)((sy * W + sx) * cin + cb * 16 + 8 * hh) : 0u; + v4u32 u = fg_ld128(x, e); + if (!in) u = (v4u32){0u, 0u, 0u, 0u}; // zero border + fb[nt] = fg_as_v8h(u); + } + for (int mt = 0; mt < MT; ++mt) + fa[mt] = fg_as_v8h(fg_ld128(wf, (uint)((((mg * MT + mt) * nq + q) * 32 + (int)lane) * 8))); + for (int mt = 0; mt < MT; ++mt) + for (int nt = 0; nt < 2; ++nt) { + v8f d = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(fa[mt], fb[nt], a32[mt][nt]); + for (int v = 0; v < 8; ++v) { const float t = d[v]; a32[mt][nt][v] = fg_h2f(fg_f2h(t)); } + } + } + // epilogue: float16 bits, activation, store; pool + for (int mt = 0; mt < MT; ++mt) { + const int o0 = (mg * MT + mt) * 16 + 8 * hh; + uint r[2][8]; + for (int nt = 0; nt < 2; ++nt) + for (int v = 0; v < 8; ++v) { + const float t = a32[mt][nt][v]; + uint hb = fg_f2h(t); + if (act == 1) hb = fg_leaky16(hb); + r[nt][v] = hb; + } + for (int nt = 0; nt < 2; ++nt) { + const int py = py0 + nt; + if (px < W && py < H) { + const v4u32 u = {r[nt][0] | (r[nt][1] << 16), r[nt][2] | (r[nt][3] << 16), r[nt][4] | (r[nt][5] << 16), + r[nt][6] | (r[nt][7] << 16)}; + *(v4u32*)(y + (size_t)(py * W + px) * cout + o0) = u; + } + } + if (pool == 1) { + uint m[8]; + for (int v = 0; v < 8; ++v) { + const uint x01 = fg_xor1(r[0][v]), x11 = fg_xor1(r[1][v]); // pixel px + 1 (lane n + 1) + uint t = r[0][v]; + t = fg_pool_step16(t, x01); + t = fg_pool_step16(t, r[1][v]); + t = fg_pool_step16(t, x11); + m[v] = t; + } + const int Ho = H / 2, Wo = W / 2, ox = px / 2, oy = py0 / 2; + if ((n & 1) == 0 && ox < Wo && oy < Ho) { + const v4u32 u = {m[0] | (m[1] << 16), m[2] | (m[3] << 16), m[4] | (m[5] << 16), m[6] | (m[7] << 16)}; + *(v4u32*)(p + (size_t)(oy * Wo + ox) * cout + o0) = u; + } + } + } +} + +#define FG_MCONV_ARGS const u16* __restrict__ x, const u16* __restrict__ wf, const u16* __restrict__ b, \ + u16* __restrict__ y, u16* __restrict__ p, int cin, int cout, int H, int W, int k, int order, int act, int pool, \ + int tiles_x, int tiles_y +#define FG_MCONV_CALL x, wf, b, y, p, cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y +WG_KERNEL_N(fg_mconv1, 256)(FG_MCONV_ARGS) { fg_mconv<1>(FG_MCONV_CALL); } +WG_KERNEL_N(fg_mconv2, 256)(FG_MCONV_ARGS) { fg_mconv<2>(FG_MCONV_CALL); } +WG_KERNEL_N(fg_mconv4, 256)(FG_MCONV_ARGS) { fg_mconv<4>(FG_MCONV_CALL); } + +// ---- upscale on NHWC (the arithmetic of fg_upscale_f16) ------------------------------------------------------------ +struct FgTap { int i0, i1; float w0, w1; }; +WG_FN FgTap fg_axis(float r, int o, int n_in) { + FgTap t; + const float s = r * (float)o; + t.i0 = (int)s; + t.i1 = (int)(s + 1.0f); + if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; } + t.w1 = fg_h2f(fg_f2h(s - (float)t.i0)); + t.w0 = fg_h2f(fg_f2h(((float)t.i0 - s) + 1.0f)); + return t; +} +WG_FN float fg_lerp16(float wa, float a, float wb, float b) { + return fg_h2f(fg_f2h(fg_h2f(fg_f2h(wa * a)) + fg_h2f(fg_f2h(wb * b)))); +} + +WG_KERNEL_N(fg_upscale_hwc_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y, int C, int hi, int wi, int ho, + int wo, uint rx_bits, uint ry_bits) { + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * ho * wo) return; + const int c = idx % C; + const int pix = idx / C; + const int oy = pix / wo; + const int ox = pix - oy * wo; + const FgTap tx = fg_axis(__builtin_bit_cast(float, rx_bits), ox, wi); + const FgTap ty = fg_axis(__builtin_bit_cast(float, ry_bits), oy, hi); + const float top = fg_lerp16(tx.w0, fg_h2f(x[(ty.i0 * wi + tx.i0) * C + c]), tx.w1, fg_h2f(x[(ty.i0 * wi + tx.i1) * C + c])); + const float bot = fg_lerp16(tx.w0, fg_h2f(x[(ty.i1 * wi + tx.i0) * C + c]), tx.w1, fg_h2f(x[(ty.i1 * wi + tx.i1) * C + c])); + uint out = fg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot)); + if (fg_is_nan16(out) != 0u) out = FG_NAN16; + y[idx] = (u16)out; +} diff --git a/clean/fg/records/net1-tierb-2026-11-14.json b/clean/fg/records/net1-tierb-2026-11-14.json new file mode 100644 index 00000000..a5c9324c --- /dev/null +++ b/clean/fg/records/net1-tierb-2026-11-14.json @@ -0,0 +1 @@ +{"cases":[{"case":"synthetic_64","hip_equals_d3d12":true,"input_pixels":[64,64],"negative_controls":{"fg_add_f16":{"caught":true,"differ":1024,"step":8},"fg_mconv1":{"caught":true,"differ":16346,"step":26},"fg_mconv2":{"caught":true,"differ":40873,"step":0},"fg_mconv4":{"caught":true,"differ":1274,"step":4},"fg_upscale_hwc_f16":{"caught":true,"differ":1018,"step":7}},"ok":true,"vs_cpu_buffers":{"a0":{"nonfinite_mismatch":0,"over_1_step":11,"over_4_steps":2,"steps_max":16,"values":32768},"a1":{"nonfinite_mismatch":0,"over_1_step":36,"over_4_steps":12,"steps_max":20,"values":16384},"a2":{"nonfinite_mismatch":0,"over_1_step":227,"over_4_steps":57,"steps_max":60,"values":8192},"a3":{"nonfinite_mismatch":0,"over_1_step":351,"over_4_steps":94,"steps_max":512,"values":2048},"a4":{"nonfinite_mismatch":0,"over_1_step":421,"over_4_steps":133,"steps_max":982,"values":1024},"b5":{"nonfinite_mismatch":0,"over_1_step":265,"over_4_steps":84,"steps_max":161,"values":512},"d1":{"nonfinite_mismatch":0,"over_1_step":2873,"over_4_steps":891,"steps_max":1117,"values":8192},"d2":{"nonfinite_mismatch":0,"over_1_step":1707,"over_4_steps":471,"steps_max":1796,"values":4096},"d3":{"nonfinite_mismatch":0,"over_1_step":1150,"over_4_steps":467,"steps_max":1218,"values":2048},"d4":{"nonfinite_mismatch":0,"over_1_step":297,"over_4_steps":137,"steps_max":761,"values":512},"d5":{"nonfinite_mismatch":0,"over_1_step":153,"over_4_steps":56,"steps_max":215,"values":256},"e0":{"nonfinite_mismatch":0,"over_1_step":4215,"over_4_steps":1119,"steps_max":1036,"values":32768},"e1":{"nonfinite_mismatch":0,"over_1_step":3589,"over_4_steps":1118,"steps_max":2702,"values":16384},"e2":{"nonfinite_mismatch":0,"over_1_step":2359,"over_4_steps":681,"steps_max":1936,"values":8192},"e3":{"nonfinite_mismatch":0,"over_1_step":867,"over_4_steps":264,"steps_max":1846,"values":2048},"e4":{"nonfinite_mismatch":0,"over_1_step":549,"over_4_steps":207,"steps_max":3224,"values":1024},"out":{"nonfinite_mismatch":0,"over_1_step":3727,"over_4_steps":1283,"steps_max":4201,"values":16384},"p1":{"nonfinite_mismatch":0,"over_1_step":3,"over_4_steps":1,"steps_max":16,"values":8192},"p2":{"nonfinite_mismatch":0,"over_1_step":10,"over_4_steps":3,"steps_max":16,"values":4096},"p3":{"nonfinite_mismatch":0,"over_1_step":61,"over_4_steps":13,"steps_max":20,"values":2048},"p4":{"nonfinite_mismatch":0,"over_1_step":78,"over_4_steps":21,"steps_max":53,"values":512},"p5":{"nonfinite_mismatch":0,"over_1_step":94,"over_4_steps":26,"steps_max":41,"values":256},"s0":{"nonfinite_mismatch":0,"over_1_step":4199,"over_4_steps":1306,"steps_max":2624,"values":32768},"s1":{"nonfinite_mismatch":0,"over_1_step":2013,"over_4_steps":509,"steps_max":992,"values":16384},"s2":{"nonfinite_mismatch":0,"over_1_step":1668,"over_4_steps":590,"steps_max":4136,"values":8192},"s3":{"nonfinite_mismatch":0,"over_1_step":648,"over_4_steps":172,"steps_max":1456,"values":2048},"s4":{"nonfinite_mismatch":0,"over_1_step":447,"over_4_steps":116,"steps_max":948,"values":1024},"u0":{"nonfinite_mismatch":0,"over_1_step":10113,"over_4_steps":2831,"steps_max":1462,"values":32768},"u1":{"nonfinite_mismatch":0,"over_1_step":5516,"over_4_steps":1312,"steps_max":1796,"values":16384},"u2":{"nonfinite_mismatch":0,"over_1_step":4092,"over_4_steps":1447,"steps_max":2349,"values":8192},"u3":{"nonfinite_mismatch":0,"over_1_step":1120,"over_4_steps":405,"steps_max":1045,"values":2048},"u4":{"nonfinite_mismatch":0,"over_1_step":612,"over_4_steps":224,"steps_max":215,"values":1024}},"vs_cpu_output":{"nonfinite_mismatch":0,"over_1_step":3727,"over_4_steps":1283,"steps_max":4201,"values":16384},"vs_cpu_output_per_value":[{"nonfinite_mismatch":0,"over_1_step":1425,"over_4_steps":502,"steps_max":630,"values":4096},{"nonfinite_mismatch":0,"over_1_step":530,"over_4_steps":188,"steps_max":230,"values":4096},{"nonfinite_mismatch":0,"over_1_step":954,"over_4_steps":317,"steps_max":4201,"values":4096},{"nonfinite_mismatch":0,"over_1_step":818,"over_4_steps":276,"steps_max":1889,"values":4096}]},{"case":"synthetic_70x90","hip_equals_d3d12":true,"input_pixels":[90,70],"negative_controls":{"fg_add_f16":{"caught":true,"differ":1024,"step":8},"fg_mconv1":{"caught":true,"differ":19127,"step":26},"fg_mconv2":{"caught":true,"differ":46798,"step":0},"fg_mconv4":{"caught":true,"differ":2812,"step":1},"fg_upscale_hwc_f16":{"caught":true,"differ":1024,"step":7}},"ok":true,"vs_cpu_buffers":{"a0":{"nonfinite_mismatch":0,"over_1_step":25,"over_4_steps":3,"steps_max":41,"values":50400},"a1":{"nonfinite_mismatch":0,"over_1_step":59,"over_4_steps":15,"steps_max":71,"values":23936},"a2":{"nonfinite_mismatch":0,"over_1_step":466,"over_4_steps":120,"steps_max":64,"values":11264},"a3":{"nonfinite_mismatch":0,"over_1_step":607,"over_4_steps":169,"steps_max":440,"values":2560},"a4":{"nonfinite_mismatch":0,"over_1_step":481,"over_4_steps":175,"steps_max":1330,"values":1024},"b5":{"nonfinite_mismatch":0,"over_1_step":276,"over_4_steps":85,"steps_max":298,"values":512},"d1":{"nonfinite_mismatch":0,"over_1_step":3675,"over_4_steps":1150,"steps_max":2564,"values":11968},"d2":{"nonfinite_mismatch":0,"over_1_step":2347,"over_4_steps":705,"steps_max":1250,"values":5632},"d3":{"nonfinite_mismatch":0,"over_1_step":1181,"over_4_steps":417,"steps_max":2379,"values":2560},"d4":{"nonfinite_mismatch":0,"over_1_step":312,"over_4_steps":128,"steps_max":1572,"values":512},"d5":{"nonfinite_mismatch":0,"over_1_step":159,"over_4_steps":72,"steps_max":311,"values":256},"e0":{"nonfinite_mismatch":0,"over_1_step":9672,"over_4_steps":2835,"steps_max":1332,"values":50400},"e1":{"nonfinite_mismatch":0,"over_1_step":5061,"over_4_steps":1466,"steps_max":1552,"values":23936},"e2":{"nonfinite_mismatch":0,"over_1_step":3339,"over_4_steps":847,"steps_max":2076,"values":11264},"e3":{"nonfinite_mismatch":0,"over_1_step":1074,"over_4_steps":338,"steps_max":1685,"values":2560},"e4":{"nonfinite_mismatch":0,"over_1_step":588,"over_4_steps":239,"steps_max":820,"values":1024},"out":{"nonfinite_mismatch":0,"over_1_step":6221,"over_4_steps":2004,"steps_max":2027,"values":25200},"p1":{"nonfinite_mismatch":0,"over_1_step":3,"over_4_steps":0,"steps_max":2,"values":11968},"p2":{"nonfinite_mismatch":0,"over_1_step":9,"over_4_steps":4,"steps_max":71,"values":5632},"p3":{"nonfinite_mismatch":0,"over_1_step":103,"over_4_steps":27,"steps_max":20,"values":2560},"p4":{"nonfinite_mismatch":0,"over_1_step":114,"over_4_steps":35,"steps_max":72,"values":512},"p5":{"nonfinite_mismatch":0,"over_1_step":111,"over_4_steps":45,"steps_max":51,"values":256},"s0":{"nonfinite_mismatch":0,"over_1_step":6190,"over_4_steps":1652,"steps_max":992,"values":50400},"s1":{"nonfinite_mismatch":0,"over_1_step":3214,"over_4_steps":880,"steps_max":2048,"values":23936},"s2":{"nonfinite_mismatch":0,"over_1_step":2272,"over_4_steps":761,"steps_max":2319,"values":11264},"s3":{"nonfinite_mismatch":0,"over_1_step":886,"over_4_steps":307,"steps_max":1136,"values":2560},"s4":{"nonfinite_mismatch":0,"over_1_step":500,"over_4_steps":166,"steps_max":2336,"values":1024},"u0":{"nonfinite_mismatch":0,"over_1_step":13349,"over_4_steps":3401,"steps_max":1776,"values":50400},"u1":{"nonfinite_mismatch":0,"over_1_step":8316,"over_4_steps":2040,"steps_max":1907,"values":23936},"u2":{"nonfinite_mismatch":0,"over_1_step":4523,"over_4_steps":1454,"steps_max":2319,"values":11264},"u3":{"nonfinite_mismatch":0,"over_1_step":1381,"over_4_steps":499,"steps_max":1815,"values":2560},"u4":{"nonfinite_mismatch":0,"over_1_step":636,"over_4_steps":288,"steps_max":311,"values":1024}},"vs_cpu_output":{"nonfinite_mismatch":0,"over_1_step":6221,"over_4_steps":2004,"steps_max":2027,"values":25200},"vs_cpu_output_per_value":[{"nonfinite_mismatch":0,"over_1_step":1596,"over_4_steps":521,"steps_max":1035,"values":6300},{"nonfinite_mismatch":0,"over_1_step":1721,"over_4_steps":584,"steps_max":258,"values":6300},{"nonfinite_mismatch":0,"over_1_step":1204,"over_4_steps":342,"steps_max":2027,"values":6300},{"nonfinite_mismatch":0,"over_1_step":1700,"over_4_steps":557,"steps_max":1547,"values":6300}]},{"case":"specials_64","hip_equals_d3d12":true,"input_pixels":[64,64],"negative_controls":{"fg_add_f16":{"caught":true,"differ":1024,"step":8},"fg_mconv1":{"caught":true,"differ":16384,"step":26},"fg_mconv2":{"caught":true,"differ":40873,"step":0},"fg_mconv4":{"caught":true,"differ":1280,"step":4},"fg_upscale_hwc_f16":{"caught":true,"differ":1024,"step":7}},"ok":true,"vs_cpu_buffers":{"a0":{"nonfinite_mismatch":0,"over_1_step":4,"over_4_steps":1,"steps_max":10,"values":32768},"a1":{"nonfinite_mismatch":0,"over_1_step":11,"over_4_steps":2,"steps_max":10,"values":16384},"a2":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":8192},"a3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"a4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":1024},"b5":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":512},"d1":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":8192},"d2":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":4096},"d3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"d4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":512},"d5":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":256},"e0":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":32768},"e1":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":16384},"e2":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":8192},"e3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"e4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":1024},"out":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":16384},"p1":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":1,"values":8192},"p2":{"nonfinite_mismatch":0,"over_1_step":4,"over_4_steps":0,"steps_max":4,"values":4096},"p3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"p4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":512},"p5":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":256},"s0":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":32768},"s1":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":16384},"s2":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":8192},"s3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"s4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":1024},"u0":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":32768},"u1":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":16384},"u2":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":8192},"u3":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":2048},"u4":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":1024}},"vs_cpu_output":{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":16384},"vs_cpu_output_per_value":[{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":4096},{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":4096},{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":4096},{"nonfinite_mismatch":0,"over_1_step":0,"over_4_steps":0,"steps_max":0,"values":4096}]},{"case":"finite_specials_64","hip_equals_d3d12":true,"input_pixels":[64,64],"negative_controls":{"fg_add_f16":{"caught":true,"differ":1015,"step":8},"fg_mconv1":{"caught":true,"differ":16181,"step":26},"fg_mconv2":{"caught":true,"differ":40815,"step":0},"fg_mconv4":{"caught":true,"differ":1267,"step":4},"fg_upscale_hwc_f16":{"caught":true,"differ":1012,"step":7}},"ok":true,"vs_cpu_buffers":{"a0":{"nonfinite_mismatch":0,"over_1_step":4,"over_4_steps":1,"steps_max":8,"values":32768},"a1":{"nonfinite_mismatch":0,"over_1_step":5,"over_4_steps":1,"steps_max":10,"values":16384},"a2":{"nonfinite_mismatch":0,"over_1_step":10,"over_4_steps":3,"steps_max":8,"values":8192},"a3":{"nonfinite_mismatch":0,"over_1_step":88,"over_4_steps":25,"steps_max":655,"values":2048},"a4":{"nonfinite_mismatch":0,"over_1_step":284,"over_4_steps":80,"steps_max":369,"values":1024},"b5":{"nonfinite_mismatch":0,"over_1_step":223,"over_4_steps":71,"steps_max":287,"values":512},"d1":{"nonfinite_mismatch":0,"over_1_step":3170,"over_4_steps":1188,"steps_max":19781,"values":8192},"d2":{"nonfinite_mismatch":0,"over_1_step":1428,"over_4_steps":466,"steps_max":2852,"values":4096},"d3":{"nonfinite_mismatch":0,"over_1_step":883,"over_4_steps":330,"steps_max":2767,"values":2048},"d4":{"nonfinite_mismatch":0,"over_1_step":300,"over_4_steps":138,"steps_max":573,"values":512},"d5":{"nonfinite_mismatch":0,"over_1_step":151,"over_4_steps":53,"steps_max":1209,"values":256},"e0":{"nonfinite_mismatch":0,"over_1_step":8098,"over_4_steps":2751,"steps_max":22606,"values":32768},"e1":{"nonfinite_mismatch":0,"over_1_step":4437,"over_4_steps":1401,"steps_max":19876,"values":16384},"e2":{"nonfinite_mismatch":0,"over_1_step":2231,"over_4_steps":663,"steps_max":24074,"values":8192},"e3":{"nonfinite_mismatch":0,"over_1_step":735,"over_4_steps":249,"steps_max":980,"values":2048},"e4":{"nonfinite_mismatch":0,"over_1_step":503,"over_4_steps":171,"steps_max":23282,"values":1024},"out":{"nonfinite_mismatch":0,"over_1_step":5079,"over_4_steps":1753,"steps_max":27578,"values":16384},"p1":{"nonfinite_mismatch":0,"over_1_step":2,"over_4_steps":1,"steps_max":8,"values":8192},"p2":{"nonfinite_mismatch":0,"over_1_step":2,"over_4_steps":0,"steps_max":3,"values":4096},"p3":{"nonfinite_mismatch":0,"over_1_step":4,"over_4_steps":1,"steps_max":6,"values":2048},"p4":{"nonfinite_mismatch":0,"over_1_step":24,"over_4_steps":7,"steps_max":128,"values":512},"p5":{"nonfinite_mismatch":0,"over_1_step":58,"over_4_steps":12,"steps_max":23,"values":256},"s0":{"nonfinite_mismatch":0,"over_1_step":8018,"over_4_steps":2764,"steps_max":21845,"values":32768},"s1":{"nonfinite_mismatch":0,"over_1_step":2836,"over_4_steps":868,"steps_max":25088,"values":16384},"s2":{"nonfinite_mismatch":0,"over_1_step":1435,"over_4_steps":462,"steps_max":29116,"values":8192},"s3":{"nonfinite_mismatch":0,"over_1_step":550,"over_4_steps":209,"steps_max":28448,"values":2048},"s4":{"nonfinite_mismatch":0,"over_1_step":368,"over_4_steps":93,"steps_max":54,"values":1024},"u0":{"nonfinite_mismatch":0,"over_1_step":11535,"over_4_steps":3676,"steps_max":25216,"values":32768},"u1":{"nonfinite_mismatch":0,"over_1_step":5006,"over_4_steps":1451,"steps_max":24963,"values":16384},"u2":{"nonfinite_mismatch":0,"over_1_step":3194,"over_4_steps":964,"steps_max":25706,"values":8192},"u3":{"nonfinite_mismatch":0,"over_1_step":1159,"over_4_steps":456,"steps_max":6918,"values":2048},"u4":{"nonfinite_mismatch":0,"over_1_step":604,"over_4_steps":212,"steps_max":1209,"values":1024}},"vs_cpu_output":{"nonfinite_mismatch":0,"over_1_step":5079,"over_4_steps":1753,"steps_max":27578,"values":16384},"vs_cpu_output_per_value":[{"nonfinite_mismatch":0,"over_1_step":972,"over_4_steps":253,"steps_max":431,"values":4096},{"nonfinite_mismatch":0,"over_1_step":1536,"over_4_steps":591,"steps_max":27578,"values":4096},{"nonfinite_mismatch":0,"over_1_step":1367,"over_4_steps":558,"steps_max":26282,"values":4096},{"nonfinite_mismatch":0,"over_1_step":1204,"over_4_steps":351,"steps_max":24864,"values":4096}]},{"case":"converter_model_fixed_input","hip_equals_d3d12":true,"input_pixels":[64,64],"negative_controls":{"fg_add_f16":{"caught":true,"differ":1024,"step":8},"fg_mconv1":{"caught":true,"differ":16345,"step":26},"fg_mconv2":{"caught":true,"differ":40876,"step":0},"fg_mconv4":{"caught":true,"differ":1275,"step":4},"fg_upscale_hwc_f16":{"caught":true,"differ":1022,"step":7}},"ok":true,"vs_cpu_buffers":{"a0":{"nonfinite_mismatch":0,"over_1_step":15,"over_4_steps":5,"steps_max":20,"values":32768},"a1":{"nonfinite_mismatch":0,"over_1_step":44,"over_4_steps":14,"steps_max":27,"values":16384},"a2":{"nonfinite_mismatch":0,"over_1_step":156,"over_4_steps":42,"steps_max":180,"values":8192},"a3":{"nonfinite_mismatch":0,"over_1_step":186,"over_4_steps":49,"steps_max":253,"values":2048},"a4":{"nonfinite_mismatch":0,"over_1_step":233,"over_4_steps":76,"steps_max":147,"values":1024},"b5":{"nonfinite_mismatch":0,"over_1_step":127,"over_4_steps":39,"steps_max":286,"values":512},"d1":{"nonfinite_mismatch":0,"over_1_step":3897,"over_4_steps":1498,"steps_max":4837,"values":8192},"d2":{"nonfinite_mismatch":0,"over_1_step":1937,"over_4_steps":739,"steps_max":1182,"values":4096},"d3":{"nonfinite_mismatch":0,"over_1_step":1009,"over_4_steps":380,"steps_max":3270,"values":2048},"d4":{"nonfinite_mismatch":0,"over_1_step":218,"over_4_steps":66,"steps_max":3759,"values":512},"d5":{"nonfinite_mismatch":0,"over_1_step":55,"over_4_steps":9,"steps_max":26,"values":256},"e0":{"nonfinite_mismatch":0,"over_1_step":13915,"over_4_steps":5219,"steps_max":6574,"values":32768},"e1":{"nonfinite_mismatch":0,"over_1_step":8507,"over_4_steps":3687,"steps_max":4715,"values":16384},"e2":{"nonfinite_mismatch":0,"over_1_step":3605,"over_4_steps":1188,"steps_max":4436,"values":8192},"e3":{"nonfinite_mismatch":0,"over_1_step":817,"over_4_steps":265,"steps_max":2177,"values":2048},"e4":{"nonfinite_mismatch":0,"over_1_step":357,"over_4_steps":102,"steps_max":470,"values":1024},"out":{"nonfinite_mismatch":0,"over_1_step":7410,"over_4_steps":2881,"steps_max":14732,"values":16384},"p1":{"nonfinite_mismatch":0,"over_1_step":2,"over_4_steps":0,"steps_max":2,"values":8192},"p2":{"nonfinite_mismatch":0,"over_1_step":10,"over_4_steps":4,"steps_max":24,"values":4096},"p3":{"nonfinite_mismatch":0,"over_1_step":44,"over_4_steps":11,"steps_max":180,"values":2048},"p4":{"nonfinite_mismatch":0,"over_1_step":46,"over_4_steps":11,"steps_max":72,"values":512},"p5":{"nonfinite_mismatch":0,"over_1_step":57,"over_4_steps":20,"steps_max":147,"values":256},"s0":{"nonfinite_mismatch":0,"over_1_step":9033,"over_4_steps":2364,"steps_max":4112,"values":32768},"s1":{"nonfinite_mismatch":0,"over_1_step":4576,"over_4_steps":1348,"steps_max":3264,"values":16384},"s2":{"nonfinite_mismatch":0,"over_1_step":2441,"over_4_steps":706,"steps_max":1296,"values":8192},"s3":{"nonfinite_mismatch":0,"over_1_step":512,"over_4_steps":152,"steps_max":640,"values":2048},"s4":{"nonfinite_mismatch":0,"over_1_step":205,"over_4_steps":55,"steps_max":216,"values":1024},"u0":{"nonfinite_mismatch":0,"over_1_step":14465,"over_4_steps":4571,"steps_max":6924,"values":32768},"u1":{"nonfinite_mismatch":0,"over_1_step":6830,"over_4_steps":2241,"steps_max":6777,"values":16384},"u2":{"nonfinite_mismatch":0,"over_1_step":3611,"over_4_steps":1144,"steps_max":1744,"values":8192},"u3":{"nonfinite_mismatch":0,"over_1_step":762,"over_4_steps":216,"steps_max":3759,"values":2048},"u4":{"nonfinite_mismatch":0,"over_1_step":220,"over_4_steps":36,"steps_max":26,"values":1024}},"vs_cpu_output":{"nonfinite_mismatch":0,"over_1_step":7410,"over_4_steps":2881,"steps_max":14732,"values":16384},"vs_cpu_output_per_value":[{"nonfinite_mismatch":0,"over_1_step":2257,"over_4_steps":849,"steps_max":2705,"values":4096},{"nonfinite_mismatch":0,"over_1_step":2522,"over_4_steps":1110,"steps_max":14732,"values":4096},{"nonfinite_mismatch":0,"over_1_step":748,"over_4_steps":42,"steps_max":9,"values":4096},{"nonfinite_mismatch":0,"over_1_step":1883,"over_4_steps":880,"steps_max":13427,"values":4096}]}],"date":"2026-11-14","gate":{"hip_equals_d3d12_all_buffers":true,"pass":true},"gpu":"gfx1201","report_vs_cpu_reference":"steps = distance in float16 values; report only (the gate is the generated frame)","timing":{"max_launch_ms":100.0,"method":"HIP: device events around 10 back-to-back network runs, mean per run (per-op: events around each launch). D3D12: kernel_replay chain wall time from submission to completion, synchronised after each of 10 runs, mean per run. Synthetic weights, buffers resident, warm-up first. Information only.","modes":{"tier_b":{"1080p":{"d3d12_exit":0,"d3d12_ms":12.47,"hip_ms":4.461,"max_launch_ms_measured":1.183,"out_hip_equals_d3d12":true,"per_op_ms":{"add":0.448,"conv":4.219,"upscale":0.888},"pixels":[1920,1080],"predicted_max_launch_ms":3.17},"1440p":{"d3d12_exit":0,"d3d12_ms":20.462,"hip_ms":7.101,"max_launch_ms_measured":1.717,"out_hip_equals_d3d12":true,"per_op_ms":{"add":0.713,"conv":6.074,"upscale":1.563},"pixels":[2560,1440],"predicted_max_launch_ms":5.63},"probe":{"max_step":2,"max_step_ms":0.401,"pixels":[512,512],"sum_of_steps_ms":3.189}}},"reps":10},"what":"net1 tier B (v_wmma_f32_16x16x16_f16, float16 rounding per chunk) on HIP and D3D12, clean/fg/runner/fg_net1_mma.py"} diff --git a/clean/fg/runner/fg_net1_mma.py b/clean/fg/runner/fg_net1_mma.py new file mode 100644 index 00000000..a865f363 --- /dev/null +++ b/clean/fg/runner/fg_net1_mma.py @@ -0,0 +1,421 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (issue #96): net1 on gfx12's matrix instructions (tier B of the fast path), HIP and D3D12. + + python clean/fg/runner/fg_net1_mma.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/fg/runner/fg_net1_mma.py --build --rocm ... --run [--model ] [--json out.json] + python clean/fg/runner/fg_net1_mma.py --build --rocm ... --timing [--json out.json] + +The network (NET1_WIRING) on NHWC float16 buffers in 27 launches: the 17 convolutions as fg_mconv (one +v_wmma_f32_16x16x16_f16 per spec chunk in the layer's chunk order, the float16 accumulator from the bias as C and the +result rounded to float16 after every chunk; leaky ReLU, and for net1_01..05 the 2 x 2 max-pool, in the epilogue), +five NHWC upscales and five adds (fg_add_f16). + +Gates (the port standard) -- HIP against D3D12, byte for byte: +- every saved buffer (a_k, p_k, b5, d5, u_k, s_k, e_k, d_k, out) compared raw, each filled with its back end's + sentinel (HIP 0xA5, D3D12 0x5A) and followed by 64 slack bytes nobody may write; +- a negative control per kernel: the first launch of that kernel (with 4 or more workgroups) on half of its grid on + D3D12 must change its output. +Reported only (oversight 2026-11-14: the gate is the generated frame against the oracle): every buffer and the +unfolded output against the CPU reference (net1.run), as the largest distance in float16 steps and the number of +values more than 1 and more than 4 steps off (non-finite disagreements counted apart). +Cases: synthetic weights at 64 x 64 and 70 x 90 pixels, special values in the input and biases, and with --model the +weights of a converter output file (execute-only: the files written for the replay are deleted after the run). +--timing (information only): a 512 x 512 probe times every launch first; 1920 x 1080 and 2560 x 1440 input pixels run +only when every launch, scaled by the cell count, stays under MAX_LAUNCH_MS. +""" +import argparse +import ctypes +import hashlib +import json +import os +import sys +import time + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +sys.path.insert(0, HERE) + +from clean.fg.reference import net1 as N # noqa: E402 +import fg_d3d12_check as K # noqa: E402 +import fg_net1_gpu as G # noqa: E402 + +CO_MMA, CO_NET1 = "fg_net1_mma.co", "fg_net1.co" +MCONV = ["fg_mconv%d" % mt for mt in (1, 2, 4)] +KERNELS = MCONV + ["fg_upscale_hwc_f16", "fg_add_f16"] +MODULE_OF = dict([(k, CO_MMA) for k in MCONV + ["fg_upscale_hwc_f16"]] + [("fg_add_f16", CO_NET1)]) +MAX_LAUNCH_MS = 100.0 +TIMING_SIZES = G.TIMING_SIZES +PROBE = G.PROBE + + +# ---- the fast plan ------------------------------------------------------------------------------------------------------ + +def swizzle(w, order): + """Canonical weights [out, k, k, in] -> A fragments in chunk order: [out / 16][chunks][32 lanes][8] float16; lane l + holds row 16 mt + l % 16 and input channels 16 cb + 8 (l // 16) + 0..7 of the chunk's tap.""" + w = np.asarray(w, np.float16) + cout, k, _, cin = w.shape + nb = cin // 16 + chunks = [] + for q in range(nb * k * k): + if order == "ascending": + cb, ky, kx = q, 0, 0 + elif order == "block_kx_ky": + cb, t = divmod(q, k * k) + kx, ky = divmod(t, k) + else: + kx, r = divmod(q, nb * k) + cb, ky = divmod(r, k) + chunks.append(w[:, ky, kx, 16 * cb:16 * cb + 16]) # [out, 16] + a = np.stack(chunks, 1) # [out, nq, 16] + a = a.reshape(cout // 16, 16, len(chunks), 2, 8) # [mt, m, q, h, j] + return np.ascontiguousarray(a.transpose(0, 2, 3, 1, 4)).reshape(-1).tobytes() # [mt, q, h, m, j] = lane 16h + m + + +def mconv_mt(cout): + return 4 if cout % 64 == 0 else (2 if cout % 32 == 0 else 1) + + +def plan(H0, W0, params): + """(data, acts, steps) of net1 on cells [H0, W0, 48]: data {id: bytes}; acts {id: (shape HWC, bytes)}; steps + [(kernel, grid, args, outs, op)] with args "@id" or integers.""" + data, acts, steps = {}, {}, [] + shape = {"cells": (H0, W0, N.CELL)} + pooled = {s["in"]: s["out"] for s in N.NET1_WIRING if s["op"] == "pool"} + for s in N.NET1_WIRING: + op = s["op"] + if op == "pool": + continue # fused into the convolution before it + if op == "conv": + w, b = params[s["layer"]] + cout, k, _, cin = np.shape(w) + wid, bid = "wf_" + s["layer"], "b_" + s["layer"] + data[wid] = swizzle(w, s["chunks"]) + data[bid] = np.asarray(b, np.float16).tobytes() + H, W, C = shape[s["in"]] + if C != cin: + raise ValueError("%s: %d input channels, %s has %d" % (s["layer"], cin, s["in"], C)) + mt = mconv_mt(cout) + tx, ty = -(-W // 16), -(-H // 16) + shape[s["out"]] = (H, W, cout) + outs = [s["out"]] + pool = s["out"] in pooled + if pool: + shape[pooled[s["out"]]] = (H // 2, W // 2, cout) + outs.append(pooled[s["out"]]) + args = ["@" + s["in"], "@" + wid, "@" + bid, "@" + s["out"], "@" + (outs[-1]), cin, cout, H, W, k, + K.ORDER_CODE[s["chunks"]], 1 if s["act"] == "leaky" else 0, 1 if pool else 0, tx, ty] + steps.append(("fg_mconv%d" % mt, tx * ty * (cout // (16 * mt)), args, outs, "conv")) + elif op == "upscale": + hi, wi, C = shape[s["in"]] + ho, wo, _ = shape[s["like"]] + shape[s["out"]] = (ho, wo, C) + n = ho * wo * C + steps.append(("fg_upscale_hwc_f16", -(-n // 256), + ["@" + s["in"], "@" + s["out"], C, hi, wi, ho, wo, K._ratio_bits(wi, wo), K._ratio_bits(hi, ho)], + [s["out"]], "upscale")) + elif op == "add": + a, b = s["in"] + shape[s["out"]] = shape[a] + n = int(np.prod(shape[a])) + steps.append(("fg_add_f16", -(-n // 256), ["@" + a, "@" + b, "@" + s["out"], n], [s["out"]], "add")) + for s in steps: + for o in s[3]: + acts[o] = (shape[o], 2 * int(np.prod(shape[o]))) + return data, acts, steps + + +def cells_hwc(cells): + return np.ascontiguousarray(np.asarray(cells, np.float64).astype(np.float16).transpose(1, 2, 0)).tobytes() + + +# ---- HIP ---------------------------------------------------------------------------------------------------------------- + +class HipNet: + def __init__(self, h, mods, data, acts, steps): + self.h, self.steps, self.acts = h, steps, acts + self.fn = {k: h.get_function(mods[MODULE_OF[k]], k) for k in set(s[0] for s in steps)} + self.ptr = {} + for i, b in data.items(): + self.ptr[i] = h.malloc(max(len(b), 4)) + h.upload(self.ptr[i], b) + for i, (_, n) in acts.items(): + self.ptr[i] = h.malloc(n + K.SLACK) + h.memset(self.ptr[i], K.HIP_SENTINEL, n + K.SLACK) + self.args = [[ctypes.c_void_p(self.ptr[a[1:]]) if isinstance(a, str) else ctypes.c_uint32(a) for a in s[2]] + for s in steps] + + def launch(self, only=None): + for i, s in enumerate(self.steps): + if only is None or i == only: + self.h.launch(self.fn[s[0]], s[1], 256, self.args[i]) + + def run(self): + self.launch() + self.h.sync() + return {i: bytes(self.h.download(self.ptr[i], n + K.SLACK)) for i, (_, n) in self.acts.items()} + + time_steps = G.HipNet.time_steps + time_network = G.HipNet.time_network + free = G.HipNet.free + + +def load_modules(h, build): + return {co: h.load_module(open(os.path.join(build, co), "rb").read()) for co in (CO_MMA, CO_NET1)} + + +# ---- D3D12 chains -------------------------------------------------------------------------------------------------------- + +def chain_lines(name, files, acts, steps, half=None, stop=None, saves=None, time_runs=0): + lines = ["chain " + name, "module " + CO_MMA, "module " + CO_NET1] + for i, f in files.items(): + lines.append("buf %s %d %s" % (i, os.path.getsize(f["path"]), f["name"])) + for i, (_, n) in acts.items(): + lines.append("buf %s %d - %d" % (i, n + K.SLACK, K.D3D12_SENTINEL)) + for j, (kernel, grid, args, _, _) in enumerate(steps): + if stop is not None and j > stop: + break + g = max(1, grid // 2) if j == half else grid + lines.append("step %s %d 256 %s" % (kernel, g, " ".join(str(a) for a in args))) + for i in (saves if saves is not None else acts): + lines.append("save %s %s.%s" % (i, name, i)) + if time_runs: + lines.append("time %d" % time_runs) + return lines + [""] + + +# ---- report against the CPU reference ------------------------------------------------------------------------------------ + +def step_report(got_bits, want): + """got: float16 bits; want: float64 values (float16-valued). Largest finite distance in float16 steps, counts over + 1 and over 4 steps, non-finite disagreements.""" + w = np.asarray(want, np.float64).astype(np.float16).view(np.uint16).reshape(-1) + g = np.asarray(got_bits, np.uint16).reshape(-1) + + def key(b): + b = b.astype(np.int64) + return np.where(b & 0x8000, -(b & 0x7FFF), b & 0x7FFF) + fin = ((g & 0x7C00) != 0x7C00) & ((w & 0x7C00) != 0x7C00) + d = np.where(fin, np.abs(key(g) - key(w)), 0) + gn = ((g & 0x7C00) == 0x7C00) & ((g & 0x3FF) != 0) + wn = ((w & 0x7C00) == 0x7C00) & ((w & 0x3FF) != 0) + nonfin_bad = ~fin & ~((g == w) | (gn & wn)) + return {"values": int(g.size), "steps_max": int(d.max()) if d.size else 0, "over_1_step": int(np.sum(d > 1)), + "over_4_steps": int(np.sum(d > 4)), "nonfinite_mismatch": int(np.sum(nonfin_bad))} + + +# ---- run -------------------------------------------------------------------------------------------------------------------- + +def run(build, rocm, model): + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + h = Hip(rocm) + mods = load_modules(h, build) + cdir = os.path.join(build, "replay-mma") + os.makedirs(cdir, exist_ok=True) + lines, runs, written = [], [], [] + need_mb = 256 + try: + for cname, values, params in G.cases(model): + cells = N.fold(values) + _, gh, gw = values.shape + H0, W0 = cells.shape[1:] + ref = N.run(cells, params) + name = cname + data, acts, steps = plan(H0, W0, params) + data = dict(cells=cells_hwc(cells), **data) + net = HipNet(h, mods, data, acts, steps) + try: + hip = net.run() + finally: + net.free() + files = G.write_files(cdir, name, data) + written += [f["name"] for f in files.values()] + ["%s.%s" % (name, i) for i in acts] + lines += chain_lines(name, files, acts, steps) + negs = {} + for kernel in sorted(set(s[0] for s in steps)): + j = next((j for j, s in enumerate(steps) if s[0] == kernel and s[1] >= 4), None) + if j is None: + continue + nn = "negative_%s_%s" % (name, kernel) + negs[kernel] = (nn, j, steps[j][3]) + lines += chain_lines(nn, files, acts, steps, half=j, stop=j, saves=steps[j][3]) + written += ["%s.%s" % (nn, o) for o in steps[j][3]] + need_mb = max(need_mb, G.arena_mb(data, acts)) + runs.append({"name": name, "case": cname, "acts": acts, "steps": steps, "hip": hip, + "ref": ref, "negs": negs, "dims": (H0, W0, gh, gw)}) + r, secs = G.replay(build, cdir, lines, need_mb, timeout=150) + + def read(fn): + p = os.path.join(cdir, fn) + return open(p, "rb").read() if os.path.exists(p) else b"" + + res = {"replay_exit": r.returncode, "replay_s": round(secs, 1), "replay_tail": r.stdout[-400:] + r.stderr[-200:], + "cases": []} + ok = r.returncode == 0 + for c in runs: + raw_bad, rep = [], {} + for out, (shape, n) in c["acts"].items(): + raw = K.compare_raw(read("%s.%s" % (c["name"], out)), c["hip"][out], n) + if raw["differ"] != 0 or not raw["slack_ok"]: + raw_bad.append({"buffer": out, **raw}) + got = np.frombuffer(c["hip"][out][:n], np.uint16).reshape(shape) # HWC + rep[out] = step_report(got.transpose(2, 0, 1), c["ref"][out]) + negs = {} + for kernel, (nn, j, outs) in c["negs"].items(): + diff = 0 + for o in outs: + _, n = c["acts"][o] + nraw = K.compare_raw(read("%s.%s" % (nn, o)), c["hip"][o], n) + diff += max(nraw["differ"], 0) + (0 if nraw["slack_ok"] else 1) + negs[kernel] = {"step": j, "caught": diff != 0, "differ": diff} + H0, W0, gh, gw = c["dims"] + shape, n = c["acts"]["out"] + o_hwc = np.frombuffer(c["hip"]["out"][:n], np.uint16).reshape(shape).astype(np.uint16) + o = o_hwc.transpose(2, 0, 1).view(np.float16).astype(np.float64) + px = N.unfold(o, gh, gw) + rpx = N.unfold(c["ref"]["out"], gh, gw) + out_rep = step_report(px.astype(np.float16).view(np.uint16), rpx) + per_value = [step_report(px[t].astype(np.float16).view(np.uint16), rpx[t]) for t in range(px.shape[0])] + good = not raw_bad and all(v["caught"] for v in negs.values()) and len(negs) >= 3 + ok = ok and good + res["cases"].append({"case": c["case"], "input_pixels": [gw, gh], "hip_equals_d3d12": not raw_bad, + "raw_differences": raw_bad[:5], "negative_controls": negs, + "vs_cpu_output": out_rep, "vs_cpu_output_per_value": per_value, "vs_cpu_buffers": rep, + "output_sha256": hashlib.sha256(px.astype("= MAX_LAUNCH_MS: + entry["skipped"] = "a launch is predicted at %.0f ms (limit %.0f ms)" % (predicted, MAX_LAUNCH_MS) + continue + data, acts, steps, _ = setup(w, hgt) + net = HipNet(h, mods, data, acts, steps) + try: + net.launch() + h.sync() + st = net.time_steps(2) + entry["max_launch_ms_measured"] = round(max(st), 3) + entry["per_op_ms"] = {op: round(sum(t for t, s in zip(st, steps) if s[4] == op), 3) + for op in ("conv", "upscale", "add")} + entry["hip_ms"] = round(net.time_network(reps), 3) + hip_out = net.run()["out"] + finally: + net.free() + name = "timing_%s_%s" % (acc, label) + files = G.write_files(cdir, name, data) + written = [f["name"] for f in files.values()] + ["%s.out" % name] + try: + r, _ = G.replay(build, cdir, chain_lines(name, files, acts, steps, saves=["out"], time_runs=reps), + G.arena_mb(data, acts), timeout=120) + entry["d3d12_exit"] = r.returncode + for line in r.stdout.splitlines(): + if line.startswith("chain %s: time" % name): + entry["d3d12_ms"] = float(line.split("time ")[1].split(" ms")[0]) + p = os.path.join(cdir, "%s.out" % name) + got = open(p, "rb").read() if (r.returncode == 0 and os.path.exists(p)) else b"" + entry["out_hip_equals_d3d12"] = bool(got) and K.compare_raw(got, hip_out, acts["out"][1])["differ"] == 0 + finally: + G.remove_written(cdir, written) + res["modes"][acc] = mode + res["method"] = ("HIP: device events around %d back-to-back network runs, mean per run (per-op: events around each " + "launch). D3D12: kernel_replay chain wall time from submission to completion, synchronised after each " + "of %d runs, mean per run. Synthetic weights, buffers resident, warm-up first. Information only." + % (reps, reps)) + return res + + +def prepare(build, rocm): + from clean.l.runner.l_d3d12_check import prepare_replay + from clean.runner import build as BLD + from clean.runner.hip import clean_process_environment + os.makedirs(build, exist_ok=True) + clean_process_environment(rocm) + clang = BLD.find_clang(rocm) + for src, co in (("fg_net1_mma.hip", CO_MMA), ("fg_net1.hip", CO_NET1)): + BLD.run(BLD.device_command(clang, rocm, os.path.join(ROOT, "clean", "fg", "kernels", src), + os.path.join(build, co)) + ["-I", os.path.join(ROOT, "clean", "kernels")]) + return prepare_replay(build, rocm, KERNELS) + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--timing", action="store_true") + ap.add_argument("--model", default=None) + ap.add_argument("--json", default=None) + a = ap.parse_args(argv) + if a.prepare: + prepare(a.build, a.rocm) + res = None + if a.run: + res = run(a.build, a.rocm, a.model) + print(json.dumps(res["summary"])) + for c in res["cases"]: + print(c["case"], "ok" if c["ok"] else "FAIL", "| hip == d3d12:", c["hip_equals_d3d12"], + "| negatives caught:", sum(v["caught"] for v in c["negative_controls"].values()), "of", + len(c["negative_controls"]), "| output vs cpu:", json.dumps(c["vs_cpu_output"])) + for d in c["raw_differences"]: + print(" raw difference", json.dumps(d)) + print("replay exit", res["replay_exit"], res["replay_tail"][-200:] if res["replay_exit"] else "") + elif a.timing: + res = timing(a.build, a.rocm) + print(json.dumps(res, indent=1)) + if res is not None and a.json: + with open(a.json, "w", encoding="utf-8", newline="\n") as f: + f.write(json.dumps(res, indent=1) + "\n") + if a.run: + return 0 if res["summary"]["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/fg/tests/test_fg_net1_mma_plan.py b/clean/fg/tests/test_fg_net1_mma_plan.py new file mode 100644 index 00000000..98492549 --- /dev/null +++ b/clean/fg/tests/test_fg_net1_mma_plan.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: MIT +"""net1's matrix fast path as planned on the CPU (clean/fg/runner/fg_net1_mma.py): the weight fragments, the fused +plan's buffers and launches.""" +import os +import sys +import unittest + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "clean", "fg", "runner")) + +import fg_net1_gpu as G # noqa: E402 +import fg_net1_mma as M # noqa: E402 +from clean.fg.reference import net1 as N # noqa: E402 + + +def chunk_tap(q, order, k, nb): + if order == "ascending": + return q, 0, 0 + if order == "block_kx_ky": + cb, t = divmod(q, k * k) + kx, ky = divmod(t, k) + return cb, ky, kx + kx, r = divmod(q, nb * k) + cb, ky = divmod(r, k) + return cb, ky, kx + + +class SwizzleTest(unittest.TestCase): + def test_every_fragment_element(self): + rng = np.random.default_rng(2) + for cout, k, cin, order in ((32, 3, 48, "block_kx_ky"), (256, 3, 128, "kx_block_ky"), (16, 1, 32, "ascending")): + w = rng.uniform(-1, 1, (cout, k, k, cin)).astype(np.float16) + nb, nq = cin // 16, (cin // 16) * k * k + f = np.frombuffer(M.swizzle(w, order), np.float16).reshape(cout // 16, nq, 32, 8) + for _ in range(500): + mt, q, lane, j = rng.integers(cout // 16), rng.integers(nq), rng.integers(32), rng.integers(8) + cb, ky, kx = chunk_tap(int(q), order, k, nb) + o = 16 * mt + lane % 16 + c = 16 * cb + 8 * (lane // 16) + j + self.assertEqual(f[mt, q, lane, j], w[o, ky, kx, c]) + + +class PlanTest(unittest.TestCase): + def test_buffers_and_launches(self): + params = G.synthetic_params(np.random.default_rng(3)) + for gh, gw in ((64, 64), (70, 90)): + cells = N.fold(np.zeros((10, gh, gw))) + H0, W0 = cells.shape[1:] + ref = N.run(cells, params) + data, acts, steps = M.plan(H0, W0, params) + self.assertEqual(len(steps), 27) # 32 minus the five fused pools + self.assertEqual(sum(s[4] == "conv" for s in steps), 17) + self.assertEqual(set(acts), set(ref) - {"cells"}) + for out, (shape, nbytes) in acts.items(): + self.assertEqual(tuple(np.array(ref[out].shape)[[1, 2, 0]]), shape, out) # HWC + self.assertEqual(nbytes, 2 * int(np.prod(shape))) + for kernel, grid, args, outs, op in steps: + self.assertIn(kernel, M.KERNELS) + if op == "conv": + cout, H, W, mt = args[6], args[7], args[8], int(kernel[8]) + self.assertEqual(grid, -(-W // 16) * -(-H // 16) * (cout // (16 * mt))) + self.assertEqual(len(args), 15) + pooled = [s[3] for s in steps if len(s[3]) == 2] + self.assertEqual(pooled, [["a0", "p1"], ["a1", "p2"], ["a2", "p3"], ["a3", "p4"], ["a4", "p5"]]) + + +if __name__ == "__main__": + unittest.main()