Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions clean/dx12/kernels/fg_probe.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's matrix probe (issue #96; clean/fg/kernels/fg_probe.hip, run by
// clean/fg/runner/fg_wmma_probe.py): fg_probe_mma16 (v_wmma_f16_16x16x16_f16 through the driver channel, format F16 ->
// F16) and fg_probe_mma32 (v_wmma_f32_16x16x16_f16, format F16 -> F32, with the float16 accumulator in between), the
// same fragments, memory layout and outputs as the HIP kernels. The exact reference (fg_probe_exact) runs on HIP only.
// Built only when --only names them (fg_ kernels are in no model's set).
//
// @wg-kernel fg_probe_mma16 co=fg_probe.co block=256 wave=32 args=p,p,p,p,p,u,u defines=FG_PROBE_FORM=16
// @wg-kernel fg_probe_mma32 co=fg_probe.co block=256 wave=32 args=p,p,p,p,p,u,u defines=FG_PROBE_FORM=32
#include "wg_platform.hlsli"

#define FG_FN
#include "../../fg/kernels/fg_probe_mid.h"

WG_ARGS_BEGIN
uint A, B, C, D, F; // pointers (arena byte offsets)
int tiles;
int L;
WG_ARGS_END

// this lane's A row (8 consecutive halves) and B column (8 halves, stride 16) of chunk l of tile t, as v8h (uint4)
void fg_probe_frags(uint t, uint l, uint lane, out uint4 fa, out uint4 fb)
{
const uint r = lane & 15u, h = lane >> 4;
const uint base = (t * (uint)L + l) * 512u;
fa = wg_ld128(A, base + (r * 16u + 8u * h) * 2u);
uint v[8];
[unroll] for (uint j = 0; j < 8; ++j) v[j] = wg_ld16(B, base + ((8u * h + j) * 16u + r) * 2u);
fb = uint4(v[0] | (v[1] << 16), v[2] | (v[3] << 16), v[4] | (v[5] << 16), v[6] | (v[7] << 16));
}

#if FG_PROBE_FORM == 16
WG_KERNEL(fg_probe_mma16)
{
WG_SETUP();
const uint lane = wg_lane_id() & 31u;
const uint t = wg_block_id() * 8u + (wg_lane_id() >> 5);
if (t >= (uint)tiles) return; // wave-uniform
const uint r = lane & 15u, h = lane >> 4;
uint c[8];
[unroll] for (uint v = 0; v < 8; ++v) c[v] = wg_ld16(C, (t * 256u + (8u * h + v) * 16u + r) * 2u);
uint4 acc = uint4(c[0] | (c[1] << 16), c[2] | (c[3] << 16), c[4] | (c[5] << 16), c[6] | (c[7] << 16));
for (uint l = 0; l < (uint)L; ++l) {
uint4 fa, fb;
fg_probe_frags(t, l, lane, fa, fb);
wg_frag d = wg_ags_mma(WG_AGS_F16_F16, wg_frag4(fa), wg_frag4(fb), wg_frag4(acc));
acc = uint4(d.v[0], d.v[1], d.v[2], d.v[3]);
}
const uint o[4] = { acc.x, acc.y, acc.z, acc.w };
[unroll] for (uint v = 0; v < 8; ++v)
wg_st16(D, (t * 256u + (8u * h + v) * 16u + r) * 2u, (o[v >> 1] >> ((v & 1u) * 16u)) & 0xffffu);
}
#else
WG_KERNEL(fg_probe_mma32)
{
WG_SETUP();
const uint lane = wg_lane_id() & 31u;
const uint t = wg_block_id() * 8u + (wg_lane_id() >> 5);
if (t >= (uint)tiles) return; // wave-uniform
const uint r = lane & 15u, h = lane >> 4;
uint acc[8], flag[8];
v8f d = wg_splat(0.0f);
[unroll] for (uint v = 0; v < 8; ++v) {
acc[v] = wg_ld16(C, (t * 256u + (8u * h + v) * 16u + r) * 2u);
flag[v] = 0u;
}
for (uint l = 0; l < (uint)L; ++l) {
uint4 fa, fb;
fg_probe_frags(t, l, lane, fa, fb);
v8f c;
[unroll] for (uint v = 0; v < 8; ++v) c.v[v] = wg_h2f(acc[v]);
d = wg_wmma_f16(fa, fb, c);
[unroll] for (uint v = 0; v < 8; ++v) {
flag[v] |= fg_mid16(asuint(d.v[v]));
acc[v] = wg_f2h(d.v[v]);
}
}
// Stores through two pointers at one per-lane offset were merged by the driver, keeping only the last (README
// "Driver exceptions"; seen here too: D kept its sentinel). Remedy: the pointers read through wg_uniform().
const uint pd = (uint)wg_uniform((int)D), pf = (uint)wg_uniform((int)F);
[unroll] for (uint v = 0; v < 8; ++v) wg_st32(pd, (t * 256u + (8u * h + v) * 16u + r) * 4u, asuint(d.v[v]));
[unroll] for (uint v = 0; v < 8; ++v) wg_st32(pf, (t * 256u + (8u * h + v) * 16u + r) * 4u, (flag[v] << 16) | acc[v]);
}
#endif
3 changes: 3 additions & 0 deletions clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula), `fg_upscale_f16` (bilinear, align corners, float16 steps) and `fg_add_f16` (the skip add); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl`, `fg_add.hlsl` |
| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values |
| `runner/fg_net1_gpu.py` | net1's 32 steps chained on the GPU (HIP, and D3D12 through kernel_replay's `chain` blocks): every step output on both back ends byte for byte and against the CPU reference bit for bit, a half-grid negative control per kernel, synthetic weights (also odd sizes and special values) and, with `--model`, a converter output file's weights (execute-only: its per-run files are deleted, digests only); `--timing` prints net1's milliseconds at 1080p- and 1440p-sized inputs as information, after a probe rules out long single launches |
| `runner/fg_wmma_probe.py`, `kernels/fg_probe.hip`, `kernels/fg_probe_mid.h`, `clean/dx12/kernels/fg_probe.hlsl` | probe of gfx12's 16 x 16 x 16 matrix instructions (float16 and float32 accumulate) against net1's chunk (RN16 of the accumulator plus 16 products, one rounding): random, adversarial and layer-chain classes, counts and step distances per form, mismatching examples; HIP and D3D12 byte for byte. Evidence only, no runtime change |
| `records/wmma-probe-2026-11-14.json` | the probe's record (HIP): neither form equals the exact chunk. On ordinary data (random, net1-like, the layer chains; 62.4 million outputs) float32 accumulate rounded to float16 per chunk is more than one step off in 1,216 outputs and more than four in 211; float16 accumulate in 6.05 million and 1.86 million |
| `tests/test_fg_wmma_probe.py` | the probe's CPU parts: the float16-midpoint test against brute force, the RN32 reference against exact fractions, the case classes |
| `tests/test_fg_net1_gpu_plan.py` | the chain as planned on the CPU: every step output's shape against the reference's buffers, launch arguments, natural weight order, chain lines |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
Expand Down
146 changes: 146 additions & 0 deletions clean/fg/kernels/fg_probe.hip
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// SPDX-License-Identifier: MIT
// Frame generation (issue #96): probe of gfx12's 16 x 16 x 16 matrix instructions against net1's chunk arithmetic
// (clean/fg/runner/fg_wmma_probe.py). A spec chunk is acc' = RN16(acc + sum of 16 float16 products), one rounding. One
// WMMA has K = 16 and adds C in the same instruction, so each of its 256 outputs is one chunk when C is the accumulator.
// The question: does the instruction form that sum exactly and round once?
//
// Memory, per tile t (one wave per tile): A [t][L][m][k], B [t][L][k][n] float16 (L chunks chained), C [t][m][n]
// float16 (the starting accumulator). Fragments (gfx12 wave32, as in clean/kernels/emu/wg_emu.cpp): lane l, h = l / 16:
// A row l % 16 and B column l % 16, input element j = K index 8 h + j; accumulator element v = row v + 8 h, column
// l % 16. A wrong guess would show as mismatches in nearly every random case.
//
// fg_probe_exact per output element, the exact chain (clean/fg/kernels/fg_exact16.h, the method of fp16acc.py):
// E16 = the float16 accumulator after L chunks; E32 = RN32 of the exact last chunk (the float32
// a perfect float32-accumulate instruction would give on it), NaN 0x7FC00000.
// fg_probe_mma16 v_wmma_f16_16x16x16_f16 chained L times (D fed back as C): D16 = the final accumulator.
// fg_probe_mma32 v_wmma_f32_16x16x16_f16 chained L times with the float16 accumulator in between (C = the
// accumulator widened exactly, the result rounded to float16 by v_cvt_f16_f32): D32 = the raw
// float32 result of the last chunk; F = (flag << 16) | final float16 accumulator, flag = some
// chunk's float32 result lay exactly on a float16 midpoint (fg_mid16: the only place where
// RN16(RN32(s)) can differ from RN16(s)).
// One shared signature (A, B, C, D, F, tiles, L); fg_probe_mma16 and fg_probe_exact do not use every argument.
#include "wg_platform.h"
#include "wg_mem.h"

typedef unsigned int uint;
typedef unsigned short u16;
#define FG_FN WG_FN
#define FG_I64 long long
#define FG_FBH32(v) (31u - (uint)__builtin_clz(v))
#include "fg_exact16.h"
#include "fg_probe_mid.h"

WG_FN uint fg_f2h(float f) { return (uint)__builtin_bit_cast(u16, (_Float16)f); } // round to nearest even
WG_FN float fg_h2f(uint h) { return (float)__builtin_bit_cast(_Float16, (u16)h); } // exact

// RN32 of the exact value of s (finite terms only): (hh * 2^40 + mid * 2^20 + ll) * 2^-48, nearest even. A non-zero
// value is at least 2^-48 in magnitude and below 2^90, so it is a normal float32. An exact zero is +0 (fg_round16's rule).
WG_FN uint fg_round32(FgAcc s) {
if (s.nan_ != 0u || (s.pinf != 0u && s.ninf != 0u)) return 0x7FC00000u;
if (s.pinf != 0u) return 0x7F800000u;
if (s.ninf != 0u) return 0xFF800000u;
__int128 t = ((__int128)s.hh << 40) + ((__int128)s.mid << 20) + (__int128)s.ll;
if (t == 0) return 0u;
const uint neg = t < 0 ? 1u : 0u;
unsigned __int128 mag = (unsigned __int128)(neg != 0u ? -t : t);
const unsigned long long hi = (unsigned long long)(mag >> 64), lo = (unsigned long long)mag;
const int e = hi != 0ull ? 64 + 63 - __builtin_clzll(hi) : 63 - __builtin_clzll(lo); // leading bit index
unsigned long long q;
int ex = e;
if (e <= 23) {
q = (unsigned long long)(mag << (23 - e));
} else {
const int sh = e - 23;
q = (unsigned long long)(mag >> sh);
const unsigned __int128 rem = mag & ((((unsigned __int128)1) << sh) - 1);
const unsigned __int128 half = ((unsigned __int128)1) << (sh - 1);
if (rem > half || (rem == half && (q & 1ull) != 0ull)) ++q;
if (q == (1ull << 24)) { q >>= 1; ++ex; }
}
const uint bexp = (uint)(ex - 48 + 127);
return (neg << 31) | (bexp << 23) | ((uint)q & 0x7FFFFFu);
}

WG_KERNEL_N(fg_probe_exact, 256)(const u16* __restrict__ A, const u16* __restrict__ B, const u16* __restrict__ C,
u16* __restrict__ E16, uint* __restrict__ E32, int tiles, int L) {
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= tiles * 256) return;
const int t = idx >> 8, m = (idx >> 4) & 15, n = idx & 15;
uint acc = C[idx];
uint e32 = 0u;
for (int l = 0; l < L; ++l) {
const u16* a = A + ((size_t)t * L + l) * 256 + m * 16;
const u16* b = B + ((size_t)t * L + l) * 256 + n;
FgAcc s = fg_acc_begin(acc);
for (int k = 0; k < 16; ++k) s = fg_acc_add(s, a[k], b[k * 16]);
if (l == L - 1) e32 = fg_round32(s);
acc = fg_acc_end(s);
}
E16[idx] = (u16)acc;
E32[idx] = e32;
}

// this lane's A row / B column fragments of chunk l of tile t
WG_FN void fg_probe_frags(const u16* A, const u16* B, int t, int l, int L, uint lane, v8h* fa, v8h* fb) {
const uint r = lane & 15u, h = lane >> 4;
const u16* a = A + ((size_t)t * L + l) * 256;
const u16* b = B + ((size_t)t * L + l) * 256;
for (int j = 0; j < 8; ++j) {
const uint k = 8u * h + (uint)j;
(*fa)[j] = __builtin_bit_cast(_Float16, a[r * 16u + k]);
(*fb)[j] = __builtin_bit_cast(_Float16, b[k * 16u + r]);
}
}

WG_KERNEL_N(fg_probe_mma16, 256)(const u16* __restrict__ A, const u16* __restrict__ B, const u16* __restrict__ C,
u16* __restrict__ D, uint* __restrict__ F, int tiles, int L) {
const uint lane = wg_lane_id() & 31u;
const int t = (int)(wg_block_id() * 8u + (wg_lane_id() >> 5));
if (t >= tiles) return; // wave-uniform
const uint r = lane & 15u, h = lane >> 4;
v8h acc;
for (int v = 0; v < 8; ++v) acc[v] = __builtin_bit_cast(_Float16, C[(size_t)t * 256 + (8u * h + v) * 16u + r]);
for (int l = 0; l < L; ++l) {
v8h fa, fb;
fg_probe_frags(A, B, t, l, L, lane, &fa, &fb);
acc = __builtin_amdgcn_wmma_f16_16x16x16_f16_w32_gfx12(fa, fb, acc);
}
// (an element is copied to a scalar first: __builtin_bit_cast of a subscripted vector element read element 0)
for (int v = 0; v < 8; ++v) {
const _Float16 x = acc[v];
D[(size_t)t * 256 + (8u * h + v) * 16u + r] = __builtin_bit_cast(u16, x);
}
}

WG_KERNEL_N(fg_probe_mma32, 256)(const u16* __restrict__ A, const u16* __restrict__ B, const u16* __restrict__ C,
uint* __restrict__ D, uint* __restrict__ F, int tiles, int L) {
const uint lane = wg_lane_id() & 31u;
const int t = (int)(wg_block_id() * 8u + (wg_lane_id() >> 5));
if (t >= tiles) return; // wave-uniform
const uint r = lane & 15u, h = lane >> 4;
uint acc[8], flag[8];
v8f d;
for (int v = 0; v < 8; ++v) {
acc[v] = C[(size_t)t * 256 + (8u * h + v) * 16u + r];
flag[v] = 0u;
d[v] = 0.0f;
}
for (int l = 0; l < L; ++l) {
v8h fa, fb;
fg_probe_frags(A, B, t, l, L, lane, &fa, &fb);
v8f c;
for (int v = 0; v < 8; ++v) c[v] = fg_h2f(acc[v]);
d = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32_gfx12(fa, fb, c);
for (int v = 0; v < 8; ++v) {
const float x = d[v]; // a scalar copy (see fg_probe_mma16)
flag[v] |= fg_mid16(__builtin_bit_cast(uint, x));
acc[v] = fg_f2h(x);
}
}
for (int v = 0; v < 8; ++v) {
const size_t o = (size_t)t * 256 + (8u * h + v) * 16u + r;
const float x = d[v];
D[o] = __builtin_bit_cast(uint, x);
F[o] = (flag[v] << 16) | acc[v];
}
}
26 changes: 26 additions & 0 deletions clean/fg/kernels/fg_probe_mid.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// SPDX-License-Identifier: MIT
// Frame generation (issue #96), the matrix probe (fg_probe.hip, clean/dx12/kernels/fg_probe.hlsl): is a float32 value
// (bits) exactly halfway between two adjacent float16 values? Only there can RN16(RN32(s)) differ from RN16(s) for an
// exact sum s, so a float32-accumulate path is exact once these values are redone exactly. Integer code, the same on
// both back ends; FG_FN is defined by the including file.
// float16 normal range (2^-14 <= |x| < 2^16, which includes 65520, the midpoint above 65504): the 13 low fraction
// bits are exactly 0x1000;
// float16 subnormal range (|x| < 2^-14): x = (odd integer) * 2^-25.
// A float32 sum of float16 products is never a float32 subnormal (its magnitude is 0 or at least 2^-48).
#ifndef FG_PROBE_MID_H
#define FG_PROBE_MID_H

FG_FN uint fg_mid16(uint x) {
const uint a = x & 0x7FFFFFFFu;
const uint e = a >> 23; // biased exponent
if (e == 255u || e == 0u) return 0u; // inf, NaN, zero (no float32 subnormals here)
if (e >= 113u) return (e <= 142u && (a & 0x1FFFu) == 0x1000u) ? 1u : 0u;
// x = M * 2^(e - 150), M = 1.f in 24 bits; x * 2^25 = M * 2^(e - 125) is an odd integer exactly when M's lowest
// set bit is bit s = 125 - e (s >= 13 here; s > 23 means x < 2^-25: never a midpoint)
const uint s = 125u - e;
if (s > 23u) return 0u;
const uint M = (a & 0x7FFFFFu) | 0x800000u;
return (M & ((2u << s) - 1u)) == (1u << s) ? 1u : 0u;
}

#endif
1 change: 1 addition & 0 deletions clean/fg/records/wmma-probe-2026-11-14.json

Large diffs are not rendered by default.

Loading
Loading