Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 104 additions & 0 deletions clean/dx12/kernels/fg_mconv.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's matrix convolution (issue #96; tier B of the net1 fast path): fg_mconv<MT> of
// fg_net1_mma.co (clean/fg/kernels/fg_net1_mma.hip), byte-identical to HIP: the same fragments and chunk order,
// v_wmma_f32_16x16x16_f16 through the driver channel (wg_wmma_f16) with the float16 accumulator as C, its result rounded
// to float16 after every chunk (wg_f2h = v_cvt_f16_f32, nearest even), the exact integer leaky ReLU and pool
// (fg_exact16.h). Driver rules (clean/dx12/README.md): every load of a chunk before its first matrix call; branchless
// selects; the chunk loop bound through an opaque zero (no forced unrolling); store pointers through wg_uniform().
// Built only when --only names the kernels (fg_ kernels are in no model's set).
//
// @wg-kernel fg_mconv1 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=1
// @wg-kernel fg_mconv2 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=2
// @wg-kernel fg_mconv4 co=fg_net1_mma.co entry=fg_mconv block=256 wave=32 args=p,p,p,p,p,u,u,u,u,u,u,u,u,u,u defines=FG_MT=4
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

#ifndef FG_MT
#define FG_MT 4
#endif

WG_ARGS_BEGIN
uint x, wf, b, y, p; // pointers (arena byte offsets)
int cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y;
WG_ARGS_END

uint fg_opq0() { return wg_grid_ >> 31; } // an opaque zero (the grid size is below 2^31)

uint4 fg_pack8(uint r[8]) { return uint4(r[0] | (r[1] << 16), r[2] | (r[3] << 16), r[4] | (r[5] << 16), r[6] | (r[7] << 16)); }

WG_KERNEL(fg_mconv)
{
WG_SETUP();
const uint lane = wg_lane_id() & 31u, wave = wg_lane_id() >> 5;
const int blk = (int)wg_block_id();
const int tx = blk % tiles_x, rest = blk / tiles_x;
const int ty = rest % tiles_y, mg = rest / tiles_y;
const int n = (int)(lane & 15u), hh = (int)(lane >> 4);
const int px = tx * 16 + n;
const int py0 = ty * 16 + 2 * (int)wave;
const int pad = k / 2, nb = cin / 16, taps = k * k, nq = nb * taps;
const uint py = (uint)wg_uniform((int)y), pp = (uint)wg_uniform((int)p);

v8f a32[FG_MT][2];
[unroll] for (int mt = 0; mt < FG_MT; ++mt) {
const int o0 = (mg * FG_MT + mt) * 16 + 8 * hh;
[unroll] for (uint v = 0; v < 8; ++v) {
const float bv = wg_h2f(wg_ld16(b, (uint)(o0 + (int)v) * 2u));
a32[mt][0].v[v] = bv;
a32[mt][1].v[v] = bv;
}
}
const int nq_loop = nq + (int)fg_opq0();
for (int q = 0; q < nq_loop; ++q) {
// the chunk's (channel block, ky, kx) in the layer's order, branchless
const int t1 = q % taps, cb1 = q / taps;
const int kx2 = q / (nb * k), r2 = q % (nb * k);
const int cb = select(order == 0, q, select(order == 1, cb1, r2 / k));
const int kx = select(order == 0, 0, select(order == 1, t1 / k, kx2));
const int ky = select(order == 0, 0, select(order == 1, t1 % k, r2 % k));
uint4 fb[2], fa[FG_MT];
const int sx = px + kx - pad;
[unroll] for (int nt = 0; nt < 2; ++nt) {
const int sy = py0 + nt + ky - pad;
const bool inb = and(and(sx >= 0, sx < W), and(sy >= 0, sy < H));
const uint e = select(inb, (uint)((sy * W + sx) * cin + cb * 16 + 8 * hh), 0u);
const uint4 u = wg_ld128(x, e * 2u);
fb[nt] = uint4(select(inb, u.x, 0u), select(inb, u.y, 0u), select(inb, u.z, 0u), select(inb, u.w, 0u));
}
[unroll] for (int mt2 = 0; mt2 < FG_MT; ++mt2)
fa[mt2] = wg_ld128(wf, (uint)((((mg * FG_MT + mt2) * nq + q) * 32 + (int)lane) * 16));
[unroll] for (int mt3 = 0; mt3 < FG_MT; ++mt3)
[unroll] for (int nt2 = 0; nt2 < 2; ++nt2) {
const v8f d = wg_wmma_f16(fa[mt3], fb[nt2], a32[mt3][nt2]);
[unroll] for (uint v3 = 0; v3 < 8; ++v3) a32[mt3][nt2].v[v3] = wg_h2f(wg_f2h(d.v[v3]));
}
}
[unroll] for (int mt4 = 0; mt4 < FG_MT; ++mt4) {
const int o0 = (mg * FG_MT + mt4) * 16 + 8 * hh;
uint r0[8], r1[8];
[unroll] for (uint v4 = 0; v4 < 8; ++v4) {
const uint h0 = wg_f2h(a32[mt4][0].v[v4]), h1 = wg_f2h(a32[mt4][1].v[v4]);
r0[v4] = select(act == 1, fg_leaky16(h0), h0);
r1[v4] = select(act == 1, fg_leaky16(h1), h1);
}
if (px < W && py0 < H) wg_st128(py, (uint)((py0 * W + px) * cout + o0) * 2u, fg_pack8(r0));
if (px < W && py0 + 1 < H) wg_st128(py, (uint)(((py0 + 1) * W + px) * cout + o0) * 2u, fg_pack8(r1));
if (pool == 1) {
uint m[8];
[unroll] for (uint v5 = 0; v5 < 8; ++v5) {
const uint x01 = wg_xor_u(r0[v5], 1u), x11 = wg_xor_u(r1[v5], 1u);
uint t = r0[v5];
t = fg_pool_step16(t, x01);
t = fg_pool_step16(t, r1[v5]);
t = fg_pool_step16(t, x11);
m[v5] = t;
}
const int Ho = H / 2, Wo = W / 2, ox = px / 2, oy = py0 / 2;
if ((n & 1) == 0 && ox < Wo && oy < Ho) wg_st128(pp, (uint)((oy * Wo + ox) * cout + o0) * 2u, fg_pack8(m));
}
}
}
59 changes: 59 additions & 0 deletions clean/dx12/kernels/fg_upscale_hwc.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's net1 upscale on NHWC tensors (issue #96): fg_upscale_hwc_f16 of fg_net1_mma.co
// (clean/fg/kernels/fg_net1_mma.hip), byte for byte; the arithmetic of fg_upscale.hlsl (align corners, float16 steps,
// NaN written as 0x7E00), element (y, x, c) at (y * W + x) * C + c. Built only when --only names it.
//
// @wg-kernel fg_upscale_hwc_f16 co=fg_net1_mma.co block=256 wave=32 args=p,p,u,u,u,u,u,u,u
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

WG_ARGS_BEGIN
uint x, y; // pointers (arena byte offsets)
int C, hi, wi, ho, wo;
uint rx_bits, ry_bits;
WG_ARGS_END

struct FgTap { int i0; int i1; float w0; float w1; };

FgTap fg_axis(float r, int o, int n_in)
{
FgTap t;
const float s = r * (float)o;
t.i0 = (int)s;
t.i1 = (int)(s + 1.0f);
if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; }
t.w1 = wg_h2f(wg_f2h(s - (float)t.i0));
t.w0 = wg_h2f(wg_f2h(((float)t.i0 - s) + 1.0f));
return t;
}

float fg_lerp16(float wa, float a, float wb, float b)
{
return wg_h2f(wg_f2h(wg_h2f(wg_f2h(wa * a)) + wg_h2f(wg_f2h(wb * b))));
}

WG_KERNEL(fg_upscale_hwc_f16)
{
WG_SETUP();
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= C * ho * wo) return;
const int c = idx % C;
const int pix = idx / C;
const int oy = pix / wo;
const int ox = pix - oy * wo;
const FgTap tx = fg_axis(asfloat(rx_bits), ox, wi);
const FgTap ty = fg_axis(asfloat(ry_bits), oy, hi);
const float p00 = wg_h2f(wg_ld16(x, (uint)((ty.i0 * wi + tx.i0) * C + c) * 2u));
const float p01 = wg_h2f(wg_ld16(x, (uint)((ty.i0 * wi + tx.i1) * C + c) * 2u));
const float p10 = wg_h2f(wg_ld16(x, (uint)((ty.i1 * wi + tx.i0) * C + c) * 2u));
const float p11 = wg_h2f(wg_ld16(x, (uint)((ty.i1 * wi + tx.i1) * C + c) * 2u));
const float top = fg_lerp16(tx.w0, p00, tx.w1, p01);
const float bot = fg_lerp16(tx.w0, p10, tx.w1, p11);
uint outv = wg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot));
if (fg_is_nan16(outv) != 0u) outv = FG_NAN16;
wg_st16(y, (uint)idx * 2u, outv);
}
20 changes: 19 additions & 1 deletion clean/fg/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,25 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c
results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's
and Model L's kernel sets and DXILs do not change.
- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference;
`runner/fg_net1_gpu.py` the whole network, every step output.
`runner/fg_net1_gpu.py` the whole network, every step output. These exact kernels stay as the validation mode.

## Fast path, tier B (`kernels/fg_net1_mma.hip`, code object `fg_net1_mma.co`; D3D12 `fg_mconv.hlsl`, `fg_upscale_hwc.hlsl`)

- Tensors NHWC float16 (element (y, x, c) at (y W + x) C + c), so 8 consecutive channels of a pixel are one 16-byte
fragment load or store. Weights as WMMA A fragments in chunk order: `[cout / 16][chunks][32 lanes][8]` float16, lane
l holding output channel 16 mt + l % 16 and input channels 16 cb + 8 (l / 16) + 0..7 of the chunk's tap
(`runner/fg_net1_mma.py` `swizzle`).
- `fg_mconv{1,2,4}(x, wf, b, y, p, cin, cout, H, W, k, order, act, pool, tiles_x, tiles_y)`: the spec's chunk structure
on `v_wmma_f32_16x16x16_f16`: the accumulator starts at the bias; each chunk (16 input channels of one tap, in the
layer's chunk order) is one instruction with the float16 accumulator as C, and its result is rounded to float16
(nearest even) before the next chunk. Not bit-exact: the instruction's sum is not the exact one
(`records/wmma-probe-2026-11-14.json`). An exact-zero sum gives +0, the reference's convention. Leaky ReLU and, with
`pool` 1, the 2 x 2 max-pool are the exact helpers of `fg_exact16.h`, in the epilogue. 1 / 2 / 4 output-channel tiles
of 16 per wave; a workgroup is 8 waves over 16 x 16 pixels.
- `fg_upscale_hwc_f16`: `fg_upscale_f16`'s arithmetic on NHWC tensors. The adds are `fg_add_f16`.
- net1 is 27 launches: five encoder convolutions with the pool fused, the bottleneck's two, and per decoder level an
upscale, an add and two convolutions. HIP and D3D12 are byte-identical; against the CPU reference the output is a
report (net1's output is not gated; the gate is the generated frame against the oracle).

## Configuration (`reference/config.py`)

Expand Down
4 changes: 4 additions & 0 deletions clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `records/wmma-probe-2026-11-14.json` | the probe's record (HIP): neither form equals the exact chunk. On ordinary data (random, net1-like, the layer chains; 62.4 million outputs) float32 accumulate rounded to float16 per chunk is more than one step off in 1,216 outputs and more than four in 211; float16 accumulate in 6.05 million and 1.86 million |
| `tests/test_fg_wmma_probe.py` | the probe's CPU parts: the float16-midpoint test against brute force, the RN32 reference against exact fractions, the case classes |
| `tests/test_fg_net1_gpu_plan.py` | the chain as planned on the CPU: every step output's shape against the reference's buffers, launch arguments, natural weight order, chain lines |
| `kernels/fg_net1_mma.hip`, `clean/dx12/kernels/fg_mconv.hlsl`, `fg_upscale_hwc.hlsl` | net1's fast path, tier B (code object `fg_net1_mma.co`): `fg_mconv1/2/4` (one `v_wmma_f32_16x16x16_f16` per spec chunk in the layer's chunk order, the float16 accumulator from the bias as C, the result rounded to float16 after every chunk; leaky ReLU and the fused 2 x 2 pool in the epilogue; NHWC float16) and `fg_upscale_hwc_f16`. Not bit-exact to the CPU reference (the matrix unit's chunk sum is not exact); HIP and D3D12 byte-identical |
| `runner/fg_net1_mma.py` | net1 on the fast path in 27 launches, HIP and D3D12: every buffer byte for byte between the back ends (sentinels, slack, a half-grid negative control per kernel; synthetic weights, odd sizes, special values, `--model` execute-only); against the CPU reference as a report (largest step distance, values over 1 and over 4 steps, per buffer and per output value); `--timing` at 1080p- and 1440p-sized inputs (information only, after a launch-length probe) |
| `records/net1-tierb-2026-11-14.json` | the fast path's record: the gate results, the report against the CPU reference (synthetic and converted weights, counts only) and the timing |
| `tests/test_fg_net1_mma_plan.py` | the fast path as planned on the CPU: every weight fragment element against the weights, chunk order and tap; buffers and launches of the fused plan |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
| `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` |
Expand Down
Loading
Loading