Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions clean/dx12/kernels/fg_upscale.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's net1 upscale (issue #96): fg_upscale_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip),
// byte for byte. Bilinear, align corners, x [C][hi][wi] -> y [C][ho][wo] float16. rx, ry: the ratios (wi - 1) / (wo - 1)
// and (hi - 1) / (ho - 1) as single-precision bits (0 for an extent of 1), from the caller, so no division runs here.
// Per axis: s = r * o, i0 = trunc(s), i1 = trunc(s + 1) (both n_in - 1 when i1 >= n_in), w1 = h(s - i0),
// w0 = h((i0 - s) + 1). top = h(h(wx0 P00) + h(wx1 P01)), bottom on row y1, out = h(h(wy0 top) + h(wy1 bottom)): each
// single-precision operation as written (one IEEE operation each, build_dx12.py's exact mode), each float16 rounding
// explicit (wg_f2h, nearest even). NaN results are written as 0x7E00. Built only when --only names it.
//
// @wg-kernel fg_upscale_f16 co=fg_net1.co block=256 wave=32 args=p,p,u,u,u,u,u,u,u
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

WG_ARGS_BEGIN
uint x, y; // pointers (arena byte offsets)
int C, hi, wi, ho, wo;
uint rx_bits, ry_bits;
WG_ARGS_END

struct FgTap { int i0; int i1; float w0; float w1; };

FgTap fg_axis(float r, int o, int n_in)
{
FgTap t;
const float s = r * (float)o;
t.i0 = (int)s;
t.i1 = (int)(s + 1.0f);
if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; }
t.w1 = wg_h2f(wg_f2h(s - (float)t.i0));
t.w0 = wg_h2f(wg_f2h(((float)t.i0 - s) + 1.0f));
return t;
}

float fg_lerp16(float wa, float a, float wb, float b)
{
return wg_h2f(wg_f2h(wg_h2f(wg_f2h(wa * a)) + wg_h2f(wg_f2h(wb * b))));
}

WG_KERNEL(fg_upscale_f16)
{
WG_SETUP();
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= C * ho * wo) return;
const int c = idx / (ho * wo);
const int rem = idx - c * (ho * wo);
const int oy = rem / wo;
const int ox = rem - oy * wo;
const FgTap tx = fg_axis(asfloat(rx_bits), ox, wi);
const FgTap ty = fg_axis(asfloat(ry_bits), oy, hi);
const uint base = (uint)(c * hi * wi);
const float p00 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i0 * wi + tx.i0)) * 2u));
const float p01 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i0 * wi + tx.i1)) * 2u));
const float p10 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i1 * wi + tx.i0)) * 2u));
const float p11 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i1 * wi + tx.i1)) * 2u));
const float top = fg_lerp16(tx.w0, p00, tx.w1, p01);
const float bot = fg_lerp16(tx.w0, p10, tx.w1, p11);
uint outv = wg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot));
if (fg_is_nan16(outv) != 0u) outv = FG_NAN16;
wg_st16(y, (uint)idx * 2u, outv);
}
5 changes: 5 additions & 0 deletions clean/fg/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,11 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c
- `fg_maxpool2_f16(x, y, C, H, W)`: net1's max-pool exactly as `net1.maxpool2`: 2 x 2, stride 2, floor sizes, taps
(0, 0), (0, 1), (1, 0), (1, 1) through the delivered step m <- (x > m) x + (x <= m) m with IEEE products and sums
(0 * inf is NaN; a sum of zeros is -0 only when both are -0).
- `fg_upscale_f16(x, y, C, hi, wi, ho, wo, rx, ry)`: net1's upscale exactly as `net1.upscale_to`: bilinear, align
corners; rx / ry are the ratios (n_in - 1) / (n_out - 1) as correctly rounded single-precision bits from the caller
(0 for an extent of 1), so no division runs on the GPU. Positions in single precision one operation at a time
(HIP: no contraction); every product and sum rounded to float16 (a product of two float16 values is exact in single
precision, and single precision then float16 equals one rounding of a sum of two float16 values).
- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN
results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's
and Model L's kernel sets and DXILs do not change.
Expand Down
2 changes: 1 addition & 1 deletion clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals |
| `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only |
| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) and `fg_maxpool2_f16` (2 x 2, the delivered formula); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl` |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula) and `fg_upscale_f16` (bilinear, align corners, float16 steps); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl` |
| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
Expand Down
48 changes: 48 additions & 0 deletions clean/fg/kernels/fg_net1.hip
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,12 @@ typedef unsigned short u16;
#define FG_FBH32(v) (31u - (uint)__builtin_clz(v))
#include "fg_exact16.h"

// Single-precision steps are one IEEE operation each, as written (the upscale's positions): no contraction.
#pragma clang fp contract(off)

WG_FN uint fg_f2h(float f) { return (uint)__builtin_bit_cast(u16, (_Float16)f); } // round to nearest even
WG_FN float fg_h2f(uint h) { return (float)__builtin_bit_cast(_Float16, (u16)h); } // exact

// fg_conv_f16: one convolution of net1. Stride 1, zero border (k = 3) or none (k = 1), cross-correlation; the float16
// accumulator starts at the bias and takes 16-term chunks (16 input channels of one tap), each summed exactly and added
// with one float16 rounding, in the chunk order `order`: 0 ascending (1 x 1: channel blocks in order), 1 channel block
Expand Down Expand Up @@ -77,3 +83,45 @@ WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y
m = fg_pool_step16(m, p[W + 1]);
y[idx] = (u16)m;
}

// One axis of the align-corners rule: s = r * o (single precision), i0 = trunc(s), i1 = trunc(s + 1), both n_in - 1 when
// i1 >= n_in; weights w1 = h(s - i0), w0 = h((i0 - s) + 1) as float16 values.
struct FgTap { int i0, i1; float w0, w1; };
WG_FN FgTap fg_axis(float r, int o, int n_in) {
FgTap t;
const float s = r * (float)o;
t.i0 = (int)s;
t.i1 = (int)(s + 1.0f);
if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; }
t.w1 = fg_h2f(fg_f2h(s - (float)t.i0));
t.w0 = fg_h2f(fg_f2h(((float)t.i0 - s) + 1.0f));
return t;
}

// h(h(wa * a) + h(wb * b)): float16 products (exact in single precision, then rounded) and a float16 sum (single
// precision then float16: the same as one rounding of the exact sum, 24 >= 2 * 11 + 2)
WG_FN float fg_lerp16(float wa, float a, float wb, float b) {
return fg_h2f(fg_f2h(fg_h2f(fg_f2h(wa * a)) + fg_h2f(fg_f2h(wb * b))));
}

// fg_upscale_f16: net1's bilinear upscale, align corners, x [C][hi][wi] -> y [C][ho][wo] float16. rx, ry: the ratios
// (wi - 1) / (wo - 1) and (hi - 1) / (ho - 1) as single-precision bits (0 for an extent of 1), computed by the caller:
// top = h(h(wx0 P[y0][x0]) + h(wx1 P[y0][x1])), bottom the same on row y1, out = h(h(wy0 top) + h(wy1 bottom)).
// NaN results are written as 0x7E00. Output element idx = (c * ho + oy) * wo + ox.
WG_KERNEL_N(fg_upscale_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y, int C, int hi, int wi, int ho, int wo,
uint rx_bits, uint ry_bits) {
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= C * ho * wo) return;
const int c = idx / (ho * wo);
const int rem = idx - c * (ho * wo);
const int oy = rem / wo;
const int ox = rem - oy * wo;
const FgTap tx = fg_axis(__builtin_bit_cast(float, rx_bits), ox, wi);
const FgTap ty = fg_axis(__builtin_bit_cast(float, ry_bits), oy, hi);
const u16* p = x + c * hi * wi;
const float top = fg_lerp16(tx.w0, fg_h2f(p[ty.i0 * wi + tx.i0]), tx.w1, fg_h2f(p[ty.i0 * wi + tx.i1]));
const float bot = fg_lerp16(tx.w0, fg_h2f(p[ty.i1 * wi + tx.i0]), tx.w1, fg_h2f(p[ty.i1 * wi + tx.i1]));
uint out = fg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot));
if (fg_is_nan16(out) != 0u) out = FG_NAN16;
y[idx] = (u16)out;
}
38 changes: 36 additions & 2 deletions clean/fg/runner/fg_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@
payload is not specified).
Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several
sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps
of several sizes (odd ones too) and every 2 x 2 window over ten special values. Output: counts only, no timing.
of several sizes (odd ones too) and every 2 x 2 window over ten special values; fg_upscale_f16: net1's level pairs,
other sizes, an output extent of 1, equal sizes, special values. Output: counts only, no timing.
"""
import argparse
import ctypes
Expand Down Expand Up @@ -134,8 +135,41 @@ def _pool_case(kernel, x, label):
"defined": 2 * n, "cpu": lambda: N.maxpool2(x.astype(np.float64))}


def _ratio_bits(n_in, n_out):
"""The align-corners ratio (n_in - 1) / (n_out - 1) as single-precision bits (correctly rounded; 0 for n_out 1), as
net1.upscale_to computes it."""
r = np.float32(n_in - 1) / np.float32(n_out - 1) if n_out > 1 else np.float32(0)
return int(np.array([r], np.float32).view(np.uint32)[0])


def upscale_cases(kernel, rng):
"""net1's level pairs (the coarse level is the fine one halved with floor), other odd sizes, an output extent of 1,
equal sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the input."""
cases = []
pairs = [((17, 33), (35, 67)), ((1, 2), (2, 4)), ((4, 8), (8, 16)), ((8, 16), (17, 33)), ((3, 4), (7, 9)),
((1, 1), (2, 3)), ((5, 2), (11, 4)), ((2, 3), (2, 3)), ((1, 3), (1, 7)), ((1, 1), (1, 1))]
for (hi, wi), (ho, wo) in pairs:
for C in (1, 32):
x = _f16(rng.uniform(-3, 3, (C, hi, wi)))
cases.append(_upscale_case(kernel, x, ho, wo, f"{C}x{hi}x{wi} -> {ho}x{wo}"))
x = _f16(rng.uniform(-3, 3, (16, 9, 13)))
f = rng.random(x.shape) < 0.08
x[f] = rng.choice(SPECIALS, int(f.sum()))
cases.append(_upscale_case(kernel, x, 19, 27, "specials 9x13 -> 19x27"))
return cases


def _upscale_case(kernel, x, ho, wo, label):
C, hi, wi = x.shape
n = C * ho * wo
return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label,
"args": [("buf", x.tobytes()), ("out", 2 * n + SLACK), ("u32", C), ("u32", hi), ("u32", wi), ("u32", ho),
("u32", wo), ("u32", _ratio_bits(wi, wo)), ("u32", _ratio_bits(hi, ho))],
"defined": 2 * n, "cpu": lambda: N.upscale_to(x.astype(np.float64), wo, ho)}


KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True),
"fg_maxpool2_f16": pool_cases}
"fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases}


# ---- prepare --------------------------------------------------------------------------------------------------------
Expand Down
Loading