diff --git a/clean/dx12/kernels/fg_upscale.hlsl b/clean/dx12/kernels/fg_upscale.hlsl new file mode 100644 index 00000000..5014da64 --- /dev/null +++ b/clean/dx12/kernels/fg_upscale.hlsl @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 upscale (issue #96): fg_upscale_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip), +// byte for byte. Bilinear, align corners, x [C][hi][wi] -> y [C][ho][wo] float16. rx, ry: the ratios (wi - 1) / (wo - 1) +// and (hi - 1) / (ho - 1) as single-precision bits (0 for an extent of 1), from the caller, so no division runs here. +// Per axis: s = r * o, i0 = trunc(s), i1 = trunc(s + 1) (both n_in - 1 when i1 >= n_in), w1 = h(s - i0), +// w0 = h((i0 - s) + 1). top = h(h(wx0 P00) + h(wx1 P01)), bottom on row y1, out = h(h(wy0 top) + h(wy1 bottom)): each +// single-precision operation as written (one IEEE operation each, build_dx12.py's exact mode), each float16 rounding +// explicit (wg_f2h, nearest even). NaN results are written as 0x7E00. Built only when --only names it. +// +// @wg-kernel fg_upscale_f16 co=fg_net1.co block=256 wave=32 args=p,p,u,u,u,u,u,u,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint x, y; // pointers (arena byte offsets) + int C, hi, wi, ho, wo; + uint rx_bits, ry_bits; +WG_ARGS_END + +struct FgTap { int i0; int i1; float w0; float w1; }; + +FgTap fg_axis(float r, int o, int n_in) +{ + FgTap t; + const float s = r * (float)o; + t.i0 = (int)s; + t.i1 = (int)(s + 1.0f); + if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; } + t.w1 = wg_h2f(wg_f2h(s - (float)t.i0)); + t.w0 = wg_h2f(wg_f2h(((float)t.i0 - s) + 1.0f)); + return t; +} + +float fg_lerp16(float wa, float a, float wb, float b) +{ + return wg_h2f(wg_f2h(wg_h2f(wg_f2h(wa * a)) + wg_h2f(wg_f2h(wb * b)))); +} + +WG_KERNEL(fg_upscale_f16) +{ + WG_SETUP(); + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * ho * wo) return; + const int c = idx / (ho * wo); + const int rem = idx - c * (ho * wo); + const int oy = rem / wo; + const int ox = rem - oy * wo; + const FgTap tx = fg_axis(asfloat(rx_bits), ox, wi); + const FgTap ty = fg_axis(asfloat(ry_bits), oy, hi); + const uint base = (uint)(c * hi * wi); + const float p00 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i0 * wi + tx.i0)) * 2u)); + const float p01 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i0 * wi + tx.i1)) * 2u)); + const float p10 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i1 * wi + tx.i0)) * 2u)); + const float p11 = wg_h2f(wg_ld16(x, (base + (uint)(ty.i1 * wi + tx.i1)) * 2u)); + const float top = fg_lerp16(tx.w0, p00, tx.w1, p01); + const float bot = fg_lerp16(tx.w0, p10, tx.w1, p11); + uint outv = wg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot)); + if (fg_is_nan16(outv) != 0u) outv = FG_NAN16; + wg_st16(y, (uint)idx * 2u, outv); +} diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index c7f180ad..807660ae 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -107,6 +107,11 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c - `fg_maxpool2_f16(x, y, C, H, W)`: net1's max-pool exactly as `net1.maxpool2`: 2 x 2, stride 2, floor sizes, taps (0, 0), (0, 1), (1, 0), (1, 1) through the delivered step m <- (x > m) x + (x <= m) m with IEEE products and sums (0 * inf is NaN; a sum of zeros is -0 only when both are -0). +- `fg_upscale_f16(x, y, C, hi, wi, ho, wo, rx, ry)`: net1's upscale exactly as `net1.upscale_to`: bilinear, align + corners; rx / ry are the ratios (n_in - 1) / (n_out - 1) as correctly rounded single-precision bits from the caller + (0 for an extent of 1), so no division runs on the GPU. Positions in single precision one operation at a time + (HIP: no contraction); every product and sum rounded to float16 (a product of two float16 values is exact in single + precision, and single precision then float16 equals one rounding of a sum of two float16 values). - The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's and Model L's kernel sets and DXILs do not change. diff --git a/clean/fg/README.md b/clean/fg/README.md index 35923c77..0ac4df80 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -18,7 +18,7 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals | | `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only | | `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction | -| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) and `fg_maxpool2_f16` (2 x 2, the delivered formula); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl` | +| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula) and `fg_upscale_f16` (bilinear, align corners, float16 steps); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl` | | `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | diff --git a/clean/fg/kernels/fg_net1.hip b/clean/fg/kernels/fg_net1.hip index 4cf6ae5e..ff495e47 100644 --- a/clean/fg/kernels/fg_net1.hip +++ b/clean/fg/kernels/fg_net1.hip @@ -12,6 +12,12 @@ typedef unsigned short u16; #define FG_FBH32(v) (31u - (uint)__builtin_clz(v)) #include "fg_exact16.h" +// Single-precision steps are one IEEE operation each, as written (the upscale's positions): no contraction. +#pragma clang fp contract(off) + +WG_FN uint fg_f2h(float f) { return (uint)__builtin_bit_cast(u16, (_Float16)f); } // round to nearest even +WG_FN float fg_h2f(uint h) { return (float)__builtin_bit_cast(_Float16, (u16)h); } // exact + // fg_conv_f16: one convolution of net1. Stride 1, zero border (k = 3) or none (k = 1), cross-correlation; the float16 // accumulator starts at the bias and takes 16-term chunks (16 input channels of one tap), each summed exactly and added // with one float16 rounding, in the chunk order `order`: 0 ascending (1 x 1: channel blocks in order), 1 channel block @@ -77,3 +83,45 @@ WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y m = fg_pool_step16(m, p[W + 1]); y[idx] = (u16)m; } + +// One axis of the align-corners rule: s = r * o (single precision), i0 = trunc(s), i1 = trunc(s + 1), both n_in - 1 when +// i1 >= n_in; weights w1 = h(s - i0), w0 = h((i0 - s) + 1) as float16 values. +struct FgTap { int i0, i1; float w0, w1; }; +WG_FN FgTap fg_axis(float r, int o, int n_in) { + FgTap t; + const float s = r * (float)o; + t.i0 = (int)s; + t.i1 = (int)(s + 1.0f); + if (t.i1 >= n_in) { t.i0 = n_in - 1; t.i1 = n_in - 1; } + t.w1 = fg_h2f(fg_f2h(s - (float)t.i0)); + t.w0 = fg_h2f(fg_f2h(((float)t.i0 - s) + 1.0f)); + return t; +} + +// h(h(wa * a) + h(wb * b)): float16 products (exact in single precision, then rounded) and a float16 sum (single +// precision then float16: the same as one rounding of the exact sum, 24 >= 2 * 11 + 2) +WG_FN float fg_lerp16(float wa, float a, float wb, float b) { + return fg_h2f(fg_f2h(fg_h2f(fg_f2h(wa * a)) + fg_h2f(fg_f2h(wb * b)))); +} + +// fg_upscale_f16: net1's bilinear upscale, align corners, x [C][hi][wi] -> y [C][ho][wo] float16. rx, ry: the ratios +// (wi - 1) / (wo - 1) and (hi - 1) / (ho - 1) as single-precision bits (0 for an extent of 1), computed by the caller: +// top = h(h(wx0 P[y0][x0]) + h(wx1 P[y0][x1])), bottom the same on row y1, out = h(h(wy0 top) + h(wy1 bottom)). +// NaN results are written as 0x7E00. Output element idx = (c * ho + oy) * wo + ox. +WG_KERNEL_N(fg_upscale_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y, int C, int hi, int wi, int ho, int wo, + uint rx_bits, uint ry_bits) { + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * ho * wo) return; + const int c = idx / (ho * wo); + const int rem = idx - c * (ho * wo); + const int oy = rem / wo; + const int ox = rem - oy * wo; + const FgTap tx = fg_axis(__builtin_bit_cast(float, rx_bits), ox, wi); + const FgTap ty = fg_axis(__builtin_bit_cast(float, ry_bits), oy, hi); + const u16* p = x + c * hi * wi; + const float top = fg_lerp16(tx.w0, fg_h2f(p[ty.i0 * wi + tx.i0]), tx.w1, fg_h2f(p[ty.i0 * wi + tx.i1])); + const float bot = fg_lerp16(tx.w0, fg_h2f(p[ty.i1 * wi + tx.i0]), tx.w1, fg_h2f(p[ty.i1 * wi + tx.i1])); + uint out = fg_f2h(fg_lerp16(ty.w0, top, ty.w1, bot)); + if (fg_is_nan16(out) != 0u) out = FG_NAN16; + y[idx] = (u16)out; +} diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py index 40ed7294..8028e1de 100644 --- a/clean/fg/runner/fg_d3d12_check.py +++ b/clean/fg/runner/fg_d3d12_check.py @@ -19,7 +19,8 @@ payload is not specified). Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps -of several sizes (odd ones too) and every 2 x 2 window over ten special values. Output: counts only, no timing. +of several sizes (odd ones too) and every 2 x 2 window over ten special values; fg_upscale_f16: net1's level pairs, +other sizes, an output extent of 1, equal sizes, special values. Output: counts only, no timing. """ import argparse import ctypes @@ -134,8 +135,41 @@ def _pool_case(kernel, x, label): "defined": 2 * n, "cpu": lambda: N.maxpool2(x.astype(np.float64))} +def _ratio_bits(n_in, n_out): + """The align-corners ratio (n_in - 1) / (n_out - 1) as single-precision bits (correctly rounded; 0 for n_out 1), as + net1.upscale_to computes it.""" + r = np.float32(n_in - 1) / np.float32(n_out - 1) if n_out > 1 else np.float32(0) + return int(np.array([r], np.float32).view(np.uint32)[0]) + + +def upscale_cases(kernel, rng): + """net1's level pairs (the coarse level is the fine one halved with floor), other odd sizes, an output extent of 1, + equal sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the input.""" + cases = [] + pairs = [((17, 33), (35, 67)), ((1, 2), (2, 4)), ((4, 8), (8, 16)), ((8, 16), (17, 33)), ((3, 4), (7, 9)), + ((1, 1), (2, 3)), ((5, 2), (11, 4)), ((2, 3), (2, 3)), ((1, 3), (1, 7)), ((1, 1), (1, 1))] + for (hi, wi), (ho, wo) in pairs: + for C in (1, 32): + x = _f16(rng.uniform(-3, 3, (C, hi, wi))) + cases.append(_upscale_case(kernel, x, ho, wo, f"{C}x{hi}x{wi} -> {ho}x{wo}")) + x = _f16(rng.uniform(-3, 3, (16, 9, 13))) + f = rng.random(x.shape) < 0.08 + x[f] = rng.choice(SPECIALS, int(f.sum())) + cases.append(_upscale_case(kernel, x, 19, 27, "specials 9x13 -> 19x27")) + return cases + + +def _upscale_case(kernel, x, ho, wo, label): + C, hi, wi = x.shape + n = C * ho * wo + return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label, + "args": [("buf", x.tobytes()), ("out", 2 * n + SLACK), ("u32", C), ("u32", hi), ("u32", wi), ("u32", ho), + ("u32", wo), ("u32", _ratio_bits(wi, wo)), ("u32", _ratio_bits(hi, ho))], + "defined": 2 * n, "cpu": lambda: N.upscale_to(x.astype(np.float64), wo, ho)} + + KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True), - "fg_maxpool2_f16": pool_cases} + "fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases} # ---- prepare --------------------------------------------------------------------------------------------------------