diff --git a/clean/dx12/README.md b/clean/dx12/README.md index 4a70db2b..79213c48 100644 --- a/clean/dx12/README.md +++ b/clean/dx12/README.md @@ -20,6 +20,7 @@ as its HIP kernel (`clean/dx12/compare.py` proves it per kernel on synthetic fra | `dxil_asm.cpp` | tool used by the build (assembles the rewritten DXIL listing) | | `compare.py` | HIP vs hybrid D3D12, every stage output / state buffer byte for byte | | `kernels/exposure.hlsl` | the reference port (`exposure_auto_diet`): copy its structure | +| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; check: `clean/fg/runner/fg_d3d12_check.py` | ## Arithmetic mode (why build_dx12.py and nothing else) diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 6b951950..1327395d 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -5,7 +5,8 @@ python clean/dx12/build_dx12.py --out build/dx12-m/hip [--isa] [--only exposure_auto_diet,...] [-j N] [--model-l] Model L's kernels (#89: those that reproduce one of Model L's code objects, co=l_*.co, direct-route kernels named l_*, and MODEL_L_ONLY) are built only with --model-l -or when --only names them, so that a build without the option is Model M's kernel set unchanged. +or when --only names them, so that a build without the option is Model M's kernel set unchanged. Frame generation's +kernels (#96: co=fg_*.co or named fg_*) are built only when --only names them, never by a full build. Every kernel source declares its kernels in header lines (clean/dx12/README.md): @@ -186,13 +187,23 @@ def is_model_l(decl): return decl["co"].startswith("l_") or decl["name"].startswith("l_") or decl["name"] in MODEL_L_ONLY +def is_frame_gen(decl): + """A port of frame generation's kernels (#96: code objects fg_*.co, or named fg_*). They are groundwork that no model + build launches yet: built only when --only names them, never by a full build (with or without --model-l), so the + Model M and Model L kernel sets and their DXILs stay as they are.""" + return decl["co"].startswith("fg_") or decl["name"].startswith("fg_") + + def kernel_set(model_l=False): """(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's - with model_l. From the sources' declarations alone (nothing is compiled).""" + with model_l. From the sources' declarations alone (nothing is compiled). Frame generation's kernels are never in + it (is_frame_gen).""" keys = set() for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []: if fn.endswith(".hlsl"): for d in kernel_decls(os.path.join(KERNEL_DIR, fn)): + if is_frame_gen(d): + continue if model_l or not is_model_l(d): keys.add((d["name"], d["co"])) return sorted(keys) @@ -321,6 +332,8 @@ def main(argv=None): continue if not only and not a.model_l and is_model_l(d): continue + if not only and is_frame_gen(d): + continue key = (d["name"], d["co"]) if key in kernels and key not in old: raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice") diff --git a/clean/dx12/kernels/fg_conv.hlsl b/clean/dx12/kernels/fg_conv.hlsl new file mode 100644 index 00000000..30cc25aa --- /dev/null +++ b/clean/dx12/kernels/fg_conv.hlsl @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 convolution (issue #96): fg_conv_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip), +// byte for byte. The arithmetic is the shared integer code of clean/fg/kernels/fg_exact16.h (exact float16 chunk +// accumulation, one nearest-even rounding per chunk, the leaky ReLU), so both back ends run the same operations; only +// memory access differs (byte offsets into the arena here). +// +// One output element per lane: idx = (o * H + y) * W + x of [cout][H][W] float16; x [cin][H][W] float16; w natural +// [cout][cin][k][k] float16; b [cout] float16. Stride 1, zero border; chunk order: 0 ascending (1 x 1), 1 channel block, +// kx, ky; 2 kx, channel block, ky. act: 0 none, 1 leaky ReLU. Not part of Model M's or Model L's kernel set (fg_ kernels +// are built only when --only names them). +// +// @wg-kernel fg_conv_f16 co=fg_net1.co block=256 wave=32 args=p,p,p,p,u,u,u,u,u,u,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint x, w, b, y; // pointers (arena byte offsets) + int cin, cout, H, W, k, order, act; +WG_ARGS_END + +WG_KERNEL(fg_conv_f16) +{ + WG_SETUP(); + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= cout * H * W) return; + const int o = idx / (H * W); + const int rem = idx - o * (H * W); + const int yy = rem / W; + const int xx = rem - yy * W; + const int p = k / 2; + const int nb = cin / 16; + const int taps = k * k; + uint acc = wg_ld16(b, (uint)o * 2u); + [loop] for (int n = 0; n < nb * taps; ++n) { + int cb, kx, ky; + if (order == 0) { + cb = n; kx = 0; ky = 0; + } else if (order == 1) { + cb = n / taps; + const int t = n - cb * taps; + kx = t / k; + ky = t - kx * k; + } else { + kx = n / (nb * k); + const int r = n - kx * (nb * k); + cb = r / k; + ky = r - cb * k; + } + const int sy = yy + ky - p; + const int sx = xx + kx - p; + const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W; + FgAcc s = fg_acc_begin(acc); + [loop] for (int i = 0; i < 16; ++i) { + const int c = cb * 16 + i; + const uint wv = wg_ld16(w, (uint)(((o * cin + c) * k + ky) * k + kx) * 2u); + uint xv = 0u; // outside the grid: 0 (0 * inf is NaN) + if (inside) xv = wg_ld16(x, (uint)((c * H + sy) * W + sx) * 2u); + s = fg_acc_add(s, wv, xv); + } + acc = fg_acc_end(s); + } + if (act == 1) acc = fg_leaky16(acc); + wg_st16(y, (uint)idx * 2u, acc); +} diff --git a/clean/dx12/tests/test_kernel_sets.py b/clean/dx12/tests/test_kernel_sets.py index bdb87aa8..a9925bf1 100644 --- a/clean/dx12/tests/test_kernel_sets.py +++ b/clean/dx12/tests/test_kernel_sets.py @@ -48,6 +48,19 @@ def test_model_l_adds_only_model_l_kernels(self): self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)]) + def test_frame_gen_kernels_are_in_no_set(self): + """Frame generation's ports (#96, co=fg_*.co or named fg_*) are declared but enter neither set: only --only + builds them, so the Model M and Model L builds stay as they are.""" + decls = [] + for fn in sorted(os.listdir(B.KERNEL_DIR)): + if fn.endswith(".hlsl"): + decls += B.kernel_decls(os.path.join(B.KERNEL_DIR, fn)) + fg = [(d["name"], d["co"]) for d in decls if B.is_frame_gen(d)] + self.assertTrue(fg, "no frame-generation kernel declared") + full = set(B.kernel_set(True)) + self.assertFalse([k for k in fg if k in full]) + + if __name__ == "__main__": if "--write" in sys.argv: with open(SETS, "w", newline="\n") as f: diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index 1af55c55..e2cf3f3c 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -95,6 +95,20 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c - Not part of net1 and still open: what the 10 input values are made of (images A-D, the warps) and the input grid size relative to the output. +## GPU kernels (`kernels/`, code object `fg_net1.co`; D3D12 ports `clean/dx12/kernels/fg_*.hlsl`) + +- Groundwork: no runtime launches them yet. Tensors float16 channel-first `[c][h][w]`, weights natural + `[out][in][k][k]` float16 (the map's `W`), biases `[out]` float16; a 1-D grid of 256-lane workgroups, one output + element per lane. +- `fg_conv_f16(x, w, b, y, cin, cout, H, W, k, order, act)`: one net1 convolution exactly as `net1.conv16`: stride 1, + zero border, float16 accumulator from the bias, 16-term chunks each summed exactly and rounded once, chunk order + `order` (0 ascending, 1 channel block / kx / ky, 2 kx / channel block / ky), `act` 0 none or 1 leaky ReLU. cin is a + multiple of 16. +- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN + results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's + and Model L's kernel sets and DXILs do not change. +- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference. + ## Configuration (`reference/config.py`) - No configuration value changes the networks or their weights. diff --git a/clean/fg/README.md b/clean/fg/README.md index 871febfd..5441ced2 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -17,6 +17,9 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `reference/fp16acc.py` | exact float16 chunk accumulation: RN16(acc + 16 products) with integer arithmetic | | `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals | | `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only | +| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction | +| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none); D3D12 port `clean/dx12/kernels/fg_conv.hlsl` | +| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | | `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` | diff --git a/clean/fg/kernels/fg_exact16.h b/clean/fg/kernels/fg_exact16.h new file mode 100644 index 00000000..0e7af661 --- /dev/null +++ b/clean/fg/kernels/fg_exact16.h @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: MIT +// Frame generation (issue #96): exact float16 arithmetic in integers, shared by the HIP kernels (clean/fg/kernels/*.hip) +// and their D3D12 ports (clean/dx12/kernels/fg_*.hlsl), so both back ends compute the same bits by construction. +// +// The same method as the CPU reference clean/fg/reference/fp16acc.py: a finite float16 value is m * 2^-24 with an integer +// |m| < 2^40; m splits into a high and a low 20-bit part, so a 16-term dot product is exact in three 64-bit sums +// (high * high, high * low + low * high, low * low). The accumulator acc' = RN16(acc + sum of a_i * b_i) is rounded once, +// to nearest with ties to even, by integer arithmetic only. inf and NaN follow IEEE (any NaN, 0 * inf or inf - inf gives +// NaN, written as 0x7E00; otherwise an infinite term gives that infinity). +// +// The including file defines: +// FG_FN function qualifier (HIP: WG_FN; HLSL: nothing) +// FG_I64 signed 64-bit integer (HIP: long long; HLSL: int64_t) +// FG_FBH32(v) index of the highest set bit of a non-zero 32-bit unsigned value +#ifndef FG_EXACT16_H +#define FG_EXACT16_H + +#define FG_NAN16 0x7E00u +#define FG_LOW20 1048575 + +struct FgAcc { + FG_I64 hh; // sum of high * high (units 2^-8) + FG_I64 mid; // sum of high * low + low * high, plus 16 * m(acc) (units 2^-28) + FG_I64 ll; // sum of low * low (units 2^-48), never negative + uint nan_; // 1: the result is NaN + uint pinf; // 1: a +inf term + uint ninf; // 1: a -inf term +}; + +FG_FN uint fg_is_nan16(uint h) { return ((h & 0x7C00u) == 0x7C00u && (h & 0x3FFu) != 0u) ? 1u : 0u; } +FG_FN uint fg_is_inf16(uint h) { return ((h & 0x7FFFu) == 0x7C00u) ? 1u : 0u; } + +// floor(log2(v)) of a 64-bit value v > 0 +FG_FN uint fg_log2_64(FG_I64 v) { + const uint hi = (uint)(v >> 32); + const uint lo = (uint)v; // the low 32 bits + return hi != 0u ? 32u + FG_FBH32(hi) : FG_FBH32(lo); +} + +// a finite float16 (bits) -> m with value = m * 2^-24 +FG_FN FG_I64 fg_m16(uint h) { + const uint e = (h >> 10) & 31u; + const uint f = h & 1023u; + FG_I64 m = (FG_I64)f; + if (e != 0u) m = (FG_I64)(f | 1024u) * ((FG_I64)1 << (e - 1u)); + return (h & 0x8000u) != 0u ? -m : m; +} + +FG_FN FgAcc fg_acc_begin(uint acc) { + FgAcc s; + s.hh = 0; + s.mid = 0; + s.ll = 0; + s.nan_ = fg_is_nan16(acc); + s.pinf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) == 0u) ? 1u : 0u; + s.ninf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) != 0u) ? 1u : 0u; + if ((acc & 0x7C00u) != 0x7C00u) s.mid = fg_m16(acc) * 16; + return s; +} + +// s + a * b (a, b float16 bits) +FG_FN FgAcc fg_acc_add(FgAcc s, uint a, uint b) { + if (fg_is_nan16(a) != 0u || fg_is_nan16(b) != 0u) { + s.nan_ = 1u; + } else if (fg_is_inf16(a) != 0u || fg_is_inf16(b) != 0u) { + if ((a & 0x7FFFu) == 0u || (b & 0x7FFFu) == 0u) { + s.nan_ = 1u; // 0 * inf + } else if (((a ^ b) & 0x8000u) != 0u) { + s.ninf = 1u; + } else { + s.pinf = 1u; + } + } else { + const FG_I64 ma = fg_m16(a); + const FG_I64 mb = fg_m16(b); + const FG_I64 ah = ma >> 20, al = ma & FG_LOW20; + const FG_I64 bh = mb >> 20, bl = mb & FG_LOW20; + s.hh += ah * bh; + s.mid += ah * bl + al * bh; + s.ll += al * bl; + } + return s; +} + +// RN16 of the exact value (hh * 2^40 + mid * 2^20 + ll) * 2^-48, ll >= 0, as float16 bits +FG_FN uint fg_round16(FG_I64 hh, FG_I64 mid, FG_I64 ll) { + const FG_I64 lim = (FG_I64)1 << 30; + const uint big = (hh > lim || hh < -lim) ? 1u : 0u; // |value| >= 2^22: infinite + const FG_I64 hhc = hh > lim ? lim : (hh < -lim ? -lim : hh); + // value * 2^25 = hh * 2^17 + mid2 / 8 + llr / 2^23, with mid2 = mid + (ll >> 20) and 0 <= llr < 2^20 + const FG_I64 l1 = ll >> 20; + const FG_I64 llr = ll & FG_LOW20; + const FG_I64 mid2 = mid + l1; + const FG_I64 m1 = mid2 >> 3; // floor + const FG_I64 midr = mid2 & 7; + const FG_I64 q = hhc * 131072 + m1; // floor of value * 2^25 + const uint sticky = (midr != 0 || llr != 0) ? 1u : 0u; + const uint neg = q < 0 ? 1u : 0u; + FG_I64 mag = q; + if (neg != 0u) mag = sticky != 0u ? -q - 1 : -q; + // quantum in units of 2^-25: 2 below 2^-14 (subnormal), else 2^(e - 10) in the binade [2^e, 2^(e + 1)) units + const uint e = fg_log2_64(mag > 0 ? mag : 1); + const uint qexp = mag < 2048 ? 1u : e - 10u; + const FG_I64 quantum = (FG_I64)1 << qexp; + const FG_I64 r = mag & (quantum - 1); + const FG_I64 down = mag - r; + const FG_I64 half = quantum >> 1; + const uint odd = (uint)((down >> qexp) & 1); + const uint up = (r > half || (r == half && (sticky != 0u || odd != 0u))) ? 1u : 0u; + const FG_I64 res = down + (up != 0u ? quantum : 0); // the rounded magnitude, units 2^-25 + const uint sign = big != 0u ? (hh < 0 ? 1u : 0u) : neg; + uint bits; + if (big != 0u || res >= ((FG_I64)65536 << 25)) { + bits = 0x7C00u; + } else if (res < 2048) { + bits = (uint)(res >> 1); // subnormal (or zero) + } else { + const uint e2 = fg_log2_64(res); + bits = ((e2 - 10u) << 10) | ((uint)(res >> (e2 - 10u)) & 1023u); + } + return bits | (sign << 15); +} + +FG_FN uint fg_acc_end(FgAcc s) { + if (s.nan_ != 0u || (s.pinf != 0u && s.ninf != 0u)) return FG_NAN16; + if (s.pinf != 0u) return 0x7C00u; + if (s.ninf != 0u) return 0xFC00u; + return fg_round16(s.hh, s.mid, s.ll); +} + +// the order of a float16 value for max (-0 and +0 are equal); not for NaN +FG_FN int fg_key16(uint h) { return (h & 0x8000u) != 0u ? -(int)(h & 0x7FFFu) : (int)(h & 0x7FFFu); } + +// net1's activation: y = max(x, h(s * x)) with the delivered slope s (float16 0x211E); NaN stays NaN +#define FG_LEAKY_SLOPE16 0x211Eu +FG_FN uint fg_leaky16(uint x) { + if (fg_is_nan16(x) != 0u) return x; + const uint t = fg_acc_end(fg_acc_add(fg_acc_begin(0u), FG_LEAKY_SLOPE16, x)); + return fg_key16(t) > fg_key16(x) ? t : x; +} + +#endif diff --git a/clean/fg/kernels/fg_net1.hip b/clean/fg/kernels/fg_net1.hip new file mode 100644 index 00000000..8701536b --- /dev/null +++ b/clean/fg/kernels/fg_net1.hip @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT +// Frame generation (issue #96): HIP kernels of net1's operations (clean/fg/reference/net1.py), bit for bit against the +// CPU reference. D3D12 ports: clean/dx12/kernels/fg_*.hlsl. Tensors are float16, channel-first [c][h][w]; weights are +// natural [out][in][k][k] float16; biases [out] float16. 1-D grid of 256-lane workgroups, one output element per lane. +#include "wg_platform.h" +#include "wg_mem.h" + +typedef unsigned int uint; +typedef unsigned short u16; +#define FG_FN WG_FN +#define FG_I64 long long +#define FG_FBH32(v) (31u - (uint)__builtin_clz(v)) +#include "fg_exact16.h" + +// fg_conv_f16: one convolution of net1. Stride 1, zero border (k = 3) or none (k = 1), cross-correlation; the float16 +// accumulator starts at the bias and takes 16-term chunks (16 input channels of one tap), each summed exactly and added +// with one float16 rounding, in the chunk order `order`: 0 ascending (1 x 1: channel blocks in order), 1 channel block +// outer, then kx, then ky; 2 kx outer, then channel block, then ky. act: 0 none, 1 leaky ReLU (slope 0x211E). +// cin must be a multiple of 16. Output element idx = (o * H + y) * W + x. +WG_KERNEL_N(fg_conv_f16, 256)(const u16* __restrict__ x, const u16* __restrict__ w, const u16* __restrict__ b, + u16* __restrict__ y, int cin, int cout, int H, int W, int k, int order, int act) { + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= cout * H * W) return; + const int o = idx / (H * W); + const int rem = idx - o * (H * W); + const int yy = rem / W; + const int xx = rem - yy * W; + const int p = k / 2; + const int nb = cin / 16; + const int taps = k * k; + uint acc = b[o]; + for (int n = 0; n < nb * taps; ++n) { + int cb, kx, ky; + if (order == 0) { + cb = n; kx = 0; ky = 0; + } else if (order == 1) { + cb = n / taps; + const int t = n - cb * taps; + kx = t / k; + ky = t - kx * k; + } else { + kx = n / (nb * k); + const int r = n - kx * (nb * k); + cb = r / k; + ky = r - cb * k; + } + const int sy = yy + ky - p, sx = xx + kx - p; + const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W; + FgAcc s = fg_acc_begin(acc); + for (int i = 0; i < 16; ++i) { + const int c = cb * 16 + i; + const uint wv = w[((o * cin + c) * k + ky) * k + kx]; + const uint xv = inside ? (uint)x[(c * H + sy) * W + sx] : 0u; // outside the grid: 0 (0 * inf is NaN) + s = fg_acc_add(s, wv, xv); + } + acc = fg_acc_end(s); + } + if (act == 1) acc = fg_leaky16(acc); + y[idx] = (u16)acc; +} diff --git a/clean/fg/runner/__init__.py b/clean/fg/runner/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/fg/runner/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py new file mode 100644 index 00000000..76cf3c59 --- /dev/null +++ b/clean/fg/runner/fg_d3d12_check.py @@ -0,0 +1,275 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (issue #96): net1's GPU kernels (clean/fg/kernels/fg_net1.hip) against their D3D12 ports +(clean/dx12/kernels/fg_*.hlsl) byte for byte, and against the CPU reference (clean/fg/reference/net1.py) bit for bit. + + python clean/fg/runner/fg_d3d12_check.py --build