diff --git a/clean/dx12/kernels/fg_pool.hlsl b/clean/dx12/kernels/fg_pool.hlsl new file mode 100644 index 00000000..bed3f596 --- /dev/null +++ b/clean/dx12/kernels/fg_pool.hlsl @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 max-pool (issue #96): fg_maxpool2_f16 of fg_net1.co +// (clean/fg/kernels/fg_net1.hip), byte for byte. 2 x 2 window, stride 2, floor sizes; from the first tap (0, 0), then +// (0, 1), (1, 0), (1, 1), the delivered step m <- (x > m) * x + (x <= m) * m in float16 (fg_pool_step16 of the shared +// integer code clean/fg/kernels/fg_exact16.h). x [C][H][W] float16, y [C][H / 2][W / 2] float16, one output element per +// lane. Built only when --only names it (fg_ kernels are in no model's kernel set). +// +// @wg-kernel fg_maxpool2_f16 co=fg_net1.co block=256 wave=32 args=p,p,u,u,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint x, y; // pointers (arena byte offsets) + int C, H, W; +WG_ARGS_END + +WG_KERNEL(fg_maxpool2_f16) +{ + WG_SETUP(); + const int Ho = H / 2; + const int Wo = W / 2; + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * Ho * Wo) return; + const int c = idx / (Ho * Wo); + const int rem = idx - c * (Ho * Wo); + const int oy = rem / Wo; + const int ox = rem - oy * Wo; + const uint p = (uint)((c * H + 2 * oy) * W + 2 * ox) * 2u; + uint m = wg_ld16(x, p); + m = fg_pool_step16(m, wg_ld16(x, p + 2u)); + m = fg_pool_step16(m, wg_ld16(x, p + (uint)W * 2u)); + m = fg_pool_step16(m, wg_ld16(x, p + (uint)W * 2u + 2u)); + wg_st16(y, (uint)idx * 2u, m); +} diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index e2cf3f3c..c7f180ad 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -104,6 +104,9 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c zero border, float16 accumulator from the bias, 16-term chunks each summed exactly and rounded once, chunk order `order` (0 ascending, 1 channel block / kx / ky, 2 kx / channel block / ky), `act` 0 none or 1 leaky ReLU. cin is a multiple of 16. +- `fg_maxpool2_f16(x, y, C, H, W)`: net1's max-pool exactly as `net1.maxpool2`: 2 x 2, stride 2, floor sizes, taps + (0, 0), (0, 1), (1, 0), (1, 1) through the delivered step m <- (x > m) x + (x <= m) m with IEEE products and sums + (0 * inf is NaN; a sum of zeros is -0 only when both are -0). - The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's and Model L's kernel sets and DXILs do not change. diff --git a/clean/fg/README.md b/clean/fg/README.md index 5441ced2..35923c77 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -18,7 +18,7 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals | | `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only | | `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction | -| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none); D3D12 port `clean/dx12/kernels/fg_conv.hlsl` | +| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) and `fg_maxpool2_f16` (2 x 2, the delivered formula); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl` | | `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | diff --git a/clean/fg/kernels/fg_exact16.h b/clean/fg/kernels/fg_exact16.h index 0e7af661..02df3b3c 100644 --- a/clean/fg/kernels/fg_exact16.h +++ b/clean/fg/kernels/fg_exact16.h @@ -139,4 +139,23 @@ FG_FN uint fg_leaky16(uint x) { return fg_key16(t) > fg_key16(x) ? t : x; } +// 0 * x in IEEE arithmetic, as float16 bits: NaN for x inf or NaN, else a zero with x's sign +FG_FN uint fg_zero_times16(uint x) { + if (fg_is_nan16(x) != 0u || fg_is_inf16(x) != 0u) return FG_NAN16; + return x & 0x8000u; +} + +// One step of net1's max-pool, exactly the delivered formula m' = h((x > m) * x + (x <= m) * m) with IEEE products and +// sum: the term that is not taken becomes 0 * value (NaN for an infinity), and a sum of two zeros is -0 only when both +// are -0. Any NaN gives NaN (0x7E00). +FG_FN uint fg_pool_step16(uint m, uint x) { + if (fg_is_nan16(m) != 0u || fg_is_nan16(x) != 0u) return FG_NAN16; + const uint gt = fg_key16(x) > fg_key16(m) ? 1u : 0u; // x > m (+-0 equal; infinities ordered) + const uint v = gt != 0u ? x : m; // the term multiplied by 1 + const uint z = gt != 0u ? fg_zero_times16(m) : fg_zero_times16(x); // the term multiplied by 0 + if (z == FG_NAN16) return FG_NAN16; // 0 * inf + if ((v & 0x7FFFu) == 0u) return ((v & z & 0x8000u) != 0u) ? 0x8000u : 0u; // zero + zero + return v; // value + (+-0), an infinity included +} + #endif diff --git a/clean/fg/kernels/fg_net1.hip b/clean/fg/kernels/fg_net1.hip index 8701536b..4cf6ae5e 100644 --- a/clean/fg/kernels/fg_net1.hip +++ b/clean/fg/kernels/fg_net1.hip @@ -58,3 +58,22 @@ WG_KERNEL_N(fg_conv_f16, 256)(const u16* __restrict__ x, const u16* __restrict__ if (act == 1) acc = fg_leaky16(acc); y[idx] = (u16)acc; } + +// fg_maxpool2_f16: net1's 2 x 2 max-pool, stride 2, floor sizes (Ho = H / 2, Wo = W / 2): from the first tap (dy, dx) = +// (0, 0), then (0, 1), (1, 0), (1, 1), the delivered step m <- (x > m) * x + (x <= m) * m in float16 (fg_pool_step16). +// x [C][H][W], y [C][Ho][Wo]; output element idx = (c * Ho + oy) * Wo + ox. +WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y, int C, int H, int W) { + const int Ho = H / 2, Wo = W / 2; + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= C * Ho * Wo) return; + const int c = idx / (Ho * Wo); + const int rem = idx - c * (Ho * Wo); + const int oy = rem / Wo; + const int ox = rem - oy * Wo; + const u16* p = x + (c * H + 2 * oy) * W + 2 * ox; + uint m = p[0]; + m = fg_pool_step16(m, p[1]); + m = fg_pool_step16(m, p[W]); + m = fg_pool_step16(m, p[W + 1]); + y[idx] = (u16)m; +} diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py index 76cf3c59..40ed7294 100644 --- a/clean/fg/runner/fg_d3d12_check.py +++ b/clean/fg/runner/fg_d3d12_check.py @@ -13,11 +13,13 @@ - compares the RAW downloaded bytes of both back ends; each output buffer starts filled with its back end's own sentinel (HIP 0xA5, D3D12 0x5A) and has 64 slack bytes after it that neither may write, so an unwritten byte or a write past the end differs; -- a negative control (the first case with 4 or more workgroups, again on D3D12 with half of them) must be caught; +- a negative control per kernel (its first case with 4 or more workgroups, again on D3D12 with half of them) must be + caught; - compares the HIP output with the CPU reference, value for value (float16 bits; NaN equals NaN, the reference's NaN payload is not specified). -Cases: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several sizes, and -special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights. Output: counts only, no timing. +Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several +sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps +of several sizes (odd ones too) and every 2 x 2 window over ten special values. Output: counts only, no timing. """ import argparse import ctypes @@ -103,7 +105,37 @@ def _conv_case(kernel, x, w, b, k, order, act, label): b.astype(np.float64), chunks=order, act=act)} -KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True)} +POOL_SPECIALS = np.array([1.0, -2.0, np.inf, -np.inf, np.nan, 0.0, -0.0, 65504.0, -65504.0, 2.0 ** -24], + dtype=np.float16) + + +def pool_cases(kernel, rng): + """Random maps (odd sizes drop their last row / column), and every window of four taps over POOL_SPECIALS (10^4 + windows side by side in one 2-row map).""" + cases = [] + for C, H, W in ((3, 7, 9), (32, 8, 8), (64, 5, 5), (1, 2, 2), (16, 3, 17), (128, 4, 6)): + x = _f16(rng.uniform(-4, 4, (C, H, W))) + cases.append(_pool_case(kernel, x, f"{C}x{H}x{W}")) + v = POOL_SPECIALS + n = len(v) + idx = np.arange(n ** 4) + taps = [v[(idx // n ** p) % n] for p in range(4)] # (0,0), (0,1), (1,0), (1,1) of each window + x = np.zeros((1, 2, 2 * n ** 4), np.float16) + x[0, 0, 0::2], x[0, 0, 1::2], x[0, 1, 0::2], x[0, 1, 1::2] = taps + cases.append(_pool_case(kernel, x, "every special window")) + return cases + + +def _pool_case(kernel, x, label): + C, H, W = x.shape + n = C * (H // 2) * (W // 2) + return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label, + "args": [("buf", x.tobytes()), ("out", 2 * n + SLACK), ("u32", C), ("u32", H), ("u32", W)], + "defined": 2 * n, "cpu": lambda: N.maxpool2(x.astype(np.float64))} + + +KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True), + "fg_maxpool2_f16": pool_cases} # ---- prepare -------------------------------------------------------------------------------------------------------- @@ -193,7 +225,7 @@ def run(build, rocm, seeds, kernels): mod = h.load_module(f.read()) cdir = os.path.join(build, "replay-fg") os.makedirs(cdir, exist_ok=True) - lines, hip_out, neg = [], {}, None + lines, hip_out, negs = [], {}, {} # negs: per kernel, its first case with 4 or more workgroups for kernel in kernels: for si, seed in enumerate(seeds): rng = np.random.default_rng(seed) @@ -202,8 +234,8 @@ def run(build, rocm, seeds, kernels): outs = hip_run(h, mod, case) hip_out[name] = (kernel, case, outs) lines += replay_lines(name, case, cdir) - if neg is None and case["grid"] >= 4: - neg = name + if kernel not in negs and case["grid"] >= 4: + negs[kernel] = name lines += replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_") with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: f.write("\n".join(lines)) @@ -227,15 +259,18 @@ def read(path): ok = ok and good res["cases"].append({"case": name, "kernel": kernel, "label": case["label"], "raw": raw, "cpu": cpu, "ok": good}) - if neg: - kernel, case, outs = hip_out[neg] + res["negative_controls"] = {} + for kernel in kernels: # every kernel needs its own caught negative control + neg = negs.get(kernel) + if neg is None: + res["negative_controls"][kernel] = None + ok = False + continue + _, case, outs = hip_out[neg] nraw = compare_raw(read(os.path.join(cdir, f"negative_{neg}.o0")), outs[0], case["defined"]) caught = nraw["differ"] != 0 or not nraw["slack_ok"] - res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": nraw["differ"]} + res["negative_controls"][kernel] = {"case": neg, "half_grid": True, "caught": caught, "differ": nraw["differ"]} ok = ok and caught - else: - res["negative_control"] = None - ok = False res["summary"] = {"cases": len(res["cases"]), "cases_ok": sum(c["ok"] for c in res["cases"]), "values": sum(c["cpu"]["values"] for c in res["cases"]), "nan_values": sum(c["cpu"]["nan"] for c in res["cases"]), @@ -262,7 +297,7 @@ def main(argv=None): if a.json: with open(a.json, "w", encoding="utf-8", newline="\n") as f: f.write(text + "\n") - print(json.dumps(res["summary"]), "negative control:", json.dumps(res["negative_control"])) + print(json.dumps(res["summary"]), "negative controls:", json.dumps(res["negative_controls"])) for c in res["cases"]: if not c["ok"]: print("FAIL", c["case"], c["label"], json.dumps(c["raw"]), json.dumps(c["cpu"]))