diff --git a/clean/dx12/README.md b/clean/dx12/README.md index 79213c48..66f711db 100644 --- a/clean/dx12/README.md +++ b/clean/dx12/README.md @@ -20,7 +20,8 @@ as its HIP kernel (`clean/dx12/compare.py` proves it per kernel on synthetic fra | `dxil_asm.cpp` | tool used by the build (assembles the rewritten DXIL listing) | | `compare.py` | HIP vs hybrid D3D12, every stage output / state buffer byte for byte | | `kernels/exposure.hlsl` | the reference port (`exposure_auto_diet`): copy its structure | -| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; check: `clean/fg/runner/fg_d3d12_check.py` | +| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; checks: `clean/fg/runner/fg_d3d12_check.py` (each kernel), `clean/fg/runner/fg_net1_gpu.py` (net1's 32 steps as a `kernel_replay` chain) | +| `tests/kernel_replay.cpp` | replays launches from a case file on the native D3D12 device; `chain` blocks run several launches on buffers that persist between them (a whole network, optionally timed) | ## Arithmetic mode (why build_dx12.py and nothing else) diff --git a/clean/dx12/kernels/fg_add.hlsl b/clean/dx12/kernels/fg_add.hlsl new file mode 100644 index 00000000..f4a0a03d --- /dev/null +++ b/clean/dx12/kernels/fg_add.hlsl @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: MIT +// D3D12 port of frame generation's net1 skip add (issue #96): fg_add_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip), +// byte for byte. y = h(a + b) per element of n float16 values: the single-precision sum (one IEEE operation), then +// float16 by wg_f2h (nearest even); single precision then float16 equals one rounding of the exact sum of two float16 +// values. NaN results are written as 0x7E00. Built only when --only names it (fg_ kernels are in no model's set). +// +// @wg-kernel fg_add_f16 co=fg_net1.co block=256 wave=32 args=p,p,p,u +#include "wg_platform.hlsli" + +#define FG_FN +#define FG_I64 int64_t +#define FG_FBH32(v) firstbithigh((uint)(v)) +#include "../../fg/kernels/fg_exact16.h" + +WG_ARGS_BEGIN + uint a, b, y; // pointers (arena byte offsets) + int n; +WG_ARGS_END + +WG_KERNEL(fg_add_f16) +{ + WG_SETUP(); + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= n) return; + uint outv = wg_f2h(wg_h2f(wg_ld16(a, (uint)idx * 2u)) + wg_h2f(wg_ld16(b, (uint)idx * 2u))); + if (fg_is_nan16(outv) != 0u) outv = FG_NAN16; + wg_st16(y, (uint)idx * 2u, outv); +} diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index 2800c1f5..cf989632 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -25,9 +25,22 @@ // direct_out after the launch, the output texture's texels are written to , // row by row (tightly packed); its initial texels come from the output // slot's file (a sentinel fill, so that an unwritten texel shows) +// +// A chain (frame generation, #96) runs several launches in order on buffers that persist between them, so that a whole +// network runs on the device with no host round trip between its steps: +// chain +// module [] (zero or more) +// buf [fill] device buffer: the file's bytes (padded with fill when the file is +// shorter), or every byte set to fill (0..255, default 0) with "-" +// step ... a launch; each arg is @ (a buffer's pointer) or a 32-bit decimal +// save after the run, the buffer's bytes are written to +// time after the run (and the saves), the steps run more times; the mean +// wall time per run (submission to completion, synchronised after each +// run) is printed as "chain : time ms per run ( runs)" #include "../../runtime/device.h" #include "../../runtime/device_d3d12.h" +#include #include #include #include @@ -337,6 +350,109 @@ int run_case(Device* d, Case& c, const std::string& base) { return 0; } +struct ChainBuf { + std::string id, file; + size_t bytes = 0; + int fill = 0; + DevPtr ptr = 0; +}; + +struct ChainStep { + std::string kernel; + unsigned grid = 0, block = 0; + std::vector args; // "@id" or a decimal value +}; + +struct Chain { + std::string name; + std::vector> modules; + std::vector bufs; + std::vector steps; + std::vector> saves; // (id, file) + int time_runs = 0; +}; + +int run_chain(Device* d, Chain& c, const std::string& base) { + for (auto& m : c.modules) { + const bool ok = m.second.empty() ? d->load_module(m.first) : d->load_module_tagged(m.first, m.second); + if (!ok) { std::printf("%s: module %s: %s\n", c.name.c_str(), m.first.c_str(), d->err().c_str()); return 1; } + } + auto find = [&](const std::string& id) -> ChainBuf* { + for (auto& b : c.bufs) + if (b.id == id) return &b; + return nullptr; + }; + int rc = 0; + for (auto& b : c.bufs) { + std::vector data(b.bytes, (uint8_t)(b.fill & 0xff)); + if (b.file != "-") { + std::vector f; + if (!read_file(base + "/" + b.file, &f) || f.size() > b.bytes) { + std::printf("%s: buf %s: cannot read %s (or larger than %zu bytes)\n", c.name.c_str(), b.id.c_str(), + b.file.c_str(), b.bytes); + rc = 1; + break; + } + std::memcpy(data.data(), f.data(), f.size()); + } + if (!d->alloc(&b.ptr, b.bytes ? b.bytes : 4) || !d->upload(b.ptr, data.data(), data.size())) { + std::printf("%s: buf %s: %s\n", c.name.c_str(), b.id.c_str(), d->err().c_str()); + rc = 1; + break; + } + } + // the argument values of every step, resolved once + std::vector> ptrs(c.steps.size()); + std::vector> vals(c.steps.size()); + std::vector> argv(c.steps.size()); + for (size_t i = 0; rc == 0 && i < c.steps.size(); ++i) { + auto& s = c.steps[i]; + ptrs[i].reserve(s.args.size()); + vals[i].reserve(s.args.size()); + for (auto& a : s.args) { + if (!a.empty() && a[0] == '@') { + ChainBuf* b = find(a.substr(1)); + if (!b) { std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); rc = 1; break; } + ptrs[i].push_back(b->ptr); + argv[i].push_back(&ptrs[i].back()); + } else { + vals[i].push_back((uint32_t)std::stoll(a)); + argv[i].push_back(&vals[i].back()); + } + } + } + auto run_steps = [&]() -> bool { + for (size_t i = 0; i < c.steps.size(); ++i) { + if (!d->launch(c.steps[i].kernel, c.steps[i].grid, c.steps[i].block, argv[i].data())) { + std::printf("%s: step %zu launch %s: %s\n", c.name.c_str(), i, c.steps[i].kernel.c_str(), d->err().c_str()); + return false; + } + } + if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return false; } + return true; + }; + if (rc == 0 && !run_steps()) rc = 1; + for (size_t i = 0; rc == 0 && i < c.saves.size(); ++i) { + ChainBuf* b = find(c.saves[i].first); + std::vector data(b ? b->bytes : 0); + if (!b || !d->download(data.data(), b->ptr, b->bytes) || !write_file(base + "/" + c.saves[i].second, data)) { + std::printf("%s: save %s: %s\n", c.name.c_str(), c.saves[i].first.c_str(), b ? d->err().c_str() : "unknown"); + rc = 1; + } + } + if (rc == 0 && c.time_runs > 0) { + const auto t0 = std::chrono::steady_clock::now(); + for (int r = 0; rc == 0 && r < c.time_runs; ++r) + if (!run_steps()) rc = 1; + const double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (rc == 0) std::printf("chain %s: time %.3f ms per run (%d runs)\n", c.name.c_str(), ms / c.time_runs, c.time_runs); + } + for (auto& b : c.bufs) + if (b.ptr) d->free(b.ptr); + if (rc == 0) std::printf("%s: ok (chain of %zu steps)\n", c.name.c_str(), c.steps.size()); + return rc; +} + } // namespace int main(int argc, char** argv) { @@ -349,13 +465,45 @@ int main(int argc, char** argv) { if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; } const std::string base = dir_of(argv[2]); std::vector cases; + std::vector chains; + bool in_chain = false; // the lines after "chain" belong to it until the next "case" or "chain" std::string line; while (std::getline(f, line)) { if (!line.empty() && line.back() == '\r') line.pop_back(); std::istringstream s(line); std::string w; if (!(s >> w)) continue; - if (w == "case") { cases.emplace_back(); s >> cases.back().name; continue; } + if (w == "case") { cases.emplace_back(); s >> cases.back().name; in_chain = false; continue; } + if (w == "chain") { chains.emplace_back(); s >> chains.back().name; in_chain = true; continue; } + if (in_chain) { + Chain& ch = chains.back(); + if (w == "module") { + std::string m, tag; + s >> m >> tag; + ch.modules.emplace_back(m, tag); + } else if (w == "buf") { + ChainBuf b; + s >> b.id >> b.bytes >> b.file; + if (!(s >> b.fill)) b.fill = 0; + ch.bufs.push_back(b); + } else if (w == "step") { + ChainStep st; + s >> st.kernel >> st.grid >> st.block; + std::string a; + while (s >> a) st.args.push_back(a); + ch.steps.push_back(st); + } else if (w == "save") { + std::string id, file; + s >> id >> file; + ch.saves.emplace_back(id, file); + } else if (w == "time") { + s >> ch.time_runs; + } else { + std::printf("unknown chain line: %s\n", line.c_str()); + return 2; + } + continue; + } if (cases.empty()) { std::printf("line before the first case: %s\n", line.c_str()); return 2; } Case& c = cases.back(); if (w == "module") { @@ -398,7 +546,8 @@ int main(int argc, char** argv) { } int fails = 0; for (auto& c : cases) fails += run_case(d.get(), c, base); + for (auto& c : chains) fails += run_chain(d.get(), c, base); if (!d->sync()) { std::printf("final sync: %s\n", d->err().c_str()); return 1; } - std::printf("kernel_replay: %d of %zu cases failed\n", fails, cases.size()); + std::printf("kernel_replay: %d of %zu cases failed\n", fails, cases.size() + chains.size()); return fails ? 1 : 0; } diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md index 807660ae..d2f7c878 100644 --- a/clean/fg/FORMATS.md +++ b/clean/fg/FORMATS.md @@ -112,10 +112,15 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c (0 for an extent of 1), so no division runs on the GPU. Positions in single precision one operation at a time (HIP: no contraction); every product and sum rounded to float16 (a product of two float16 values is exact in single precision, and single precision then float16 equals one rounding of a sum of two float16 values). +- `fg_add_f16(a, b, y, n)`: net1's skip add exactly as `net1.add16`: y = h(a + b) per element (single precision then + float16, one rounding of the exact sum). +- net1 on the GPU is these four kernels in NET1_WIRING's order, one launch per step, every buffer resident on the + device (`runner/fg_net1_gpu.py`; on D3D12 a kernel_replay `chain` block, with a UAV barrier between launches). - The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's and Model L's kernel sets and DXILs do not change. -- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference. +- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference; + `runner/fg_net1_gpu.py` the whole network, every step output. ## Configuration (`reference/config.py`) diff --git a/clean/fg/README.md b/clean/fg/README.md index 0ac4df80..907fff8f 100644 --- a/clean/fg/README.md +++ b/clean/fg/README.md @@ -18,8 +18,10 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM | `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals | | `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only | | `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction | -| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula) and `fg_upscale_f16` (bilinear, align corners, float16 steps); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl` | +| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula), `fg_upscale_f16` (bilinear, align corners, float16 steps) and `fg_add_f16` (the skip add); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl`, `fg_add.hlsl` | | `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values | +| `runner/fg_net1_gpu.py` | net1's 32 steps chained on the GPU (HIP, and D3D12 through kernel_replay's `chain` blocks): every step output on both back ends byte for byte and against the CPU reference bit for bit, a half-grid negative control per kernel, synthetic weights (also odd sizes and special values) and, with `--model`, a converter output file's weights (execute-only: its per-run files are deleted, digests only); `--timing` prints net1's milliseconds at 1080p- and 1440p-sized inputs as information, after a probe rules out long single launches | +| `tests/test_fg_net1_gpu_plan.py` | the chain as planned on the CPU: every step output's shape against the reference's buffers, launch arguments, natural weight order, chain lines | | `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | | `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` | | `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` | diff --git a/clean/fg/kernels/fg_net1.hip b/clean/fg/kernels/fg_net1.hip index ff495e47..7dbf7a9b 100644 --- a/clean/fg/kernels/fg_net1.hip +++ b/clean/fg/kernels/fg_net1.hip @@ -84,6 +84,16 @@ WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y y[idx] = (u16)m; } +// fg_add_f16: net1's skip add, y = h(a + b) per element (float16; single precision then float16 equals one rounding of +// the exact sum). NaN results are written as 0x7E00. n elements. +WG_KERNEL_N(fg_add_f16, 256)(const u16* __restrict__ a, const u16* __restrict__ b, u16* __restrict__ y, int n) { + const int idx = (int)(wg_block_id() * 256u + wg_lane_id()); + if (idx >= n) return; + uint out = fg_f2h(fg_h2f(a[idx]) + fg_h2f(b[idx])); + if (fg_is_nan16(out) != 0u) out = FG_NAN16; + y[idx] = (u16)out; +} + // One axis of the align-corners rule: s = r * o (single precision), i0 = trunc(s), i1 = trunc(s + 1), both n_in - 1 when // i1 >= n_in; weights w1 = h(s - i0), w0 = h((i0 - s) + 1) as float16 values. struct FgTap { int i0, i1; float w0, w1; }; diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py index 8028e1de..1a89326d 100644 --- a/clean/fg/runner/fg_d3d12_check.py +++ b/clean/fg/runner/fg_d3d12_check.py @@ -20,7 +20,8 @@ Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps of several sizes (odd ones too) and every 2 x 2 window over ten special values; fg_upscale_f16: net1's level pairs, -other sizes, an output extent of 1, equal sizes, special values. Output: counts only, no timing. +other sizes, an output extent of 1, equal sizes, special values; fg_add_f16: random pairs of several lengths and every +pair of thirteen special values. Output: counts only, no timing. The whole network: fg_net1_gpu.py. """ import argparse import ctypes @@ -168,8 +169,34 @@ def _upscale_case(kernel, x, ho, wo, label): "defined": 2 * n, "cpu": lambda: N.upscale_to(x.astype(np.float64), wo, ho)} +ADD_SPECIALS = np.array([np.nan, np.inf, -np.inf, 65504.0, -65504.0, 2.0 ** -24, -2.0 ** -24, -0.0, 0.0, 1.0, -1.0, + 32768.0, 0.000999], dtype=np.float16) + + +def add_cases(kernel, rng): + """Random pairs of several lengths, and every pair over ADD_SPECIALS (overflow, inf - inf, -0 + -0, subnormal + sums).""" + cases = [] + for n in (1, 255, 256, 1000, 4096 * 3 + 7): + a = _f16(rng.uniform(-100, 100, n)) + b = _f16(rng.uniform(-100, 100, n)) + cases.append(_add_case(kernel, a, b, f"n {n}")) + v = ADD_SPECIALS + a = np.repeat(v, len(v)).astype(np.float16) + b = np.tile(v, len(v)).astype(np.float16) + cases.append(_add_case(kernel, a, b, "every special pair")) + return cases + + +def _add_case(kernel, a, b, label): + n = a.size + return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label, + "args": [("buf", a.tobytes()), ("buf", b.tobytes()), ("out", 2 * n + SLACK), ("u32", n)], + "defined": 2 * n, "cpu": lambda: N.add16(a.astype(np.float64), b.astype(np.float64))} + + KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True), - "fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases} + "fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases, "fg_add_f16": add_cases} # ---- prepare -------------------------------------------------------------------------------------------------------- diff --git a/clean/fg/runner/fg_net1_gpu.py b/clean/fg/runner/fg_net1_gpu.py new file mode 100644 index 00000000..005081d5 --- /dev/null +++ b/clean/fg/runner/fg_net1_gpu.py @@ -0,0 +1,491 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (issue #96): net1's 32 steps (clean/fg/reference/net1.py NET1_WIRING) chained on the GPU, on HIP +and on the native D3D12 device, against the CPU reference bit for bit. + + python clean/fg/runner/fg_net1_gpu.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --prepare + python clean/fg/runner/fg_net1_gpu.py --build --rocm ... --run [--model ] [--json out.json] + python clean/fg/runner/fg_net1_gpu.py --build --rocm ... --timing [--json out.json] + +--prepare (CPU) is fg_d3d12_check.py's: fg_net1.co, the four D3D12 ports (build_dx12.py --only) and kernel_replay.exe. +--run (GPU, one GPU slot): every case runs the whole network on HIP in this process (all buffers on the device, one +launch per step, no host round trip) and through one kernel_replay "chain" per case on D3D12, then: +- every one of the 32 step outputs is compared as RAW bytes between the back ends; each starts filled with its back + end's sentinel (HIP 0xA5, D3D12 0x5A) and has 64 slack bytes after it that no step may write; +- every step output on HIP is compared with the CPU reference's buffer (float16 bits; NaN equals NaN); +- negative controls: per kernel, a D3D12 chain whose first step of that kernel (with 4 or more workgroups) runs on half + of its workgroups must differ in that step's output; +- the network output (unfolded to [4, G, G]) gets a SHA-256 on both sides. +Cases: synthetic weights on 64 x 64 pixels (the smallest input with a cell on every level), on 70 x 90 pixels (odd +level sizes: floor halving), and with special values (NaN, +-inf, subnormals, +-65504, -0) in the input and biases; +with --model, also the weights of a converter output file of the preset-FG map on clean/fg/tools/net1_digest.py's fixed +input. Weights from --model are execute-only: the per-case files this tool writes for kernel_replay (input, weights, +outputs) are deleted after the run, and only counts and digests are printed. +--timing (GPU, one GPU slot; information only): net1 on synthetic weights at 1920 x 1080 and 2560 x 1440 input pixels +(960 x 540 and 1280 x 720 cells), mean milliseconds per network run on each back end. A probe at 512 x 512 pixels times +every step first; the large sizes run only when every single launch, scaled by the cell count, stays under +MAX_LAUNCH_MS (a long single dispatch could trip the display driver's timeout). +""" +import argparse +import ctypes +import hashlib +import json +import os +import subprocess +import sys +import time + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +sys.path.insert(0, HERE) + +from clean.fg import layout as L # noqa: E402 +from clean.fg.reference import net1 as N # noqa: E402 +import fg_d3d12_check as K # noqa: E402 + +KERNELS = ["fg_conv_f16", "fg_maxpool2_f16", "fg_upscale_f16", "fg_add_f16"] +KERNEL_OF = {"conv": "fg_conv_f16", "pool": "fg_maxpool2_f16", "upscale": "fg_upscale_f16", "add": "fg_add_f16"} +SPECIALS = K.SPECIALS +FINITE_SPECIALS = np.array([65504.0, -65504.0, 2.0 ** -24, -2.0 ** -24, 2.0 ** -20, -0.0, 0.0], dtype=np.float16) +MAX_LAUNCH_MS = 250.0 +TIMING_SIZES = {"1080p": (1920, 1080), "1440p": (2560, 1440)} +PROBE = (512, 512) + + +# ---- the network as buffers and launches ------------------------------------------------------------------------------ + +def synthetic_params(rng): + """{layer: (canonical weights [out, k, k, in], bias)} float16, scaled so that values stay in range.""" + out = {} + for l in L.NET1: + k = 3 if l.taps == 9 else 1 + s = 1.5 / np.sqrt(l.cin * k * k) + out[l.name] = (rng.uniform(-s, s, (l.out, k, k, l.cin)).astype(np.float16), + rng.uniform(-0.1, 0.1, l.out).astype(np.float16)) + return out + + +def plan(H0, W0, params): + """Buffers and launches of net1 on cells [48, H0, W0]. + + Returns (data, acts, steps): data {id: bytes} (cells, weights natural [out][in][k][k], biases), acts {id: (shape, + defined bytes)} the step outputs, steps [(kernel, grid, args, out id, op)] with args "@id" or integers.""" + data, acts, steps = {}, {}, [] + shape = {"cells": (N.CELL, H0, W0)} + for s in N.NET1_WIRING: + op = s["op"] + if op == "conv": + w, b = params[s["layer"]] + cout, k, _, cin = w.shape + wid, bid = "w_" + s["layer"], "b_" + s["layer"] + data[wid] = np.ascontiguousarray(np.asarray(w, np.float16).transpose(0, 3, 1, 2)).tobytes() + data[bid] = np.asarray(b, np.float16).tobytes() + C, H, W = shape[s["in"]] + if C != cin: + raise ValueError("%s: %d input channels, buffer %s has %d" % (s["layer"], cin, s["in"], C)) + out = (cout, H, W) + args = ["@" + s["in"], "@" + wid, "@" + bid, "@" + s["out"], cin, cout, H, W, k, + K.ORDER_CODE[s["chunks"]], 1 if s["act"] == "leaky" else 0] + elif op == "pool": + C, H, W = shape[s["in"]] + out = (C, H // 2, W // 2) + args = ["@" + s["in"], "@" + s["out"], C, H, W] + elif op == "upscale": + C, hi, wi = shape[s["in"]] + _, ho, wo = shape[s["like"]] + out = (C, ho, wo) + args = ["@" + s["in"], "@" + s["out"], C, hi, wi, ho, wo, K._ratio_bits(wi, wo), K._ratio_bits(hi, ho)] + elif op == "add": + a, b = s["in"] + if shape[a] != shape[b]: + raise ValueError("add of %s %s and %s %s" % (a, shape[a], b, shape[b])) + out = shape[a] + args = ["@" + a, "@" + b, "@" + s["out"], int(np.prod(out))] + else: + raise ValueError("unknown step %r" % op) + shape[s["out"]] = out + n = int(np.prod(out)) + acts[s["out"]] = (out, 2 * n) + steps.append((KERNEL_OF[op], -(-n // 256), args, s["out"], op)) + return data, acts, steps + + +def cells_bytes(cells): + return np.asarray(cells, np.float64).astype(np.float16).tobytes() + + +def output_digest(out_bytes, H0, W0, gh, gw): + """SHA-256 of the unfolded output [4, gh, gw] (float16 bytes as written), the same with every NaN as 0x7E00 (the + reference's NaN payload is not specified), and the number of non-finite values.""" + o = np.frombuffer(out_bytes, np.float16).astype(np.float64).reshape(16, H0, W0) + return pixel_digest(N.unfold(o, gh, gw)) + + +def pixel_digest(px): + px = np.asarray(px, np.float64).astype(" stop: + break + g = max(1, grid // 2) if j == half else grid + lines.append("step %s %d 256 %s" % (kernel, g, " ".join(str(a) for a in args))) + for i in (saves if saves is not None else acts): + lines.append("save %s %s.%s" % (i, name, i)) + if time_runs: + lines.append("time %d" % time_runs) + return lines + [""] + + +def write_files(cdir, name, data): + files = {} + for i, b in data.items(): + fn = "%s.%s" % (name, i) + path = os.path.join(cdir, fn) + with open(path, "wb") as f: + f.write(b) + files[i] = {"name": fn, "path": path} + return files + + +def replay(build, cdir, lines, arena_mb, timeout): + path = os.path.join(cdir, "chains.txt") + with open(path, "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() == "HIP_PLATFORM"} + env["WG_D3D12_ARENA_MB"] = str(arena_mb) + t0 = time.time() + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, path], env=env, capture_output=True, + text=True, timeout=timeout, creationflags=0x08000000) + return r, time.time() - t0 + + +def remove_written(cdir, written): + """Deletes the files this tool wrote into cdir (one by one; never the folder).""" + for fn in written: + p = os.path.join(cdir, fn) + if os.path.isfile(p): + os.remove(p) + + +def arena_mb(data, acts): + total = sum(len(b) + 4096 for b in data.values()) + sum(n + K.SLACK + 4096 for _, n in acts.values()) + return int(min(4000, max(256, 1.5 * total / 2 ** 20 + 64))) + + +# ---- run: correctness ------------------------------------------------------------------------------------------------- + +def cases(model): + rng = np.random.default_rng(20261114) + out = [] + p = synthetic_params(rng) + v = rng.uniform(0, 1, (10, 64, 64)) + out.append(("synthetic_64", v, p)) + v = rng.uniform(-1, 1, (10, 70, 90)) + out.append(("synthetic_70x90", v, synthetic_params(rng))) + v = rng.uniform(0, 1, (10, 64, 64)).astype(np.float16) + f = rng.random(v.shape) < 0.003 + v[f] = rng.choice(SPECIALS, int(f.sum())) + ps = synthetic_params(rng) + w, b = ps["net1_06"] + b = b.copy() + b[0], b[1], b[2], b[3] = np.inf, -np.inf, np.nan, 65504.0 + ps["net1_06"] = (w, b) + w, b = ps["net1_02"] + w = w.copy() + w[3, 0, 0, :] = 200.0 # sums that overflow to +inf + w[4, 0, 0, :] = -200.0 + ps["net1_02"] = (w, b) + out.append(("specials_64", v.astype(np.float64), ps)) + # finite specials only (a NaN anywhere reaches every output through the coarse levels) + v = rng.uniform(0, 1, (10, 64, 64)).astype(np.float16) + f = rng.random(v.shape) < 0.01 + v[f] = rng.choice(FINITE_SPECIALS, int(f.sum())) + out.append(("finite_specials_64", v.astype(np.float64), synthetic_params(rng))) + if model: + from clean.fg.reference import forward as F + from clean.fg.tools import net1_digest as D + out.append(("model", D.fixed_input(), F.params_from_model_file(model))) + return out + + +def run(build, rocm, model): + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + h = Hip(rocm) + with open(os.path.join(build, K.CO), "rb") as f: + mod = h.load_module(f.read()) + cdir = os.path.join(build, "replay-net1") + os.makedirs(cdir, exist_ok=True) + lines, runs, written, need_mb = [], [], [], 256 + try: + for name, values, params in cases(model): + t0 = time.time() + cells = N.fold(values) + _, gh, gw = values.shape + H0, W0 = cells.shape[1:] + data, acts, steps = plan(H0, W0, params) + data = dict(cells=cells_bytes(cells), **data) + net = HipNet(h, mod, data, acts, steps) + try: + hip = net.run() + finally: + net.free() + ref = N.run(cells, params) + files = write_files(cdir, name, data) + written += [f["name"] for f in files.values()] + ["%s.%s" % (name, i) for i in acts] + lines += chain_lines(name, files, acts, steps) + negs = {} + for kernel in KERNELS: + j = next(j for j, s in enumerate(steps) if s[0] == kernel and s[1] >= 4) + nn = "negative_%s_%s" % (name, kernel) + negs[kernel] = (nn, j, steps[j][3]) + lines += chain_lines(nn, files, acts, steps, half=j, stop=j, saves=[steps[j][3]]) + written.append("%s.%s" % (nn, steps[j][3])) + need_mb = max(need_mb, arena_mb(data, acts)) + runs.append({"name": name, "acts": acts, "steps": steps, "hip": hip, "ref": ref, "negs": negs, + "dims": (H0, W0, gh, gw), "cpu_s": time.time() - t0}) + r, secs = replay(build, cdir, lines, need_mb, timeout=120) + + def read(fn): + p = os.path.join(cdir, fn) + return open(p, "rb").read() if os.path.exists(p) else b"" + + res = {"replay_exit": r.returncode, "replay_s": round(secs, 1), "replay_tail": r.stdout[-600:] + r.stderr[-300:], + "cases": []} + ok = r.returncode == 0 + for c in runs: + rows, bad = [], 0 + for j, (kernel, grid, _, out, op) in enumerate(c["steps"]): + _, n = c["acts"][out] + raw = K.compare_raw(read("%s.%s" % (c["name"], out)), c["hip"][out], n) + cpu = K.compare_cpu(c["hip"][out][:n], c["ref"][out]) + good = raw["differ"] == 0 and raw["slack_ok"] and cpu["differ"] == 0 + bad += not good + rows.append({"step": j, "op": op, "out": out, "values": cpu["values"], "raw_differ": raw["differ"], + "slack_ok": raw["slack_ok"], "cpu_differ": cpu["differ"], "nan": cpu["nan"], + "inf": cpu["inf"], "ok": good}) + H0, W0, gh, gw = c["dims"] + n_out = c["acts"]["out"][1] + gpu_sha, gpu_can, nonfinite = output_digest(c["hip"]["out"][:n_out], H0, W0, gh, gw) + d3d_sha, _, _ = output_digest(read("%s.out" % c["name"])[:n_out] or b"\0" * n_out, H0, W0, gh, gw) + cpu_sha, cpu_can, _ = pixel_digest(N.unfold(c["ref"]["out"], gh, gw)) + negs = {} + for kernel, (nn, j, out) in c["negs"].items(): + _, n = c["acts"][out] + nraw = K.compare_raw(read("%s.%s" % (nn, out)), c["hip"][out], n) + negs[kernel] = {"step": j, "out": out, "differ": nraw["differ"], + "caught": nraw["differ"] != 0 or not nraw["slack_ok"]} + # HIP and D3D12: the same bytes; the CPU reference: the same with NaN payloads made equal + good = bad == 0 and gpu_sha == d3d_sha and gpu_can == cpu_can and all(v["caught"] for v in negs.values()) + ok = ok and good + res["cases"].append({"case": c["name"], "input_pixels": [gw, gh], "cells": [W0, H0], "steps": rows, + "steps_ok": len(rows) - bad, "values": sum(x["values"] for x in rows), + "nan": sum(x["nan"] for x in rows), "inf": sum(x["inf"] for x in rows), + "output_sha256": {"hip": gpu_sha, "d3d12": d3d_sha, "cpu": cpu_sha}, + "output_sha256_nan_canonical": {"hip": gpu_can, "cpu": cpu_can}, + "output_nonfinite": nonfinite, "negative_controls": negs, "cpu_s": round(c["cpu_s"], 1), + "ok": good}) + res["summary"] = {"cases": len(res["cases"]), "cases_ok": sum(c["ok"] for c in res["cases"]), + "step_outputs": sum(len(c["steps"]) for c in res["cases"]), + "values": sum(c["values"] for c in res["cases"]), + "nan_values": sum(c["nan"] for c in res["cases"]), + "inf_values": sum(c["inf"] for c in res["cases"]), "pass": bool(ok)} + return res + finally: + remove_written(cdir, written) + + +# ---- timing (information only) ---------------------------------------------------------------------------------------- + +def timing(build, rocm, reps=5): + from clean.runner.hip import Hip, clean_process_environment + clean_process_environment(rocm) + h = Hip(rocm) + with open(os.path.join(build, K.CO), "rb") as f: + mod = h.load_module(f.read()) + rng = np.random.default_rng(7) + params = synthetic_params(rng) + cdir = os.path.join(build, "replay-net1") + os.makedirs(cdir, exist_ok=True) + res = {"reps": reps, "max_launch_ms": MAX_LAUNCH_MS} + + def setup(w, hgt): + H0, W0 = -(-hgt // 2), -(-w // 2) + cells = rng.uniform(0, 1, (N.CELL, H0, W0)) + cells[4 * N.VALUES:] = 0 + data, acts, steps = plan(H0, W0, params) + return dict(cells=cells_bytes(cells), **data), acts, steps, H0 * W0 + + data, acts, steps, probe_cells = setup(*PROBE) + net = HipNet(h, mod, data, acts, steps) + try: + net.launch() + h.sync() # warm-up + per = net.time_steps(2) + finally: + net.free() + worst = int(np.argmax(per)) + res["probe"] = {"pixels": list(PROBE), "max_step_ms": round(per[worst], 3), "max_step": worst, + "network_ms_sum_of_steps": round(sum(per), 3)} + sizes = {} + for label, (w, hgt) in TIMING_SIZES.items(): + cells_n = (-(-hgt // 2)) * (-(-w // 2)) + predicted = per[worst] * cells_n / probe_cells + entry = {"pixels": [w, hgt], "predicted_max_launch_ms": round(predicted, 1)} + sizes[label] = entry + if predicted >= MAX_LAUNCH_MS: + entry["skipped"] = "a single launch is predicted at %.0f ms (limit %.0f ms)" % (predicted, MAX_LAUNCH_MS) + continue + data, acts, steps, _ = setup(w, hgt) + net = HipNet(h, mod, data, acts, steps) + try: + net.launch() + h.sync() # warm-up + entry["max_launch_ms_measured"] = round(max(net.time_steps(1)), 2) + entry["hip_ms"] = round(net.time_network(reps), 2) + hip_out = net.run()["out"] + finally: + net.free() + name = "timing_" + label + files = write_files(cdir, name, data) + written = [f["name"] for f in files.values()] + ["%s.out" % name] + try: + r, secs = replay(build, cdir, chain_lines(name, files, acts, steps, saves=["out"], time_runs=reps), + arena_mb(data, acts), timeout=120) + tail = r.stdout + entry["d3d12_exit"] = r.returncode + for line in tail.splitlines(): + if line.startswith("chain %s: time" % name): + entry["d3d12_ms"] = float(line.split("time ")[1].split(" ms")[0]) + got = open(os.path.join(cdir, "%s.out" % name), "rb").read() if r.returncode == 0 else b"" + n = acts["out"][1] + entry["out_raw_equal"] = K.compare_raw(got, hip_out, n)["differ"] == 0 if got else False + if r.returncode != 0: + entry["d3d12_tail"] = tail[-400:] + r.stderr[-200:] + finally: + remove_written(cdir, written) + res["sizes"] = sizes + res["method"] = ("HIP: device events around %d back-to-back network runs, mean per run. D3D12: kernel_replay chain " + "wall time from submission to completion, synchronised after each of %d runs, mean per run. " + "Synthetic weights, buffers resident, warm-up run first. Information only." % (reps, reps)) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--prepare", action="store_true") + ap.add_argument("--run", action="store_true") + ap.add_argument("--timing", action="store_true") + ap.add_argument("--model", default=None, help="a converter output file of the preset-FG map (execute-only)") + ap.add_argument("--json", default=None) + a = ap.parse_args(argv) + if a.prepare: + K.prepare(a.build, a.rocm, KERNELS) + res = None + if a.run: + res = run(a.build, a.rocm, a.model) + print(json.dumps(res["summary"])) + for c in res["cases"]: + print(c["case"], "ok" if c["ok"] else "FAIL", "steps_ok %d/%d" % (c["steps_ok"], len(c["steps"])), + "negative controls caught:", sum(v["caught"] for v in c["negative_controls"].values()), + "output hip == d3d12:", c["output_sha256"]["hip"] == c["output_sha256"]["d3d12"], + "== cpu (NaN canonical):", len(set(c["output_sha256_nan_canonical"].values())) == 1, + "nonfinite outputs:", c["output_nonfinite"]) + for s in c["steps"]: + if not s["ok"]: + print(" FAIL step", json.dumps(s)) + print("replay exit", res["replay_exit"]) + elif a.timing: + res = timing(a.build, a.rocm) + print(json.dumps(res, indent=1)) + if res is not None and a.json: + with open(a.json, "w", encoding="utf-8", newline="\n") as f: + f.write(json.dumps(res, indent=1) + "\n") + if a.run: + return 0 if res["summary"]["pass"] else 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/fg/tests/test_fg_net1_gpu_plan.py b/clean/fg/tests/test_fg_net1_gpu_plan.py new file mode 100644 index 00000000..81abb15e --- /dev/null +++ b/clean/fg/tests/test_fg_net1_gpu_plan.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: MIT +"""The GPU chain of net1 (clean/fg/runner/fg_net1_gpu.py) as planned on the CPU: buffers, launches and chain lines.""" +import os +import sys +import tempfile +import unittest + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "clean", "fg", "runner")) + +import fg_net1_gpu as G # noqa: E402 +from clean.fg.reference import net1 as N # noqa: E402 + +ARITY = {"fg_conv_f16": 11, "fg_maxpool2_f16": 5, "fg_upscale_f16": 9, "fg_add_f16": 4} + + +class PlanTest(unittest.TestCase): + def setUp(self): + self.params = G.synthetic_params(np.random.default_rng(3)) + + def test_every_step_output_has_the_reference_shape(self): + for gh, gw in ((64, 64), (70, 90), (65, 129)): + cells = N.fold(np.zeros((10, gh, gw))) + data, acts, steps = G.plan(cells.shape[1], cells.shape[2], self.params) + self.assertEqual(len(steps), 32) + self.assertEqual([s[3] for s in steps], [s["out"] for s in N.NET1_WIRING]) + ref = N.run(cells, self.params) + for out, (shape, nbytes) in acts.items(): + self.assertEqual(ref[out].shape, shape, out) + self.assertEqual(nbytes, 2 * int(np.prod(shape))) + + def test_launch_arguments(self): + data, acts, steps = G.plan(32, 32, self.params) + ids = set(data) | set(acts) | {"cells"} + for kernel, grid, args, out, op in steps: + self.assertEqual(len(args), ARITY[kernel], kernel) + self.assertEqual(grid, -(-int(np.prod(acts[out][0])) // 256)) + for a in args: + if isinstance(a, str): + self.assertIn(a[1:], ids) + else: + self.assertTrue(0 <= a < 2 ** 32) + self.assertEqual(sum(s[0] == "fg_conv_f16" for s in steps), 17) + self.assertEqual(sum(s[0] == "fg_add_f16" for s in steps), 5) + + def test_weights_are_natural_out_in_ky_kx(self): + data, _, _ = G.plan(32, 32, self.params) + w, _ = self.params["net1_03"] # canonical [out, k, k, in] + nat = np.frombuffer(data["w_net1_03"], np.float16).reshape(w.shape[0], w.shape[3], 3, 3) + self.assertEqual(nat[5, 7, 2, 1], w[5, 2, 1, 7]) + + def test_chain_lines(self): + data, acts, steps = G.plan(32, 32, self.params) + with tempfile.TemporaryDirectory() as d: + files = G.write_files(d, "t", {"cells": b"\0" * 8, **data}) + full = G.chain_lines("t", files, acts, steps) + neg = G.chain_lines("n", files, acts, steps, half=2, stop=2, saves=[steps[2][3]], time_runs=3) + self.assertEqual(sum(l.startswith("step ") for l in full), 32) + self.assertEqual(sum(l.startswith("save ") for l in full), 32) + nsteps = [l for l in neg if l.startswith("step ")] + self.assertEqual(len(nsteps), 3) + self.assertEqual(int(nsteps[2].split()[2]), max(1, steps[2][1] // 2)) + self.assertIn("time 3", neg) + + +if __name__ == "__main__": + unittest.main()