Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions clean/dx12/kernels/fg_pool.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's net1 max-pool (issue #96): fg_maxpool2_f16 of fg_net1.co
// (clean/fg/kernels/fg_net1.hip), byte for byte. 2 x 2 window, stride 2, floor sizes; from the first tap (0, 0), then
// (0, 1), (1, 0), (1, 1), the delivered step m <- (x > m) * x + (x <= m) * m in float16 (fg_pool_step16 of the shared
// integer code clean/fg/kernels/fg_exact16.h). x [C][H][W] float16, y [C][H / 2][W / 2] float16, one output element per
// lane. Built only when --only names it (fg_ kernels are in no model's kernel set).
//
// @wg-kernel fg_maxpool2_f16 co=fg_net1.co block=256 wave=32 args=p,p,u,u,u
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

WG_ARGS_BEGIN
uint x, y; // pointers (arena byte offsets)
int C, H, W;
WG_ARGS_END

WG_KERNEL(fg_maxpool2_f16)
{
WG_SETUP();
const int Ho = H / 2;
const int Wo = W / 2;
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= C * Ho * Wo) return;
const int c = idx / (Ho * Wo);
const int rem = idx - c * (Ho * Wo);
const int oy = rem / Wo;
const int ox = rem - oy * Wo;
const uint p = (uint)((c * H + 2 * oy) * W + 2 * ox) * 2u;
uint m = wg_ld16(x, p);
m = fg_pool_step16(m, wg_ld16(x, p + 2u));
m = fg_pool_step16(m, wg_ld16(x, p + (uint)W * 2u));
m = fg_pool_step16(m, wg_ld16(x, p + (uint)W * 2u + 2u));
wg_st16(y, (uint)idx * 2u, m);
}
3 changes: 3 additions & 0 deletions clean/fg/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c
zero border, float16 accumulator from the bias, 16-term chunks each summed exactly and rounded once, chunk order
`order` (0 ascending, 1 channel block / kx / ky, 2 kx / channel block / ky), `act` 0 none or 1 leaky ReLU. cin is a
multiple of 16.
- `fg_maxpool2_f16(x, y, C, H, W)`: net1's max-pool exactly as `net1.maxpool2`: 2 x 2, stride 2, floor sizes, taps
(0, 0), (0, 1), (1, 0), (1, 1) through the delivered step m <- (x > m) x + (x <= m) m with IEEE products and sums
(0 * inf is NaN; a sum of zeros is -0 only when both are -0).
- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN
results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's
and Model L's kernel sets and DXILs do not change.
Expand Down
2 changes: 1 addition & 1 deletion clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals |
| `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only |
| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none); D3D12 port `clean/dx12/kernels/fg_conv.hlsl` |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) and `fg_maxpool2_f16` (2 x 2, the delivered formula); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl` |
| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
Expand Down
19 changes: 19 additions & 0 deletions clean/fg/kernels/fg_exact16.h
Original file line number Diff line number Diff line change
Expand Up @@ -139,4 +139,23 @@ FG_FN uint fg_leaky16(uint x) {
return fg_key16(t) > fg_key16(x) ? t : x;
}

// 0 * x in IEEE arithmetic, as float16 bits: NaN for x inf or NaN, else a zero with x's sign
FG_FN uint fg_zero_times16(uint x) {
if (fg_is_nan16(x) != 0u || fg_is_inf16(x) != 0u) return FG_NAN16;
return x & 0x8000u;
}

// One step of net1's max-pool, exactly the delivered formula m' = h((x > m) * x + (x <= m) * m) with IEEE products and
// sum: the term that is not taken becomes 0 * value (NaN for an infinity), and a sum of two zeros is -0 only when both
// are -0. Any NaN gives NaN (0x7E00).
FG_FN uint fg_pool_step16(uint m, uint x) {
if (fg_is_nan16(m) != 0u || fg_is_nan16(x) != 0u) return FG_NAN16;
const uint gt = fg_key16(x) > fg_key16(m) ? 1u : 0u; // x > m (+-0 equal; infinities ordered)
const uint v = gt != 0u ? x : m; // the term multiplied by 1
const uint z = gt != 0u ? fg_zero_times16(m) : fg_zero_times16(x); // the term multiplied by 0
if (z == FG_NAN16) return FG_NAN16; // 0 * inf
if ((v & 0x7FFFu) == 0u) return ((v & z & 0x8000u) != 0u) ? 0x8000u : 0u; // zero + zero
return v; // value + (+-0), an infinity included
}

#endif
19 changes: 19 additions & 0 deletions clean/fg/kernels/fg_net1.hip
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,22 @@ WG_KERNEL_N(fg_conv_f16, 256)(const u16* __restrict__ x, const u16* __restrict__
if (act == 1) acc = fg_leaky16(acc);
y[idx] = (u16)acc;
}

// fg_maxpool2_f16: net1's 2 x 2 max-pool, stride 2, floor sizes (Ho = H / 2, Wo = W / 2): from the first tap (dy, dx) =
// (0, 0), then (0, 1), (1, 0), (1, 1), the delivered step m <- (x > m) * x + (x <= m) * m in float16 (fg_pool_step16).
// x [C][H][W], y [C][Ho][Wo]; output element idx = (c * Ho + oy) * Wo + ox.
WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y, int C, int H, int W) {
const int Ho = H / 2, Wo = W / 2;
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= C * Ho * Wo) return;
const int c = idx / (Ho * Wo);
const int rem = idx - c * (Ho * Wo);
const int oy = rem / Wo;
const int ox = rem - oy * Wo;
const u16* p = x + (c * H + 2 * oy) * W + 2 * ox;
uint m = p[0];
m = fg_pool_step16(m, p[1]);
m = fg_pool_step16(m, p[W]);
m = fg_pool_step16(m, p[W + 1]);
y[idx] = (u16)m;
}
63 changes: 49 additions & 14 deletions clean/fg/runner/fg_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
- compares the RAW downloaded bytes of both back ends; each output buffer starts filled with its back end's own
sentinel (HIP 0xA5, D3D12 0x5A) and has 64 slack bytes after it that neither may write, so an unwritten byte or a
write past the end differs;
- a negative control (the first case with 4 or more workgroups, again on D3D12 with half of them) must be caught;
- a negative control per kernel (its first case with 4 or more workgroups, again on D3D12 with half of them) must be
caught;
- compares the HIP output with the CPU reference, value for value (float16 bits; NaN equals NaN, the reference's NaN
payload is not specified).
Cases: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several sizes, and
special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights. Output: counts only, no timing.
Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several
sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps
of several sizes (odd ones too) and every 2 x 2 window over ten special values. Output: counts only, no timing.
"""
import argparse
import ctypes
Expand Down Expand Up @@ -103,7 +105,37 @@ def _conv_case(kernel, x, w, b, k, order, act, label):
b.astype(np.float64), chunks=order, act=act)}


KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True)}
POOL_SPECIALS = np.array([1.0, -2.0, np.inf, -np.inf, np.nan, 0.0, -0.0, 65504.0, -65504.0, 2.0 ** -24],
dtype=np.float16)


def pool_cases(kernel, rng):
"""Random maps (odd sizes drop their last row / column), and every window of four taps over POOL_SPECIALS (10^4
windows side by side in one 2-row map)."""
cases = []
for C, H, W in ((3, 7, 9), (32, 8, 8), (64, 5, 5), (1, 2, 2), (16, 3, 17), (128, 4, 6)):
x = _f16(rng.uniform(-4, 4, (C, H, W)))
cases.append(_pool_case(kernel, x, f"{C}x{H}x{W}"))
v = POOL_SPECIALS
n = len(v)
idx = np.arange(n ** 4)
taps = [v[(idx // n ** p) % n] for p in range(4)] # (0,0), (0,1), (1,0), (1,1) of each window
x = np.zeros((1, 2, 2 * n ** 4), np.float16)
x[0, 0, 0::2], x[0, 0, 1::2], x[0, 1, 0::2], x[0, 1, 1::2] = taps
cases.append(_pool_case(kernel, x, "every special window"))
return cases


def _pool_case(kernel, x, label):
C, H, W = x.shape
n = C * (H // 2) * (W // 2)
return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label,
"args": [("buf", x.tobytes()), ("out", 2 * n + SLACK), ("u32", C), ("u32", H), ("u32", W)],
"defined": 2 * n, "cpu": lambda: N.maxpool2(x.astype(np.float64))}


KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True),
"fg_maxpool2_f16": pool_cases}


# ---- prepare --------------------------------------------------------------------------------------------------------
Expand Down Expand Up @@ -193,7 +225,7 @@ def run(build, rocm, seeds, kernels):
mod = h.load_module(f.read())
cdir = os.path.join(build, "replay-fg")
os.makedirs(cdir, exist_ok=True)
lines, hip_out, neg = [], {}, None
lines, hip_out, negs = [], {}, {} # negs: per kernel, its first case with 4 or more workgroups
for kernel in kernels:
for si, seed in enumerate(seeds):
rng = np.random.default_rng(seed)
Expand All @@ -202,8 +234,8 @@ def run(build, rocm, seeds, kernels):
outs = hip_run(h, mod, case)
hip_out[name] = (kernel, case, outs)
lines += replay_lines(name, case, cdir)
if neg is None and case["grid"] >= 4:
neg = name
if kernel not in negs and case["grid"] >= 4:
negs[kernel] = name
lines += replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_")
with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f:
f.write("\n".join(lines))
Expand All @@ -227,15 +259,18 @@ def read(path):
ok = ok and good
res["cases"].append({"case": name, "kernel": kernel, "label": case["label"], "raw": raw, "cpu": cpu,
"ok": good})
if neg:
kernel, case, outs = hip_out[neg]
res["negative_controls"] = {}
for kernel in kernels: # every kernel needs its own caught negative control
neg = negs.get(kernel)
if neg is None:
res["negative_controls"][kernel] = None
ok = False
continue
_, case, outs = hip_out[neg]
nraw = compare_raw(read(os.path.join(cdir, f"negative_{neg}.o0")), outs[0], case["defined"])
caught = nraw["differ"] != 0 or not nraw["slack_ok"]
res["negative_control"] = {"case": neg, "half_grid": True, "caught": caught, "differ": nraw["differ"]}
res["negative_controls"][kernel] = {"case": neg, "half_grid": True, "caught": caught, "differ": nraw["differ"]}
ok = ok and caught
else:
res["negative_control"] = None
ok = False
res["summary"] = {"cases": len(res["cases"]), "cases_ok": sum(c["ok"] for c in res["cases"]),
"values": sum(c["cpu"]["values"] for c in res["cases"]),
"nan_values": sum(c["cpu"]["nan"] for c in res["cases"]),
Expand All @@ -262,7 +297,7 @@ def main(argv=None):
if a.json:
with open(a.json, "w", encoding="utf-8", newline="\n") as f:
f.write(text + "\n")
print(json.dumps(res["summary"]), "negative control:", json.dumps(res["negative_control"]))
print(json.dumps(res["summary"]), "negative controls:", json.dumps(res["negative_controls"]))
for c in res["cases"]:
if not c["ok"]:
print("FAIL", c["case"], c["label"], json.dumps(c["raw"]), json.dumps(c["cpu"]))
Expand Down
Loading