Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions clean/dx12/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ as its HIP kernel (`clean/dx12/compare.py` proves it per kernel on synthetic fra
| `dxil_asm.cpp` | tool used by the build (assembles the rewritten DXIL listing) |
| `compare.py` | HIP vs hybrid D3D12, every stage output / state buffer byte for byte |
| `kernels/exposure.hlsl` | the reference port (`exposure_auto_diet`): copy its structure |
| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; check: `clean/fg/runner/fg_d3d12_check.py` |

## Arithmetic mode (why build_dx12.py and nothing else)

Expand Down
17 changes: 15 additions & 2 deletions clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
python clean/dx12/build_dx12.py --out build/dx12-m/hip [--isa] [--only exposure_auto_diet,...] [-j N] [--model-l]

Model L's kernels (#89: those that reproduce one of Model L's code objects, co=l_*.co, direct-route kernels named l_*, and MODEL_L_ONLY) are built only with --model-l
or when --only names them, so that a build without the option is Model M's kernel set unchanged.
or when --only names them, so that a build without the option is Model M's kernel set unchanged. Frame generation's
kernels (#96: co=fg_*.co or named fg_*) are built only when --only names them, never by a full build.

Every kernel source declares its kernels in header lines (clean/dx12/README.md):

Expand Down Expand Up @@ -186,13 +187,23 @@ def is_model_l(decl):
return decl["co"].startswith("l_") or decl["name"].startswith("l_") or decl["name"] in MODEL_L_ONLY


def is_frame_gen(decl):
"""A port of frame generation's kernels (#96: code objects fg_*.co, or named fg_*). They are groundwork that no model
build launches yet: built only when --only names them, never by a full build (with or without --model-l), so the
Model M and Model L kernel sets and their DXILs stay as they are."""
return decl["co"].startswith("fg_") or decl["name"].startswith("fg_")


def kernel_set(model_l=False):
"""(name, code object) of every kernel a full build compiles (no --only), sorted: Model M's set, plus Model L's
with model_l. From the sources' declarations alone (nothing is compiled)."""
with model_l. From the sources' declarations alone (nothing is compiled). Frame generation's kernels are never in
it (is_frame_gen)."""
keys = set()
for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []:
if fn.endswith(".hlsl"):
for d in kernel_decls(os.path.join(KERNEL_DIR, fn)):
if is_frame_gen(d):
continue
if model_l or not is_model_l(d):
keys.add((d["name"], d["co"]))
return sorted(keys)
Expand Down Expand Up @@ -321,6 +332,8 @@ def main(argv=None):
continue
if not only and not a.model_l and is_model_l(d):
continue
if not only and is_frame_gen(d):
continue
key = (d["name"], d["co"])
if key in kernels and key not in old:
raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice")
Expand Down
68 changes: 68 additions & 0 deletions clean/dx12/kernels/fg_conv.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's net1 convolution (issue #96): fg_conv_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip),
// byte for byte. The arithmetic is the shared integer code of clean/fg/kernels/fg_exact16.h (exact float16 chunk
// accumulation, one nearest-even rounding per chunk, the leaky ReLU), so both back ends run the same operations; only
// memory access differs (byte offsets into the arena here).
//
// One output element per lane: idx = (o * H + y) * W + x of [cout][H][W] float16; x [cin][H][W] float16; w natural
// [cout][cin][k][k] float16; b [cout] float16. Stride 1, zero border; chunk order: 0 ascending (1 x 1), 1 channel block,
// kx, ky; 2 kx, channel block, ky. act: 0 none, 1 leaky ReLU. Not part of Model M's or Model L's kernel set (fg_ kernels
// are built only when --only names them).
//
// @wg-kernel fg_conv_f16 co=fg_net1.co block=256 wave=32 args=p,p,p,p,u,u,u,u,u,u,u
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

WG_ARGS_BEGIN
uint x, w, b, y; // pointers (arena byte offsets)
int cin, cout, H, W, k, order, act;
WG_ARGS_END

WG_KERNEL(fg_conv_f16)
{
WG_SETUP();
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= cout * H * W) return;
const int o = idx / (H * W);
const int rem = idx - o * (H * W);
const int yy = rem / W;
const int xx = rem - yy * W;
const int p = k / 2;
const int nb = cin / 16;
const int taps = k * k;
uint acc = wg_ld16(b, (uint)o * 2u);
[loop] for (int n = 0; n < nb * taps; ++n) {
int cb, kx, ky;
if (order == 0) {
cb = n; kx = 0; ky = 0;
} else if (order == 1) {
cb = n / taps;
const int t = n - cb * taps;
kx = t / k;
ky = t - kx * k;
} else {
kx = n / (nb * k);
const int r = n - kx * (nb * k);
cb = r / k;
ky = r - cb * k;
}
const int sy = yy + ky - p;
const int sx = xx + kx - p;
const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W;
FgAcc s = fg_acc_begin(acc);
[loop] for (int i = 0; i < 16; ++i) {
const int c = cb * 16 + i;
const uint wv = wg_ld16(w, (uint)(((o * cin + c) * k + ky) * k + kx) * 2u);
uint xv = 0u; // outside the grid: 0 (0 * inf is NaN)
if (inside) xv = wg_ld16(x, (uint)((c * H + sy) * W + sx) * 2u);
s = fg_acc_add(s, wv, xv);
}
acc = fg_acc_end(s);
}
if (act == 1) acc = fg_leaky16(acc);
wg_st16(y, (uint)idx * 2u, acc);
}
13 changes: 13 additions & 0 deletions clean/dx12/tests/test_kernel_sets.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,19 @@ def test_model_l_adds_only_model_l_kernels(self):
self.assertFalse([k for k in extra if not (k[1].startswith("l_") or k[0].startswith("l_") or k[0] in B.MODEL_L_ONLY)])


def test_frame_gen_kernels_are_in_no_set(self):
"""Frame generation's ports (#96, co=fg_*.co or named fg_*) are declared but enter neither set: only --only
builds them, so the Model M and Model L builds stay as they are."""
decls = []
for fn in sorted(os.listdir(B.KERNEL_DIR)):
if fn.endswith(".hlsl"):
decls += B.kernel_decls(os.path.join(B.KERNEL_DIR, fn))
fg = [(d["name"], d["co"]) for d in decls if B.is_frame_gen(d)]
self.assertTrue(fg, "no frame-generation kernel declared")
full = set(B.kernel_set(True))
self.assertFalse([k for k in fg if k in full])


if __name__ == "__main__":
if "--write" in sys.argv:
with open(SETS, "w", newline="\n") as f:
Expand Down
14 changes: 14 additions & 0 deletions clean/fg/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,20 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c
- Not part of net1 and still open: what the 10 input values are made of (images A-D, the warps) and the input grid
size relative to the output.

## GPU kernels (`kernels/`, code object `fg_net1.co`; D3D12 ports `clean/dx12/kernels/fg_*.hlsl`)

- Groundwork: no runtime launches them yet. Tensors float16 channel-first `[c][h][w]`, weights natural
`[out][in][k][k]` float16 (the map's `W`), biases `[out]` float16; a 1-D grid of 256-lane workgroups, one output
element per lane.
- `fg_conv_f16(x, w, b, y, cin, cout, H, W, k, order, act)`: one net1 convolution exactly as `net1.conv16`: stride 1,
zero border, float16 accumulator from the bias, 16-term chunks each summed exactly and rounded once, chunk order
`order` (0 ascending, 1 channel block / kx / ky, 2 kx / channel block / ky), `act` 0 none or 1 leaky ReLU. cin is a
multiple of 16.
- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN
results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's
and Model L's kernel sets and DXILs do not change.
- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference.

## Configuration (`reference/config.py`)

- No configuration value changes the networks or their weights.
Expand Down
3 changes: 3 additions & 0 deletions clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `reference/fp16acc.py` | exact float16 chunk accumulation: RN16(acc + 16 products) with integer arithmetic |
| `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals |
| `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only |
| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none); D3D12 port `clean/dx12/kernels/fg_conv.hlsl` |
| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
| `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` |
Expand Down
142 changes: 142 additions & 0 deletions clean/fg/kernels/fg_exact16.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// SPDX-License-Identifier: MIT
// Frame generation (issue #96): exact float16 arithmetic in integers, shared by the HIP kernels (clean/fg/kernels/*.hip)
// and their D3D12 ports (clean/dx12/kernels/fg_*.hlsl), so both back ends compute the same bits by construction.
//
// The same method as the CPU reference clean/fg/reference/fp16acc.py: a finite float16 value is m * 2^-24 with an integer
// |m| < 2^40; m splits into a high and a low 20-bit part, so a 16-term dot product is exact in three 64-bit sums
// (high * high, high * low + low * high, low * low). The accumulator acc' = RN16(acc + sum of a_i * b_i) is rounded once,
// to nearest with ties to even, by integer arithmetic only. inf and NaN follow IEEE (any NaN, 0 * inf or inf - inf gives
// NaN, written as 0x7E00; otherwise an infinite term gives that infinity).
//
// The including file defines:
// FG_FN function qualifier (HIP: WG_FN; HLSL: nothing)
// FG_I64 signed 64-bit integer (HIP: long long; HLSL: int64_t)
// FG_FBH32(v) index of the highest set bit of a non-zero 32-bit unsigned value
#ifndef FG_EXACT16_H
#define FG_EXACT16_H

#define FG_NAN16 0x7E00u
#define FG_LOW20 1048575

struct FgAcc {
FG_I64 hh; // sum of high * high (units 2^-8)
FG_I64 mid; // sum of high * low + low * high, plus 16 * m(acc) (units 2^-28)
FG_I64 ll; // sum of low * low (units 2^-48), never negative
uint nan_; // 1: the result is NaN
uint pinf; // 1: a +inf term
uint ninf; // 1: a -inf term
};

FG_FN uint fg_is_nan16(uint h) { return ((h & 0x7C00u) == 0x7C00u && (h & 0x3FFu) != 0u) ? 1u : 0u; }
FG_FN uint fg_is_inf16(uint h) { return ((h & 0x7FFFu) == 0x7C00u) ? 1u : 0u; }

// floor(log2(v)) of a 64-bit value v > 0
FG_FN uint fg_log2_64(FG_I64 v) {
const uint hi = (uint)(v >> 32);
const uint lo = (uint)v; // the low 32 bits
return hi != 0u ? 32u + FG_FBH32(hi) : FG_FBH32(lo);
}

// a finite float16 (bits) -> m with value = m * 2^-24
FG_FN FG_I64 fg_m16(uint h) {
const uint e = (h >> 10) & 31u;
const uint f = h & 1023u;
FG_I64 m = (FG_I64)f;
if (e != 0u) m = (FG_I64)(f | 1024u) * ((FG_I64)1 << (e - 1u));
return (h & 0x8000u) != 0u ? -m : m;
}

FG_FN FgAcc fg_acc_begin(uint acc) {
FgAcc s;
s.hh = 0;
s.mid = 0;
s.ll = 0;
s.nan_ = fg_is_nan16(acc);
s.pinf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) == 0u) ? 1u : 0u;
s.ninf = (fg_is_inf16(acc) != 0u && (acc & 0x8000u) != 0u) ? 1u : 0u;
if ((acc & 0x7C00u) != 0x7C00u) s.mid = fg_m16(acc) * 16;
return s;
}

// s + a * b (a, b float16 bits)
FG_FN FgAcc fg_acc_add(FgAcc s, uint a, uint b) {
if (fg_is_nan16(a) != 0u || fg_is_nan16(b) != 0u) {
s.nan_ = 1u;
} else if (fg_is_inf16(a) != 0u || fg_is_inf16(b) != 0u) {
if ((a & 0x7FFFu) == 0u || (b & 0x7FFFu) == 0u) {
s.nan_ = 1u; // 0 * inf
} else if (((a ^ b) & 0x8000u) != 0u) {
s.ninf = 1u;
} else {
s.pinf = 1u;
}
} else {
const FG_I64 ma = fg_m16(a);
const FG_I64 mb = fg_m16(b);
const FG_I64 ah = ma >> 20, al = ma & FG_LOW20;
const FG_I64 bh = mb >> 20, bl = mb & FG_LOW20;
s.hh += ah * bh;
s.mid += ah * bl + al * bh;
s.ll += al * bl;
}
return s;
}

// RN16 of the exact value (hh * 2^40 + mid * 2^20 + ll) * 2^-48, ll >= 0, as float16 bits
FG_FN uint fg_round16(FG_I64 hh, FG_I64 mid, FG_I64 ll) {
const FG_I64 lim = (FG_I64)1 << 30;
const uint big = (hh > lim || hh < -lim) ? 1u : 0u; // |value| >= 2^22: infinite
const FG_I64 hhc = hh > lim ? lim : (hh < -lim ? -lim : hh);
// value * 2^25 = hh * 2^17 + mid2 / 8 + llr / 2^23, with mid2 = mid + (ll >> 20) and 0 <= llr < 2^20
const FG_I64 l1 = ll >> 20;
const FG_I64 llr = ll & FG_LOW20;
const FG_I64 mid2 = mid + l1;
const FG_I64 m1 = mid2 >> 3; // floor
const FG_I64 midr = mid2 & 7;
const FG_I64 q = hhc * 131072 + m1; // floor of value * 2^25
const uint sticky = (midr != 0 || llr != 0) ? 1u : 0u;
const uint neg = q < 0 ? 1u : 0u;
FG_I64 mag = q;
if (neg != 0u) mag = sticky != 0u ? -q - 1 : -q;
// quantum in units of 2^-25: 2 below 2^-14 (subnormal), else 2^(e - 10) in the binade [2^e, 2^(e + 1)) units
const uint e = fg_log2_64(mag > 0 ? mag : 1);
const uint qexp = mag < 2048 ? 1u : e - 10u;
const FG_I64 quantum = (FG_I64)1 << qexp;
const FG_I64 r = mag & (quantum - 1);
const FG_I64 down = mag - r;
const FG_I64 half = quantum >> 1;
const uint odd = (uint)((down >> qexp) & 1);
const uint up = (r > half || (r == half && (sticky != 0u || odd != 0u))) ? 1u : 0u;
const FG_I64 res = down + (up != 0u ? quantum : 0); // the rounded magnitude, units 2^-25
const uint sign = big != 0u ? (hh < 0 ? 1u : 0u) : neg;
uint bits;
if (big != 0u || res >= ((FG_I64)65536 << 25)) {
bits = 0x7C00u;
} else if (res < 2048) {
bits = (uint)(res >> 1); // subnormal (or zero)
} else {
const uint e2 = fg_log2_64(res);
bits = ((e2 - 10u) << 10) | ((uint)(res >> (e2 - 10u)) & 1023u);
}
return bits | (sign << 15);
}

FG_FN uint fg_acc_end(FgAcc s) {
if (s.nan_ != 0u || (s.pinf != 0u && s.ninf != 0u)) return FG_NAN16;
if (s.pinf != 0u) return 0x7C00u;
if (s.ninf != 0u) return 0xFC00u;
return fg_round16(s.hh, s.mid, s.ll);
}

// the order of a float16 value for max (-0 and +0 are equal); not for NaN
FG_FN int fg_key16(uint h) { return (h & 0x8000u) != 0u ? -(int)(h & 0x7FFFu) : (int)(h & 0x7FFFu); }

// net1's activation: y = max(x, h(s * x)) with the delivered slope s (float16 0x211E); NaN stays NaN
#define FG_LEAKY_SLOPE16 0x211Eu
FG_FN uint fg_leaky16(uint x) {
if (fg_is_nan16(x) != 0u) return x;
const uint t = fg_acc_end(fg_acc_add(fg_acc_begin(0u), FG_LEAKY_SLOPE16, x));
return fg_key16(t) > fg_key16(x) ? t : x;
}

#endif
60 changes: 60 additions & 0 deletions clean/fg/kernels/fg_net1.hip
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: MIT
// Frame generation (issue #96): HIP kernels of net1's operations (clean/fg/reference/net1.py), bit for bit against the
// CPU reference. D3D12 ports: clean/dx12/kernels/fg_*.hlsl. Tensors are float16, channel-first [c][h][w]; weights are
// natural [out][in][k][k] float16; biases [out] float16. 1-D grid of 256-lane workgroups, one output element per lane.
#include "wg_platform.h"
#include "wg_mem.h"

typedef unsigned int uint;
typedef unsigned short u16;
#define FG_FN WG_FN
#define FG_I64 long long
#define FG_FBH32(v) (31u - (uint)__builtin_clz(v))
#include "fg_exact16.h"

// fg_conv_f16: one convolution of net1. Stride 1, zero border (k = 3) or none (k = 1), cross-correlation; the float16
// accumulator starts at the bias and takes 16-term chunks (16 input channels of one tap), each summed exactly and added
// with one float16 rounding, in the chunk order `order`: 0 ascending (1 x 1: channel blocks in order), 1 channel block
// outer, then kx, then ky; 2 kx outer, then channel block, then ky. act: 0 none, 1 leaky ReLU (slope 0x211E).
// cin must be a multiple of 16. Output element idx = (o * H + y) * W + x.
WG_KERNEL_N(fg_conv_f16, 256)(const u16* __restrict__ x, const u16* __restrict__ w, const u16* __restrict__ b,
u16* __restrict__ y, int cin, int cout, int H, int W, int k, int order, int act) {
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= cout * H * W) return;
const int o = idx / (H * W);
const int rem = idx - o * (H * W);
const int yy = rem / W;
const int xx = rem - yy * W;
const int p = k / 2;
const int nb = cin / 16;
const int taps = k * k;
uint acc = b[o];
for (int n = 0; n < nb * taps; ++n) {
int cb, kx, ky;
if (order == 0) {
cb = n; kx = 0; ky = 0;
} else if (order == 1) {
cb = n / taps;
const int t = n - cb * taps;
kx = t / k;
ky = t - kx * k;
} else {
kx = n / (nb * k);
const int r = n - kx * (nb * k);
cb = r / k;
ky = r - cb * k;
}
const int sy = yy + ky - p, sx = xx + kx - p;
const bool inside = sy >= 0 && sy < H && sx >= 0 && sx < W;
FgAcc s = fg_acc_begin(acc);
for (int i = 0; i < 16; ++i) {
const int c = cb * 16 + i;
const uint wv = w[((o * cin + c) * k + ky) * k + kx];
const uint xv = inside ? (uint)x[(c * H + sy) * W + sx] : 0u; // outside the grid: 0 (0 * inf is NaN)
s = fg_acc_add(s, wv, xv);
}
acc = fg_acc_end(s);
}
if (act == 1) acc = fg_leaky16(acc);
y[idx] = (u16)acc;
}
1 change: 1 addition & 0 deletions clean/fg/runner/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# SPDX-License-Identifier: MIT
Loading
Loading