Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion clean/dx12/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ as its HIP kernel (`clean/dx12/compare.py` proves it per kernel on synthetic fra
| `dxil_asm.cpp` | tool used by the build (assembles the rewritten DXIL listing) |
| `compare.py` | HIP vs hybrid D3D12, every stage output / state buffer byte for byte |
| `kernels/exposure.hlsl` | the reference port (`exposure_auto_diet`): copy its structure |
| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; check: `clean/fg/runner/fg_d3d12_check.py` |
| `kernels/fg_*.hlsl` | frame generation's ports (#96, `co=fg_*.co`): groundwork, built only when `--only` names them, never by a full build (`is_frame_gen`), so the Model M / Model L sets stay as they are; checks: `clean/fg/runner/fg_d3d12_check.py` (each kernel), `clean/fg/runner/fg_net1_gpu.py` (net1's 32 steps as a `kernel_replay` chain) |
| `tests/kernel_replay.cpp` | replays launches from a case file on the native D3D12 device; `chain` blocks run several launches on buffers that persist between them (a whole network, optionally timed) |

## Arithmetic mode (why build_dx12.py and nothing else)

Expand Down
28 changes: 28 additions & 0 deletions clean/dx12/kernels/fg_add.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: MIT
// D3D12 port of frame generation's net1 skip add (issue #96): fg_add_f16 of fg_net1.co (clean/fg/kernels/fg_net1.hip),
// byte for byte. y = h(a + b) per element of n float16 values: the single-precision sum (one IEEE operation), then
// float16 by wg_f2h (nearest even); single precision then float16 equals one rounding of the exact sum of two float16
// values. NaN results are written as 0x7E00. Built only when --only names it (fg_ kernels are in no model's set).
//
// @wg-kernel fg_add_f16 co=fg_net1.co block=256 wave=32 args=p,p,p,u
#include "wg_platform.hlsli"

#define FG_FN
#define FG_I64 int64_t
#define FG_FBH32(v) firstbithigh((uint)(v))
#include "../../fg/kernels/fg_exact16.h"

WG_ARGS_BEGIN
uint a, b, y; // pointers (arena byte offsets)
int n;
WG_ARGS_END

WG_KERNEL(fg_add_f16)
{
WG_SETUP();
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= n) return;
uint outv = wg_f2h(wg_h2f(wg_ld16(a, (uint)idx * 2u)) + wg_h2f(wg_ld16(b, (uint)idx * 2u)));
if (fg_is_nan16(outv) != 0u) outv = FG_NAN16;
wg_st16(y, (uint)idx * 2u, outv);
}
153 changes: 151 additions & 2 deletions clean/dx12/tests/kernel_replay.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,22 @@
// direct_out <file> after the launch, the output texture's texels are written to <file>,
// row by row (tightly packed); its initial texels come from the output
// slot's file (a sentinel fill, so that an unwritten texel shows)
//
// A chain (frame generation, #96) runs several launches in order on buffers that persist between them, so that a whole
// network runs on the device with no host round trip between its steps:
// chain <name>
// module <code object name> [<tag>] (zero or more)
// buf <id> <bytes> <file | -> [fill] device buffer: the file's bytes (padded with fill when the file is
// shorter), or every byte set to fill (0..255, default 0) with "-"
// step <launch name> <grid> <block> <arg>... a launch; each arg is @<id> (a buffer's pointer) or a 32-bit decimal
// save <id> <file> after the run, the buffer's bytes are written to <file>
// time <runs> after the run (and the saves), the steps run <runs> more times; the mean
// wall time per run (submission to completion, synchronised after each
// run) is printed as "chain <name>: time <ms> ms per run (<runs> runs)"
#include "../../runtime/device.h"
#include "../../runtime/device_d3d12.h"

#include <chrono>
#include <cstdint>
#include <cstdio>
#include <cstring>
Expand Down Expand Up @@ -337,6 +350,109 @@ int run_case(Device* d, Case& c, const std::string& base) {
return 0;
}

struct ChainBuf {
std::string id, file;
size_t bytes = 0;
int fill = 0;
DevPtr ptr = 0;
};

struct ChainStep {
std::string kernel;
unsigned grid = 0, block = 0;
std::vector<std::string> args; // "@id" or a decimal value
};

struct Chain {
std::string name;
std::vector<std::pair<std::string, std::string>> modules;
std::vector<ChainBuf> bufs;
std::vector<ChainStep> steps;
std::vector<std::pair<std::string, std::string>> saves; // (id, file)
int time_runs = 0;
};

int run_chain(Device* d, Chain& c, const std::string& base) {
for (auto& m : c.modules) {
const bool ok = m.second.empty() ? d->load_module(m.first) : d->load_module_tagged(m.first, m.second);
if (!ok) { std::printf("%s: module %s: %s\n", c.name.c_str(), m.first.c_str(), d->err().c_str()); return 1; }
}
auto find = [&](const std::string& id) -> ChainBuf* {
for (auto& b : c.bufs)
if (b.id == id) return &b;
return nullptr;
};
int rc = 0;
for (auto& b : c.bufs) {
std::vector<uint8_t> data(b.bytes, (uint8_t)(b.fill & 0xff));
if (b.file != "-") {
std::vector<uint8_t> f;
if (!read_file(base + "/" + b.file, &f) || f.size() > b.bytes) {
std::printf("%s: buf %s: cannot read %s (or larger than %zu bytes)\n", c.name.c_str(), b.id.c_str(),
b.file.c_str(), b.bytes);
rc = 1;
break;
}
std::memcpy(data.data(), f.data(), f.size());
}
if (!d->alloc(&b.ptr, b.bytes ? b.bytes : 4) || !d->upload(b.ptr, data.data(), data.size())) {
std::printf("%s: buf %s: %s\n", c.name.c_str(), b.id.c_str(), d->err().c_str());
rc = 1;
break;
}
}
// the argument values of every step, resolved once
std::vector<std::vector<DevPtr>> ptrs(c.steps.size());
std::vector<std::vector<uint32_t>> vals(c.steps.size());
std::vector<std::vector<void*>> argv(c.steps.size());
for (size_t i = 0; rc == 0 && i < c.steps.size(); ++i) {
auto& s = c.steps[i];
ptrs[i].reserve(s.args.size());
vals[i].reserve(s.args.size());
for (auto& a : s.args) {
if (!a.empty() && a[0] == '@') {
ChainBuf* b = find(a.substr(1));
if (!b) { std::printf("%s: step %zu: unknown buffer %s\n", c.name.c_str(), i, a.c_str()); rc = 1; break; }
ptrs[i].push_back(b->ptr);
argv[i].push_back(&ptrs[i].back());
} else {
vals[i].push_back((uint32_t)std::stoll(a));
argv[i].push_back(&vals[i].back());
}
}
}
auto run_steps = [&]() -> bool {
for (size_t i = 0; i < c.steps.size(); ++i) {
if (!d->launch(c.steps[i].kernel, c.steps[i].grid, c.steps[i].block, argv[i].data())) {
std::printf("%s: step %zu launch %s: %s\n", c.name.c_str(), i, c.steps[i].kernel.c_str(), d->err().c_str());
return false;
}
}
if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return false; }
return true;
};
if (rc == 0 && !run_steps()) rc = 1;
for (size_t i = 0; rc == 0 && i < c.saves.size(); ++i) {
ChainBuf* b = find(c.saves[i].first);
std::vector<uint8_t> data(b ? b->bytes : 0);
if (!b || !d->download(data.data(), b->ptr, b->bytes) || !write_file(base + "/" + c.saves[i].second, data)) {
std::printf("%s: save %s: %s\n", c.name.c_str(), c.saves[i].first.c_str(), b ? d->err().c_str() : "unknown");
rc = 1;
}
}
if (rc == 0 && c.time_runs > 0) {
const auto t0 = std::chrono::steady_clock::now();
for (int r = 0; rc == 0 && r < c.time_runs; ++r)
if (!run_steps()) rc = 1;
const double ms = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count();
if (rc == 0) std::printf("chain %s: time %.3f ms per run (%d runs)\n", c.name.c_str(), ms / c.time_runs, c.time_runs);
}
for (auto& b : c.bufs)
if (b.ptr) d->free(b.ptr);
if (rc == 0) std::printf("%s: ok (chain of %zu steps)\n", c.name.c_str(), c.steps.size());
return rc;
}

} // namespace

int main(int argc, char** argv) {
Expand All @@ -349,13 +465,45 @@ int main(int argc, char** argv) {
if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; }
const std::string base = dir_of(argv[2]);
std::vector<Case> cases;
std::vector<Chain> chains;
bool in_chain = false; // the lines after "chain" belong to it until the next "case" or "chain"
std::string line;
while (std::getline(f, line)) {
if (!line.empty() && line.back() == '\r') line.pop_back();
std::istringstream s(line);
std::string w;
if (!(s >> w)) continue;
if (w == "case") { cases.emplace_back(); s >> cases.back().name; continue; }
if (w == "case") { cases.emplace_back(); s >> cases.back().name; in_chain = false; continue; }
if (w == "chain") { chains.emplace_back(); s >> chains.back().name; in_chain = true; continue; }
if (in_chain) {
Chain& ch = chains.back();
if (w == "module") {
std::string m, tag;
s >> m >> tag;
ch.modules.emplace_back(m, tag);
} else if (w == "buf") {
ChainBuf b;
s >> b.id >> b.bytes >> b.file;
if (!(s >> b.fill)) b.fill = 0;
ch.bufs.push_back(b);
} else if (w == "step") {
ChainStep st;
s >> st.kernel >> st.grid >> st.block;
std::string a;
while (s >> a) st.args.push_back(a);
ch.steps.push_back(st);
} else if (w == "save") {
std::string id, file;
s >> id >> file;
ch.saves.emplace_back(id, file);
} else if (w == "time") {
s >> ch.time_runs;
} else {
std::printf("unknown chain line: %s\n", line.c_str());
return 2;
}
continue;
}
if (cases.empty()) { std::printf("line before the first case: %s\n", line.c_str()); return 2; }
Case& c = cases.back();
if (w == "module") {
Expand Down Expand Up @@ -398,7 +546,8 @@ int main(int argc, char** argv) {
}
int fails = 0;
for (auto& c : cases) fails += run_case(d.get(), c, base);
for (auto& c : chains) fails += run_chain(d.get(), c, base);
if (!d->sync()) { std::printf("final sync: %s\n", d->err().c_str()); return 1; }
std::printf("kernel_replay: %d of %zu cases failed\n", fails, cases.size());
std::printf("kernel_replay: %d of %zu cases failed\n", fails, cases.size() + chains.size());
return fails ? 1 : 0;
}
7 changes: 6 additions & 1 deletion clean/fg/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,10 +112,15 @@ delivered by the spec side; the code never guesses it (it raises, or takes the c
(0 for an extent of 1), so no division runs on the GPU. Positions in single precision one operation at a time
(HIP: no contraction); every product and sum rounded to float16 (a product of two float16 values is exact in single
precision, and single precision then float16 equals one rounding of a sum of two float16 values).
- `fg_add_f16(a, b, y, n)`: net1's skip add exactly as `net1.add16`: y = h(a + b) per element (single precision then
float16, one rounding of the exact sum).
- net1 on the GPU is these four kernels in NET1_WIRING's order, one launch per step, every buffer resident on the
device (`runner/fg_net1_gpu.py`; on D3D12 a kernel_replay `chain` block, with a UAV barrier between launches).
- The arithmetic is integer code shared by both back ends (`kernels/fg_exact16.h`), the method of `fp16acc.py`; NaN
results are written as 0x7E00. The D3D12 build compiles `fg_` kernels only when `--only` names them, so Model M's
and Model L's kernel sets and DXILs do not change.
- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference.
- `runner/fg_d3d12_check.py` proves each kernel on HIP against D3D12 (raw bytes) and against the CPU reference;
`runner/fg_net1_gpu.py` the whole network, every step output.

## Configuration (`reference/config.py`)

Expand Down
4 changes: 3 additions & 1 deletion clean/fg/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORM
| `tests/test_fg_net1.py`, `tests/test_fg_fp16acc.py` | net1 end to end against a hand-composed version; one convolution per chunk order against exact rationals; the upscale against the scalar rule; pooling with inf / NaN; the accumulation against exact rationals |
| `tools/net1_digest.py` | net1 on a converter model file with a fixed synthetic input; records digests of the output only |
| `kernels/fg_exact16.h` | exact float16 arithmetic in integers (chunk accumulation, one nearest-even rounding, leaky ReLU), shared by the HIP kernels and their D3D12 ports, so both compute the same bits by construction |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula) and `fg_upscale_f16` (bilinear, align corners, float16 steps); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl` |
| `kernels/fg_net1.hip` | HIP kernels of net1's operations (code object `fg_net1.co`): `fg_conv_f16` (3 x 3 / 1 x 1, zero border, chunk orders, leaky ReLU or none) , `fg_maxpool2_f16` (2 x 2, the delivered formula), `fg_upscale_f16` (bilinear, align corners, float16 steps) and `fg_add_f16` (the skip add); D3D12 ports `clean/dx12/kernels/fg_conv.hlsl`, `fg_pool.hlsl`, `fg_upscale.hlsl`, `fg_add.hlsl` |
| `runner/fg_d3d12_check.py` | GPU check (one GPU slot): every kernel on HIP and D3D12 byte for byte (sentinels 0xA5 / 0x5A, slack bytes, a half-grid negative control), and the HIP output against the CPU reference bit for bit; net1's layer shapes on several grid sizes and special values |
| `runner/fg_net1_gpu.py` | net1's 32 steps chained on the GPU (HIP, and D3D12 through kernel_replay's `chain` blocks): every step output on both back ends byte for byte and against the CPU reference bit for bit, a half-grid negative control per kernel, synthetic weights (also odd sizes and special values) and, with `--model`, a converter output file's weights (execute-only: its per-run files are deleted, digests only); `--timing` prints net1's milliseconds at 1080p- and 1440p-sized inputs as information, after a probe rules out long single launches |
| `tests/test_fg_net1_gpu_plan.py` | the chain as planned on the CPU: every step output's shape against the reference's buffers, launch arguments, natural weight order, chain lines |
| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" |
| `../../data/fg.v1.map.json` | the spec side's converter map, revision v1.1 (weights in natural order); `tests/test_fg_map.py` pins its digest and checks it against `layout.py` |
| `reference/natural.py` | the natural weight order from the map itself: its offsets as gather indices (stored blob weights to natural), natural to canonical `[out, k, k, in]` |
Expand Down
10 changes: 10 additions & 0 deletions clean/fg/kernels/fg_net1.hip
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,16 @@ WG_KERNEL_N(fg_maxpool2_f16, 256)(const u16* __restrict__ x, u16* __restrict__ y
y[idx] = (u16)m;
}

// fg_add_f16: net1's skip add, y = h(a + b) per element (float16; single precision then float16 equals one rounding of
// the exact sum). NaN results are written as 0x7E00. n elements.
WG_KERNEL_N(fg_add_f16, 256)(const u16* __restrict__ a, const u16* __restrict__ b, u16* __restrict__ y, int n) {
const int idx = (int)(wg_block_id() * 256u + wg_lane_id());
if (idx >= n) return;
uint out = fg_f2h(fg_h2f(a[idx]) + fg_h2f(b[idx]));
if (fg_is_nan16(out) != 0u) out = FG_NAN16;
y[idx] = (u16)out;
}

// One axis of the align-corners rule: s = r * o (single precision), i0 = trunc(s), i1 = trunc(s + 1), both n_in - 1 when
// i1 >= n_in; weights w1 = h(s - i0), w0 = h((i0 - s) + 1) as float16 values.
struct FgTap { int i0, i1; float w0, w1; };
Expand Down
31 changes: 29 additions & 2 deletions clean/fg/runner/fg_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@
Cases: fg_conv_f16: every net1 layer shape (cin, cout, kernel size, chunk order, activation) on small grids of several
sizes, and special values (NaN, +-inf, subnormals, +-65504, -0) in the inputs and weights; fg_maxpool2_f16: random maps
of several sizes (odd ones too) and every 2 x 2 window over ten special values; fg_upscale_f16: net1's level pairs,
other sizes, an output extent of 1, equal sizes, special values. Output: counts only, no timing.
other sizes, an output extent of 1, equal sizes, special values; fg_add_f16: random pairs of several lengths and every
pair of thirteen special values. Output: counts only, no timing. The whole network: fg_net1_gpu.py.
"""
import argparse
import ctypes
Expand Down Expand Up @@ -168,8 +169,34 @@ def _upscale_case(kernel, x, ho, wo, label):
"defined": 2 * n, "cpu": lambda: N.upscale_to(x.astype(np.float64), wo, ho)}


ADD_SPECIALS = np.array([np.nan, np.inf, -np.inf, 65504.0, -65504.0, 2.0 ** -24, -2.0 ** -24, -0.0, 0.0, 1.0, -1.0,
32768.0, 0.000999], dtype=np.float16)


def add_cases(kernel, rng):
"""Random pairs of several lengths, and every pair over ADD_SPECIALS (overflow, inf - inf, -0 + -0, subnormal
sums)."""
cases = []
for n in (1, 255, 256, 1000, 4096 * 3 + 7):
a = _f16(rng.uniform(-100, 100, n))
b = _f16(rng.uniform(-100, 100, n))
cases.append(_add_case(kernel, a, b, f"n {n}"))
v = ADD_SPECIALS
a = np.repeat(v, len(v)).astype(np.float16)
b = np.tile(v, len(v)).astype(np.float16)
cases.append(_add_case(kernel, a, b, "every special pair"))
return cases


def _add_case(kernel, a, b, label):
n = a.size
return {"kernel": kernel, "grid": -(-n // 256), "block": 256, "label": label,
"args": [("buf", a.tobytes()), ("buf", b.tobytes()), ("out", 2 * n + SLACK), ("u32", n)],
"defined": 2 * n, "cpu": lambda: N.add16(a.astype(np.float64), b.astype(np.float64))}


KINDS = {"fg_conv_f16": lambda kernel, rng: conv_cases(kernel, rng, specials=True),
"fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases}
"fg_maxpool2_f16": pool_cases, "fg_upscale_f16": upscale_cases, "fg_add_f16": add_cases}


# ---- prepare --------------------------------------------------------------------------------------------------------
Expand Down
Loading
Loading