From 6aef466f4223a55d79bddebd14118eef864616d3 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:42:08 -0500 Subject: [PATCH] Model M driver probe: distinct values per lane and element native_driver_probe multiplied an all-ones A by an all-twos B and added C = the lane number in every element, then checked 32 + lane. Because A and B were uniform and C and D share one layout, any element layout of A, B or the accumulator gave the same result, so a driver or translation layer that lays the WMMA fragments out differently still passed. The probe now reads A, B and C from its buffer. The host fills them in the fragment layout of clean/FORMATS.md (clean/runtime/wmma_probe.h): A and B are small nonzero integers whose 256 products A x B are all distinct, C = (32 v + lane) / 512 is distinct per element, every value is exact in FP32, and the expected FP16 and FP8 results are computed on the CPU in the same header. A different lane, element or half order of A, B or the accumulator, A and B swapped or transposed, or the other accumulator row table now changes at least one checked value. The cost at create stays one launch of one wave (4.5 KiB upload, 2 KiB readback). WG_TEST_DRIVER_PROBE_FAIL=1 still reports a mismatch at float 7. clean/runtime/tests/test_wmma_probe.cpp checks this on any host: a model of the instruction runs the probe's buffer under 17 named wrong layouts and every lane-pair and element-pair swap of A, B and C/D (1,572 more); the new probe fails each one and the old probe passes each one. clean/dx12/tests/native_probe_test.cpp runs the probe on the GPU through make_d3d12_device_on. Reported by @j0nsh1n in #10. Refs #10. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- BUILDING.md | 4 + clean/dx12/kernels/native_game_helpers.hlsl | 14 +- clean/dx12/tests/native_probe_test.cpp | 84 ++++++ clean/runtime/device_d3d12.cpp | 17 +- clean/runtime/tests/test_wmma_probe.cpp | 267 ++++++++++++++++++++ clean/runtime/wmma_probe.h | 133 ++++++++++ 6 files changed, 507 insertions(+), 12 deletions(-) create mode 100644 clean/dx12/tests/native_probe_test.cpp create mode 100644 clean/runtime/tests/test_wmma_probe.cpp create mode 100644 clean/runtime/wmma_probe.h diff --git a/BUILDING.md b/BUILDING.md index 117f1d40..fa1988b5 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -159,6 +159,10 @@ python -m unittest discover -s clean\rr_f\proxy\tests -t clean\rr_f\proxy\tests python -m unittest discover -s clean\rr_f\proxy\tests -t clean\rr_f\proxy\tests -p test_gen_forwarding.py ``` +`clean\runtime\tests\test_wmma_probe.cpp` checks Model M's driver-channel probe (`clean\runtime\wmma_probe.h`) on +any host: build it with any C++17 compiler (`clang++ -std=c++17 test_wmma_probe.cpp`) and run it; exit status 0 when +it passes. `clean\dx12\tests\native_probe_test.cpp` runs the same probe on the GPU (see 7). + `clean\wgshared\tests\*.py` are scripts that the wgshared ctest runs. The Ray Reconstruction plugin's CPU tests are executables that `clean\rr_f\proxy\tools\build.py --no-tests` builds; run them as that script does: `out\rr-proxy\core\test_proxy_core.exe`, `test_probe_core.exe`, `test_setupapi_identity.exe`, diff --git a/clean/dx12/kernels/native_game_helpers.hlsl b/clean/dx12/kernels/native_game_helpers.hlsl index 431852c7..134ad3a1 100644 --- a/clean/dx12/kernels/native_game_helpers.hlsl +++ b/clean/dx12/kernels/native_game_helpers.hlsl @@ -13,13 +13,19 @@ WG_ARGS_BEGIN uint dst; WG_ARGS_END +// D = A x B + C with FP16 and with FP8 operands. The host (clean/runtime/wmma_probe.h) writes A, B and C in fragment +// layout with distinct values per lane and element and checks every result, so a different fragment layout fails. +// Buffer byte offsets: wmma_probe.h kOutF16 0, kOutFp8 1024, kInC 2048, kInAF16 3072, kInBF16 3584, kInAFp8 4096, +// kInBFp8 4352. All loads come before the first matrix call (wg_ags.hlsli rules). WG_KERNEL(native_driver_probe) { WG_SETUP(); uint lane=wg_lane_id(); - v8f c=wg_splat((float)lane); - v8f h=wg_wmma_f16(uint4(0x3c003c00u,0x3c003c00u,0x3c003c00u,0x3c003c00u), - uint4(0x40004000u,0x40004000u,0x40004000u,0x40004000u),c); - v8f q=wg_wmma_fp8(uint2(0x38383838u,0x38383838u),uint2(0x40404040u,0x40404040u),c); + uint4 a16=wg_ld128(dst,3072u+lane*16u),b16=wg_ld128(dst,3584u+lane*16u); + uint2 a8=wg_ld64(dst,4096u+lane*8u),b8=wg_ld64(dst,4352u+lane*8u); + v8f c; + [unroll] for(uint i=0;i<8u;++i) c.v[i]=wg_ldf(dst,2048u+(i*32u+lane)*4u); + v8f h=wg_wmma_f16(a16,b16,c); + v8f q=wg_wmma_fp8(a8,b8,c); [unroll] for(uint k=0;k<8u;++k) { wg_stf(dst,(k*32u+lane)*4u,h.v[k]); wg_stf(dst,1024u+(k*32u+lane)*4u,q.v[k]); diff --git a/clean/dx12/tests/native_probe_test.cpp b/clean/dx12/tests/native_probe_test.cpp new file mode 100644 index 00000000..58249468 --- /dev/null +++ b/clean/dx12/tests/native_probe_test.cpp @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: MIT +// The driver-channel probe (runtime/wmma_probe.h) on the GPU, through the runtime's own borrowed-device path: creates a +// D3D12 device and DIRECT queue on the first supported RX 9070 / 9060 adapter and hands them to +// wgrt::make_d3d12_device_on, which runs native_driver_probe at create. WrongGPU's own code. No HIP runtime, no model. +// native_probe_test.exe +// Build (the HIP SDK's clang++ as a host compiler only, as native_direct_build.py does): +// clang++ -std=c++17 -O2 -D_CRT_SECURE_NO_WARNINGS -DWG_NO_MODEL_K -fms-runtime-lib=static native_probe_test.cpp +// ../../runtime/{device,device_dx12,device_d3d12,embedded,model,no_model_k,runtime}.cpp +// ../../runtime/interop/{convert,native_candidate}.cpp ../../converter/src/{json,sha256}.cpp -o native_probe_test.exe +// With WG_TEST_DRIVER_PROBE_FAIL=1 the create must fail with "probe mismatch at float 7". Exit status 0 on PASS. +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#include +#include + +#include +#include + +#include "../../runtime/device_d3d12.h" +#include "../../runtime/rdna4.h" + +int main(int argc, char** argv) { + if (argc != 2) { + std::fprintf(stderr, "usage: native_probe_test.exe \n"); + return 2; + } + HMODULE dxgi = LoadLibraryExW(L"dxgi.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); + HMODULE d3d12 = LoadLibraryExW(L"d3d12.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32); + if (!dxgi || !d3d12) { std::printf("FAIL system DXGI / D3D12 unavailable\n"); return 1; } + using Factory = HRESULT(WINAPI*)(REFIID, void**); + using Create = HRESULT(WINAPI*)(IUnknown*, D3D_FEATURE_LEVEL, REFIID, void**); + auto factory_fn = reinterpret_cast(GetProcAddress(dxgi, "CreateDXGIFactory1")); + auto create = reinterpret_cast(GetProcAddress(d3d12, "D3D12CreateDevice")); + IDXGIFactory4* factory = nullptr; + if (!factory_fn || !create || FAILED(factory_fn(__uuidof(IDXGIFactory4), reinterpret_cast(&factory)))) { + std::printf("FAIL DXGI factory\n"); + return 1; + } + IDXGIAdapter1* adapter = nullptr; + std::string name; + for (UINT i = 0;; ++i) { + IDXGIAdapter1* a = nullptr; + if (factory->EnumAdapters1(i, &a) == DXGI_ERROR_NOT_FOUND) break; + DXGI_ADAPTER_DESC1 desc{}; + char n[256] = {}; + if (SUCCEEDED(a->GetDesc1(&desc))) WideCharToMultiByte(CP_UTF8, 0, desc.Description, -1, n, sizeof n - 1, nullptr, nullptr); + if (wgrt::rdna4_adapter(desc.VendorId, (desc.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) != 0, n)) { + adapter = a; + name = n; + break; + } + a->Release(); + } + factory->Release(); + if (!adapter) { std::printf("SKIP no Radeon RX 9070 / 9060 adapter\n"); return 1; } + ID3D12Device* device = nullptr; + ID3D12CommandQueue* queue = nullptr; + HRESULT h = create(adapter, D3D_FEATURE_LEVEL_12_0, __uuidof(ID3D12Device), reinterpret_cast(&device)); + adapter->Release(); + if (FAILED(h)) { std::printf("FAIL D3D12CreateDevice on %s\n", name.c_str()); return 1; } + D3D12_COMMAND_QUEUE_DESC qd{}; + qd.Type = D3D12_COMMAND_LIST_TYPE_DIRECT; + if (FAILED(device->CreateCommandQueue(&qd, __uuidof(ID3D12CommandQueue), reinterpret_cast(&queue)))) { + std::printf("FAIL CreateCommandQueue\n"); + device->Release(); + return 1; + } + std::printf("adapter %s\n", name.c_str()); + std::string err; + int status = 1; + { + auto dev = wgrt::make_d3d12_device_on(device, queue, argv[1], &err, 64ull << 20); + if (dev) { + std::printf("PASS borrowed device created, driver-channel probe passed: %s\n", dev->name().c_str()); + status = 0; + } else { + std::printf("FAIL %s\n", err.c_str()); + } + } + queue->Release(); + device->Release(); + return status; +} diff --git a/clean/runtime/device_d3d12.cpp b/clean/runtime/device_d3d12.cpp index f710d369..0a74cccc 100644 --- a/clean/runtime/device_d3d12.cpp +++ b/clean/runtime/device_d3d12.cpp @@ -18,6 +18,7 @@ #include "native_direct_pi.h" #include "embedded.h" #include "rdna4.h" +#include "wmma_probe.h" #include #include @@ -1337,20 +1338,20 @@ class NativeD3D12Device final : public Device, public NativeD3D12Ext { probe->second.block != 32 || probe->second.wave != 32 || probe->second.dwords != 1 || probe->second.args.size() != 1 || probe->second.args[0].kind != 'p') return fail("native D3D12 borrowed device requires native_driver_probe (co=-, buffer, block/wave=32, args=p)"); + // Operands with distinct values per lane and element, expected results computed on the CPU (wmma_probe.h). DevPtr output = 0; - if (!alloc(&output, 512 * sizeof(float))) return false; - float values[512] = {}; + if (!alloc(&output, wmma_probe::kBytes)) return false; + uint8_t buffer[wmma_probe::kBytes]; + wmma_probe::fill(buffer); + float values[wmma_probe::kResults] = {}; void* args[] = {&output}; - bool ok = upload(output, values, sizeof values) && launch("native_driver_probe", 1, 32, args) && + bool ok = upload(output, buffer, sizeof buffer) && launch("native_driver_probe", 1, 32, args) && download(values, output, sizeof values); // tests of the fallback (interop test, WG_TEST_DRIVER_PROBE_FAIL=1): the probe reports a mismatch const char* fail_test = std::getenv("WG_TEST_DRIVER_PROBE_FAIL"); if (ok && fail_test && fail_test[0] == '1') values[7] = -1.0f; - if (ok) for (unsigned i = 0; i < 512; ++i) { - if (values[i] != static_cast(32 + (i % 32))) { - ok = fail("native D3D12 borrowed driver-channel probe mismatch at float " + std::to_string(i)); break; - } - } + const int bad = ok ? wmma_probe::first_mismatch(values) : -1; + if (bad >= 0) ok = fail("native D3D12 borrowed driver-channel probe mismatch at float " + std::to_string(bad)); const std::string error = err_; free(output); if (!ok) err_ = error; diff --git a/clean/runtime/tests/test_wmma_probe.cpp b/clean/runtime/tests/test_wmma_probe.cpp new file mode 100644 index 00000000..ce2ad919 --- /dev/null +++ b/clean/runtime/tests/test_wmma_probe.cpp @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: MIT +// The driver-channel probe (runtime/wmma_probe.h) on any host (no GPU, no D3D12). WrongGPU's own code. +// clang++ -std=c++17 -O2 test_wmma_probe.cpp -o test_wmma_probe.exe && test_wmma_probe.exe +// (or: cl /nologo /std:c++17 /O2 /EHsc test_wmma_probe.cpp) +// A model of the matrix instruction reads the probe's buffer with a fragment layout, computes D = A x B + C and writes +// D back in that layout. With the layout of wmma_probe.h the probe must pass; with every wrong layout below (lanes, +// elements or halves swapped, A / B / the accumulator transposed, A and B swapped, the other accumulator row table) it +// must fail. The probe used up to 0.6.0 (all-ones A, all-twos B, C = the lane) is run through the same model to show +// that it passes all of those wrong layouts. Exit status 0 when every case passed. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../wmma_probe.h" + +namespace { + +namespace P = wgrt::wmma_probe; + +int failures = 0; + +void expect(const std::string& name, bool ok) { + std::printf("%s %s\n", ok ? "pass" : "FAIL", name.c_str()); + if (!ok) ++failures; +} + +float f16_value(uint16_t h) { + const int s = h >> 15, e = (h >> 10) & 31, m = h & 1023; + const float v = e == 0 ? std::ldexp(static_cast(m), -24) : std::ldexp(static_cast(1024 + m), e - 25); + return s ? -v : v; +} + +float fp8_value(uint8_t b) { // E4M3, bias 7 (no infinities; 0x7f / 0xff are NaN, never used here) + const int s = b >> 7, e = (b >> 3) & 15, m = b & 7; + const float v = e == 0 ? std::ldexp(static_cast(m), -9) : std::ldexp(static_cast(8 + m), e - 10); + return s ? -v : v; +} + +// Where the matrix instruction takes element j (A, B) or v (C, D) of lane l from: (row, column) of the matrix. +using Map = std::function; +struct Layout { + std::string name; + Map a = P::a_entry, b = P::b_entry, d = P::d_entry; + bool swap_ab = false; // the instruction reads the A operand from the B registers and B from A +}; + +// The model of the instruction: one pass for the FP16 operands, one for the FP8 ones. +void run(const uint8_t* buffer, const Layout& hw, float* results) { + for (unsigned p = 0; p < 2; ++p) { + double a[16][16] = {}, b[16][16] = {}, c[16][16] = {}; + uint32_t a_at = p ? P::kInAFp8 : P::kInAF16, b_at = p ? P::kInBFp8 : P::kInBF16; + if (hw.swap_ab) std::swap(a_at, b_at); + for (unsigned l = 0; l < 32; ++l) + for (unsigned j = 0; j < 8; ++j) { + auto element = [&](uint32_t at) { + if (p) return fp8_value(buffer[at + 8 * l + j]); + uint16_t h; + std::memcpy(&h, buffer + at + 16 * l + 2 * j, 2); + return f16_value(h); + }; + unsigned m, k, n; + hw.a(l, j, m, k); + a[m][k] = element(a_at); + hw.b(l, j, k, n); + b[k][n] = element(b_at); + hw.d(l, j, m, n); + float cv; + std::memcpy(&cv, buffer + P::kInC + 4 * (32 * j + l), 4); + c[m][n] = cv; + } + for (unsigned l = 0; l < 32; ++l) + for (unsigned v = 0; v < 8; ++v) { + unsigned m, n; + hw.d(l, v, m, n); + double sum = c[m][n]; + for (unsigned k = 0; k < 16; ++k) sum += a[m][k] * b[k][n]; + results[256 * p + 32 * v + l] = static_cast(sum); + } + } +} + +// The probe up to 0.6.0: A all 1.0, B all 2.0 (the same bits as its shader constants 0x3c00 / 0x40 ...), C = the lane +// in every element; expected 32 + lane for every float. +void old_fill(uint8_t* buffer) { + std::memset(buffer, 0, P::kBytes); + for (unsigned l = 0; l < 32; ++l) + for (unsigned j = 0; j < 8; ++j) { + const uint16_t one = 0x3c00, two = 0x4000; + std::memcpy(buffer + P::kInAF16 + 16 * l + 2 * j, &one, 2); + std::memcpy(buffer + P::kInBF16 + 16 * l + 2 * j, &two, 2); + buffer[P::kInAFp8 + 8 * l + j] = 0x38; + buffer[P::kInBFp8 + 8 * l + j] = 0x40; + const float c = static_cast(l); + std::memcpy(buffer + P::kInC + 4 * (32 * j + l), &c, 4); + } +} +bool old_pass(const float* r) { + for (unsigned i = 0; i < P::kResults; ++i) + if (r[i] != static_cast(32 + i % 32)) return false; + return true; +} + +// Results of the new and the old probe under one layout: new_bad = first mismatch (-1 = passes), old_ok. +struct Outcome { int new_bad; bool old_ok; }; +Outcome probe(const Layout& hw) { + static uint8_t fresh[P::kBytes], old[P::kBytes]; + static bool filled = false; + if (!filled) { P::fill(fresh); old_fill(old); filled = true; } + float r[P::kResults]; + run(fresh, hw, r); + const int bad = P::first_mismatch(r); + run(old, hw, r); + return {bad, old_pass(r)}; +} + +// A layout with operand X's slots permuted: the instruction takes element j of lane l from where the model has +// element s(l, j). which: 'a', 'b' or 'd'. +Layout permuted(const std::string& name, char which, std::function(unsigned, unsigned)> s) { + Layout x; + x.name = name; + Map base = which == 'a' ? x.a : which == 'b' ? x.b : x.d; + Map map = [base, s](unsigned l, unsigned j, unsigned& r, unsigned& c) { + const auto lj = s(l, j); + base(lj.first, lj.second, r, c); + }; + (which == 'a' ? x.a : which == 'b' ? x.b : x.d) = map; + return x; +} +Layout transposed(const std::string& name, char which) { + Layout x; + x.name = name; + Map base = which == 'a' ? x.a : which == 'b' ? x.b : x.d; + (which == 'a' ? x.a : which == 'b' ? x.b : x.d) = [base](unsigned l, unsigned j, unsigned& r, unsigned& c) { + base(l, j, c, r); + }; + return x; +} + +} // namespace + +int main() { + // value encodings: exact for every table value; the old probe's constants + bool enc = P::f16_bits(1) == 0x3c00 && P::f16_bits(2) == 0x4000 && P::fp8_bits(1) == 0x38 && P::fp8_bits(2) == 0x40; + for (int x = -16; x <= 16; ++x) + enc = enc && f16_value(P::f16_bits(x)) == static_cast(x) && fp8_value(P::fp8_bits(x)) == static_cast(x); + expect("FP16 / FP8 E4M3 bits of the integers -16 .. 16 are exact", enc); + + // the table properties the probe relies on + std::set products; + std::set expected; + long largest = 0; + bool nonzero = true; + for (unsigned m = 0; m < 16; ++m) + for (unsigned n = 0; n < 16; ++n) { + long s = 0; + for (unsigned k = 0; k < 16; ++k) s += P::kA[m][k] * P::kB[k][n]; + products.insert(s); + largest = std::max(largest, std::labs(s)); + nonzero = nonzero && P::kA[m][n] != 0 && P::kB[m][n] != 0; + } + for (unsigned i = 0; i < 256; ++i) expected.insert(P::expected(i)); + expect("all 256 products A x B are distinct (" + std::to_string(products.size()) + ")", products.size() == 256); + expect("all 256 expected results of one pass are distinct (" + std::to_string(expected.size()) + ")", + expected.size() == 256); + expect("no zero operand", nonzero); + // |A x B| + C < 2^11 with C a multiple of 2^-9: every value and partial sum fits 20 bits, exact in FP32 + expect("results exact in FP32 (largest |A x B| = " + std::to_string(largest) + " < 2048)", largest < 2048); + for (unsigned i = 0; i < 256; ++i) + if (P::expected(i) != P::expected(256 + i)) { expect("FP16 and FP8 passes expect the same values", false); break; } + + // the right layout passes, both probes + Layout model; + model.name = "model"; + Outcome right = probe(model); + expect("layout of wmma_probe.h: new probe passes", right.new_bad < 0); + expect("layout of wmma_probe.h: old probe passes", right.old_ok); + + // WG_TEST_DRIVER_PROBE_FAIL=1 (device_d3d12.cpp) still reports float 7 + { + uint8_t buffer[P::kBytes]; + P::fill(buffer); + float r[P::kResults]; + run(buffer, model, r); + r[7] = -1.0f; + expect("a corrupted float 7 is reported at float 7", P::first_mismatch(r) == 7); + } + + // named wrong layouts: the new probe fails each one, the old probe passes each one + std::vector wrong; + using LJ = std::pair; + wrong.push_back(permuted("A lane halves swapped (K 0-7 <-> 8-15 in A only)", 'a', [](unsigned l, unsigned j) { return LJ(l ^ 16, j); })); + wrong.push_back(permuted("B lane halves swapped", 'b', [](unsigned l, unsigned j) { return LJ(l ^ 16, j); })); + wrong.push_back(permuted("C/D lane halves swapped (rows 0-7 <-> 8-15)", 'd', [](unsigned l, unsigned j) { return LJ(l ^ 16, j); })); + wrong.push_back(permuted("A neighbouring lanes swapped (rows 2i <-> 2i+1)", 'a', [](unsigned l, unsigned j) { return LJ(l ^ 1, j); })); + wrong.push_back(permuted("B neighbouring lanes swapped (columns)", 'b', [](unsigned l, unsigned j) { return LJ(l ^ 1, j); })); + wrong.push_back(permuted("C/D neighbouring lanes swapped (columns)", 'd', [](unsigned l, unsigned j) { return LJ(l ^ 1, j); })); + wrong.push_back(permuted("A elements reversed in each lane", 'a', [](unsigned l, unsigned j) { return LJ(l, 7 - j); })); + wrong.push_back(permuted("B elements reversed in each lane", 'b', [](unsigned l, unsigned j) { return LJ(l, 7 - j); })); + wrong.push_back(permuted("C/D elements reversed in each lane", 'd', [](unsigned l, unsigned j) { return LJ(l, 7 - j); })); + wrong.push_back(permuted("A FP16 halves of each dword swapped", 'a', [](unsigned l, unsigned j) { return LJ(l, j ^ 1); })); + wrong.push_back(permuted("A bytes of each dword reversed", 'a', [](unsigned l, unsigned j) { return LJ(l, j ^ 3); })); + wrong.push_back(permuted("B bytes of each dword reversed", 'b', [](unsigned l, unsigned j) { return LJ(l, j ^ 3); })); + wrong.push_back(transposed("A transposed", 'a')); + wrong.push_back(transposed("B transposed", 'b')); + wrong.push_back(transposed("C/D transposed", 'd')); + { + Layout x; + x.name = "A and B swapped"; + x.swap_ab = true; + wrong.push_back(x); + Layout y; + y.name = "accumulator rows 2v + h (the other D_ROW table, clean/runner/packing.py gfx11)"; + y.d = [](unsigned l, unsigned v, unsigned& m, unsigned& n) { m = 2 * v + l / 16; n = l % 16; }; + wrong.push_back(y); + } + for (const Layout& x : wrong) { + const Outcome o = probe(x); + expect(x.name + ": new probe fails" + (o.new_bad >= 0 ? " (float " + std::to_string(o.new_bad) + ")" : ""), + o.new_bad >= 0); + expect(x.name + ": old probe passes it", o.old_ok); + } + + // exhaustive: every pair of lanes and every pair of element slots swapped, in A, in B and in C/D + for (char which : {'a', 'b', 'd'}) { + int lanes_caught = 0, lanes_old = 0, lanes_total = 0, elems_caught = 0, elems_old = 0, elems_total = 0; + for (unsigned l1 = 0; l1 < 32; ++l1) + for (unsigned l2 = l1 + 1; l2 < 32; ++l2) { + const Outcome o = probe(permuted("", which, [l1, l2](unsigned l, unsigned j) { + return LJ(l == l1 ? l2 : l == l2 ? l1 : l, j); + })); + ++lanes_total; lanes_caught += o.new_bad >= 0; lanes_old += o.old_ok; + } + for (unsigned j1 = 0; j1 < 8; ++j1) + for (unsigned j2 = j1 + 1; j2 < 8; ++j2) { + const Outcome o = probe(permuted("", which, [j1, j2](unsigned l, unsigned j) { + return LJ(l, j == j1 ? j2 : j == j2 ? j1 : j); + })); + ++elems_total; elems_caught += o.new_bad >= 0; elems_old += o.old_ok; + } + const std::string op = which == 'a' ? "A" : which == 'b' ? "B" : "C/D"; + expect(op + ": new probe fails all " + std::to_string(lanes_total) + " lane-pair swaps (" + + std::to_string(lanes_caught) + "); old probe passes " + std::to_string(lanes_old), + lanes_caught == lanes_total); + expect(op + ": new probe fails all " + std::to_string(elems_total) + " element-pair swaps (" + + std::to_string(elems_caught) + "); old probe passes " + std::to_string(elems_old), + elems_caught == elems_total); + } + + // A K order that A and B share does not change any product: the probe cannot see it and need not + { + Layout x; + x.name = "the same K halves swap in A and B"; + x.a = [](unsigned l, unsigned j, unsigned& m, unsigned& k) { P::a_entry(l ^ 16, j, m, k); m = l % 16; }; + x.b = [](unsigned l, unsigned j, unsigned& k, unsigned& n) { P::b_entry(l ^ 16, j, k, n); n = l % 16; }; + expect(x.name + " (no product changes): new probe passes", probe(x).new_bad < 0); + } + + std::printf("%s\n", failures ? "TESTS FAILED" : "ALL TESTS PASSED"); + return failures ? 1 : 0; +} diff --git a/clean/runtime/wmma_probe.h b/clean/runtime/wmma_probe.h new file mode 100644 index 00000000..8280a04c --- /dev/null +++ b/clean/runtime/wmma_probe.h @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: MIT +// The native D3D12 device's driver-channel probe (native_driver_probe, clean/dx12/kernels/native_game_helpers.hlsl): +// its operands, its buffer layout and its expected result, computed here on the CPU. WrongGPU's own code. Header only, +// no device: device_d3d12.cpp runs the probe on a borrowed device at create, and tests/test_wmma_probe.cpp checks on +// any host that a matrix-fragment layout other than the one below fails it. +// +// The probe runs one wave of 32 lanes: D = A x B + C, once as FP16 x FP16 -> FP32 and once as FP8 (E4M3) x FP8 -> FP32 +// (16 x 16 x 16, the driver channel's v_wmma_f32_16x16x16_f16 / _fp8_fp8). Fragment layout (clean/FORMATS.md, "WMMA +// fragment" and "Accumulator layout", gfx12 wave32), lane l = 16 h + r: +// A: element j of lane l holds A[r][8 h + j] (FP16: half j of the 4 dwords, low half first; FP8: byte j) +// B: element j of lane l holds B[8 h + j][r] +// C, D: element v of lane l holds [v + 8 h][r] (D_ROW "gfx12") +// A and B are small nonzero integers chosen (by a random search) so that the 256 entries of A x B are all distinct, +// and C is (32 v + l) / 512, distinct per element. Every value, product and partial sum is exact in FP32, so the result +// does not depend on the order of the hardware's additions, and every expected result is a different number. A +// different row, column, lane, element or half order of A, B or the accumulator, or A and B swapped or transposed, +// changes at least one result (tests/test_wmma_probe.cpp checks these properties and each such layout). Only a K order +// that A and B share is invisible, and it changes no entry of A x B either. +// Cost at create: one 4.5 KiB upload, one launch of one wave, one 2 KiB readback. +#pragma once + +#include +#include + +namespace wgrt { +namespace wmma_probe { + +constexpr unsigned kLanes = 32, kElements = 8; +// Byte offsets in the probe's buffer (the kernel uses the same constants). The two results come first: float +// i = 256 p + 32 v + l is element v of lane l of result p (0 = FP16 operands, 1 = FP8 operands). +constexpr uint32_t kOutF16 = 0, kOutFp8 = 1024, kInC = 2048, kInAF16 = 3072, kInBF16 = 3584, kInAFp8 = 4096, + kInBFp8 = 4352, kBytes = 4608; +constexpr unsigned kResults = 512; // floats read back from offset 0 + +// A[m][k] and B[k][n]: nonzero integers in [-16, 16] (exact in FP8 E4M3 and in FP16). +constexpr int8_t kA[16][16] = { + {9, 15, -8, -10, 10, 11, 1, -12, -7, 1, -12, -12, -3, 7, -4, 11}, + {-16, -3, 1, 15, -9, 11, -14, -6, 8, 3, 15, 9, 14, 11, -12, 14}, + {12, -12, 16, 8, -6, -12, -4, 14, 10, -9, -1, 12, 5, -7, -13, 16}, + {-9, 1, 7, -6, 1, -7, 11, 4, -11, -6, 7, 6, -3, -12, -12, -9}, + {10, 12, -6, 4, -1, -8, 1, -12, -1, -12, -9, -9, -16, -4, 8, 5}, + {9, 11, -15, 9, -13, 15, -14, -12, -1, -2, -5, -2, -3, 3, 4, -5}, + {9, 5, -1, -13, 11, -9, -6, 9, 12, 16, 11, 10, 8, 11, 2, 10}, + {-9, -2, 10, 8, 14, 3, -5, -2, -15, -8, 8, 12, 10, -14, -3, 11}, + {5, 13, -2, -3, -9, 11, -12, -6, -14, 1, 4, 2, 2, -10, -4, -6}, + {6, -14, 5, -2, 15, 6, -11, 6, -6, 1, -10, -2, -15, 1, 1, -6}, + {1, -11, 14, -7, 10, 5, 7, -14, 8, 5, -14, -8, 13, -11, 11, -1}, + {-6, -5, -14, -13, 11, 7, -13, 10, 10, -15, 13, 5, 4, -9, 9, -3}, + {10, 10, -4, 2, -1, -16, 5, -16, -2, -4, 5, 13, 13, -9, -7, -2}, + {-3, -12, -12, 10, -10, -4, 12, -5, -6, -8, 15, 11, 10, -3, -3, -15}, + {-6, 10, 12, -8, -13, -11, 6, 6, -3, 2, 13, 15, -12, -7, 2, -2}, + {-15, 4, 6, 16, 15, 15, -8, -4, -6, -5, -4, -3, 11, -13, 12, 11}, +}; +constexpr int8_t kB[16][16] = { + {3, 13, 15, -4, 7, 15, 2, 8, 11, -2, 16, 12, -2, 16, -5, 15}, + {3, -3, -9, -11, -9, 10, 8, 15, 1, 1, 13, -8, 8, 14, 6, 12}, + {1, -1, 12, 12, 11, -4, -13, -16, 10, -13, -7, -1, -6, -8, 11, 3}, + {12, -10, 13, 9, -11, -11, -5, -10, -3, 14, 10, -13, -6, -5, 7, -13}, + {-15, 6, -11, -16, 16, 12, -3, -12, -6, 12, -5, 3, -11, -4, -12, -1}, + {9, -4, -7, -7, -9, 15, -9, -3, 9, -1, 4, 12, 1, -14, 9, -13}, + {5, 16, 14, 6, 8, -5, 10, 16, -12, 1, -12, -2, -6, -9, -12, 2}, + {-7, -6, -15, 6, -2, -8, 3, 6, -13, -3, -6, -15, -12, -13, -10, -12}, + {13, -5, 3, 15, -3, 3, -12, -12, -9, 2, 14, -5, 9, -13, -3, -4}, + {-8, -10, -4, 10, -9, 7, 10, -7, 9, 10, 7, -3, -3, 16, -16, 10}, + {15, -4, -7, -1, 16, -1, -7, 3, -7, -12, 1, -10, -9, -6, -4, 1}, + {-4, -11, 14, 13, -1, -11, -4, 11, 2, -11, -15, 2, -13, -12, 16, 2}, + {-15, -13, 14, 5, 2, -12, -5, 7, 10, -12, -10, 15, -5, 1, -7, -8}, + {-9, -16, 5, -6, -12, 15, 4, -4, -9, 13, 9, -14, -3, -8, -15, 14}, + {-7, -5, -2, -13, -1, 8, 7, 8, -5, -9, -4, 4, -16, 15, -3, -5}, + {-7, -12, -8, 13, -5, -1, 6, 3, 4, 10, -10, -14, -6, -9, 5, 7}, +}; + +// The layout above: which matrix entry element j / v of lane l holds. +inline void a_entry(unsigned lane, unsigned j, unsigned& m, unsigned& k) { m = lane % 16; k = 8 * (lane / 16) + j; } +inline void b_entry(unsigned lane, unsigned j, unsigned& k, unsigned& n) { n = lane % 16; k = 8 * (lane / 16) + j; } +inline void d_entry(unsigned lane, unsigned v, unsigned& m, unsigned& n) { m = v + 8 * (lane / 16); n = lane % 16; } + +inline float c_value(unsigned lane, unsigned v) { return static_cast(32 * v + lane) / 512.0f; } + +// Bits of an integer x with |x| <= 16 (exact in both formats). +inline uint16_t f16_bits(int x) { + if (x == 0) return 0; + const unsigned s = x < 0 ? 1u : 0u, a = static_cast(x < 0 ? -x : x); + unsigned e = 0; + while ((a >> (e + 1)) != 0) ++e; + return static_cast(s << 15 | (e + 15) << 10 | ((a << (10 - e)) & 0x3ffu)); +} +inline uint8_t fp8_bits(int x) { // E4M3, bias 7 + if (x == 0) return 0; + const unsigned s = x < 0 ? 1u : 0u, a = static_cast(x < 0 ? -x : x); + unsigned e = 0; + while ((a >> (e + 1)) != 0) ++e; + return static_cast(s << 7 | (e + 7) << 3 | ((a << 3 >> e) & 0x7u)); +} + +// The probe's buffer before the launch (kBytes): A, B and C in fragment layout, the results zero. +inline void fill(uint8_t* buffer) { + std::memset(buffer, 0, kBytes); + for (unsigned l = 0; l < kLanes; ++l) + for (unsigned j = 0; j < kElements; ++j) { + unsigned m, n, k; + a_entry(l, j, m, k); + const uint16_t a16 = f16_bits(kA[m][k]); + std::memcpy(buffer + kInAF16 + 16 * l + 2 * j, &a16, 2); + buffer[kInAFp8 + 8 * l + j] = fp8_bits(kA[m][k]); + b_entry(l, j, k, n); + const uint16_t b16 = f16_bits(kB[k][n]); + std::memcpy(buffer + kInBF16 + 16 * l + 2 * j, &b16, 2); + buffer[kInBFp8 + 8 * l + j] = fp8_bits(kB[k][n]); + const float c = c_value(l, j); + std::memcpy(buffer + kInC + 4 * (32 * j + l), &c, 4); + } +} + +// Expected value of result float i (both results are the same exact numbers). +inline float expected(unsigned i) { + const unsigned lane = i % 32, v = (i % 256) / 32; + unsigned m, n; + d_entry(lane, v, m, n); + int sum = 0; + for (unsigned k = 0; k < 16; ++k) sum += kA[m][k] * kB[k][n]; + return static_cast(sum) + c_value(lane, v); +} + +// First result float that differs from the expected one, or -1 when all kResults match. +inline int first_mismatch(const float* results) { + for (unsigned i = 0; i < kResults; ++i) + if (results[i] != expected(i)) return static_cast(i); + return -1; +} + +} // namespace wmma_probe +} // namespace wgrt