diff --git a/clean/dx12/kernels/post.hlsl b/clean/dx12/kernels/post.hlsl index 21f0be3f..8d6541e8 100644 --- a/clean/dx12/kernels/post.hlsl +++ b/clean/dx12/kernels/post.hlsl @@ -50,7 +50,24 @@ // @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448 // @wg-kernel pmo/post_tf4_dl_f32_p12_mo co=post_mo_w64__orig.co entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448 // @wg-kernel pmo/post_tf4_dl_f32_p12_mo_direct co=post_mo_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448 +// +// Analytic output filter (WG_LZ_ANALYTIC = 1; HIP: post_w64.co, post.hip built with -mwavefrontsize64 and the default +// WG_LZ_PROFILE = 0, the build the runtime loads under "w64" when the model file has no output-filter weights, e.g. +// Model L's): the same kernels with spec 06's own Lanczos weights as FP32 constants (kLz0 ... kLz4): v = kLz0 t0, then +// one fused multiply-add per tap, clamp to the taps' range, every value FP32 (the horizontal results go to the ring as +// FP32, the vertical result straight into Dec). No weight arguments. Forms 0 and 3 (direct; without the FP16-indexed +// output table, which needs the original's FP16 filter result). +// @wg-kernel w64/post_tf_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_LZ_ANALYTIC=1 +// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1 +// @wg-kernel w64/post_tf_f32_p12_diet_vsit_direct co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit_direct__analytic rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_LZ_ANALYTIC=1 +// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit_direct__analytic rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1 #include "wg_platform.hlsli" +#ifndef WG_LZ_ANALYTIC +#define WG_LZ_ANALYTIC 0 +#endif +#if WG_LZ_ANALYTIC && ((POST_OUT != 0 && POST_OUT != 3) || defined(WG_POST_MO)) +#error "analytic output filter: forms 0 and 3 (direct) only" +#endif #if defined(WG_DIRECT_TEXTURES) #include "native_direct.hlsli" #include "wg_f16lut.hlsli" @@ -85,7 +102,9 @@ WG_ARGS_BEGIN int mo_mw, mo_mh, mo_pitch, mo_wide; // WgMoParams (40 bytes) float mo_sx, mo_sy, mo_bx, mo_by, mo_idw, mo_idh; #endif +#if !WG_LZ_ANALYTIC uint lz_w0, lz_w1, lz_w2, lz_w3, lz_w4; // FP16 bit patterns of the filter weights of taps -2 ... +2 +#endif WG_ARGS_END // ---- constants (FP32 bit patterns of post.hip's constexpr values) --------------------------------------------------- @@ -697,6 +716,26 @@ void mo_core(Gather g, float2 mv, int x, int y, int fx, int fy, float s, out flo } #endif +#if WG_LZ_ANALYTIC +// ---- output filter, profile 0 ("analytic", post_w64.co): FP32 constant weights of taps -2 ... +2 for an even output +// index (post.hip kLz0 ... kLz4; mirrored for an odd one), v = w0 t0, one fma per further tap, clamp to the taps' range +static const float kLz0 = asfloat(0xbdba3b4bu), kLz1 = asfloat(0x3de71ca9u), kLz2 = asfloat(0x3f23b70fu); +static const float kLz3 = asfloat(0x3ed182b5u), kLz4 = asfloat(0xbd90a4acu); +float lz5a(float t0, float t1, float t2, float t3, float t4, float w0, float w1, float w3, float w4) +{ + float v = w0 * t0; + v = mad(w1, t1, v); + v = mad(kLz2, t2, v); + v = mad(w3, t3, v); + v = mad(w4, t4, v); + const float mid_lo = min(min(t1, t2), t3), mid_hi = max(max(t1, t2), t3); + const float lo = min(min(mid_lo, t0), t4), hi = max(max(mid_hi, t0), t4); + return min(max(v, lo), hi); // med3 (the taps are finite, lo <= hi) +} +// lz5 / lz5 on the FP32 taps t[o ... o + 4], as FP32 bits +uint lz5a_even(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz0, kLz1, kLz3, kLz4)); } +uint lz5a_odd(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz4, kLz3, kLz1, kLz0)); } +#else // ---- output filter, profile 2 ("original"): FP16 weights, one FP16 fma per tap, clamp to the taps' range ------------- // Values are "boxes": FP16 bits in the low half of a dword. half hbox(uint v) { return asfloat16((uint16_t)(v & 0xffffu)); } @@ -726,6 +765,7 @@ uint lz5_odd(uint t[10], int o) return lz5_original(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], hbox(lz_w4), hbox(lz_w3), hbox(lz_w2), hbox(lz_w1), hbox(lz_w0)); } +#endif // The reduced history's 10-bit code of one channel from its 2 x 2 sum (the chain of post.hip's reduced store) uint rh_code(float sum) @@ -774,6 +814,19 @@ void pd_vitem(int y, int xg, int xo, int urel, uint wa, uint wb, int p_next, boo t[k][c] = lds[kRingBase + (uint)sl * kRowDw + (uint)(c * kTfOutX + xo)]; } } +#if WG_LZ_ANALYTIC + // the analytic filter: wa = 1 for the even phase (pd_vsplit); FP32 results, so no FP16-indexed output table + const bool ev = wa != 0u; + const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1; + float vo[3], qv[3]; + bool vsmall = false; + [unroll] for (int c = 0; c < 3; ++c) { + qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]), + w0, w1, w3, w4), k1em4, 3.0f); + vo[c] = exp2(qv[c] * kDecK) - 1.0f; + vsmall = or(vsmall, qv[c] < kDecPolyMax); + } +#else const half w0 = hbox(wa), w4 = hbox(wa >> 16), w1 = hbox(wb), w3 = hbox(wb >> 16); if (lut) { // the output values T[r] (wg_f16lut.hlsli, lev-f16lut d7149e2): the chain below, looked up float3 res; @@ -792,6 +845,7 @@ void pd_vitem(int y, int xg, int xo, int urel, uint wa, uint wb, int p_next, boo vo[c] = exp2(qv[c] * kDecK) - 1.0f; vsmall = or(vsmall, qv[c] < kDecPolyMax); } +#endif if (WaveActiveAnyTrue(vsmall)) { // Dec's polynomial only when some lane needs it (post.hip's branches) [unroll] for (int c = 0; c < 3; ++c) vo[c] = qv[c] < kDecPolyMax ? dec_m1_poly(qv[c]) : vo[c]; } @@ -823,8 +877,13 @@ void pd_vsplit(int it, out int yrel, out int xo, out int urel, out uint wa, out yrel = 2 * pr + dy; urel = 3 * pr + 2 * dy; const bool ev = dy == 0; +#if WG_LZ_ANALYTIC + wa = ev ? 1u : 0u; // the phase only (pd_vitem selects the constant weights) + wb = 0u; +#else wa = ((ev ? lz_w0 : lz_w4) & 0xffffu) | ((ev ? lz_w4 : lz_w0) << 16); wb = ((ev ? lz_w1 : lz_w3) & 0xffffu) | ((ev ? lz_w3 : lz_w1) << 16); +#endif } #endif @@ -928,7 +987,11 @@ WG_KERNEL(post_tf_p12_diet_vsit) dec_scale = unif(wg_ldf(expo, 16u)); } // the output table of this frame (wg_f16lut.hlsli; color: the table, 0 = none), wave-uniform +#if WG_LZ_ANALYTIC + const bool lut = false; // the table is indexed by an FP16 filter result (the original's); the analytic one is FP32 +#else const bool lut = WaveReadLaneFirst(f16lut_valid(color, dec_scale, nd_color_pi, nd_output_rne16) ? 1u : 0u) != 0u; +#endif #endif // this lane's D column (fixed over the walk) @@ -1173,6 +1236,18 @@ WG_KERNEL(post_tf_p12_diet_vsit) #else const uint src = scr + (uint)hk * kRowDw + (uint)hc * 32u + (uint)hg * 3u; #endif +#if WG_LZ_ANALYTIC + float t[10]; // the FP16 display taps as FP32 values (exact); results FP32 bits + [unroll] for (int i = 0; i < 5; ++i) { + const uint q = lds[src + (uint)i]; + t[2 * i] = f16tof32(q); + t[2 * i + 1] = f16tof32(q >> 16); + } + hv[0] = lz5a_even(t, 0); + hv[1] = lz5a_odd(t, 2); + hv[2] = lz5a_even(t, 3); + hv[3] = lz5a_odd(t, 5); +#else uint t[10]; [unroll] for (int i = 0; i < 5; ++i) { const uint q = lds[src + (uint)i]; @@ -1183,6 +1258,7 @@ WG_KERNEL(post_tf_p12_diet_vsit) hv[1] = lz5_odd(t, 2); hv[2] = lz5_even(t, 3); hv[3] = lz5_odd(t, 5); +#endif } tf_row_sync(); if (hact) { @@ -1255,13 +1331,22 @@ WG_KERNEL(post_tf_p12_diet_vsit) } // emit (n = 1): weights of the lane's phase by selects (lz_wsel), lz5w, Dec, store 3 x f32 const bool ev = dy == 0; +#if WG_LZ_ANALYTIC + const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1; +#else const half w0 = hbox(ev ? lz_w0 : lz_w4), w4 = hbox(ev ? lz_w4 : lz_w0); const half w1 = hbox(ev ? lz_w1 : lz_w3), w3 = hbox(ev ? lz_w3 : lz_w1); +#endif float vo[3], qv[3]; bool vsmall = false; [unroll] for (int c = 0; c < 3; ++c) { +#if WG_LZ_ANALYTIC + qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]), + w0, w1, w3, w4), k1em4, 3.0f); +#else const uint r = lz5_original(t[0][c], t[1][c], t[2][c], t[3][c], t[4][c], w0, w1, hbox(lz_w2), w3, w4); qv[c] = clampf(f16tof32(r), k1em4, 3.0f); +#endif vo[c] = exp2(qv[c] * kDecK) - 1.0f; vsmall = or(vsmall, qv[c] < kDecPolyMax); } diff --git a/clean/dx12/tests/kernel_replay.cpp b/clean/dx12/tests/kernel_replay.cpp index f436f29e..e663d49b 100644 --- a/clean/dx12/tests/kernel_replay.cpp +++ b/clean/dx12/tests/kernel_replay.cpp @@ -22,6 +22,9 @@ // (NativeD3D12Ext::bind_direct_frame) around its launch. // direct_params the frame's active render rectangle; motion scale 1, offset 0, // color_pi 1, no exposure texture +// direct_out after the launch, the output texture's texels are written to , +// row by row (tightly packed); its initial texels come from the output +// slot's file (a sentinel fill, so that an unwritten texel shows) #include "../../runtime/device.h" #include "../../runtime/device_d3d12.h" @@ -75,8 +78,71 @@ struct Case { std::vector args; std::vector direct; NativeDirectParams dparams; + std::string direct_out; // the output texture's readback file ("" = none) }; +// The output texture (UNORDERED_ACCESS) copied to a readback buffer after the case's work, rows packed into `out`. +bool read_direct_output(NativeD3D12Ext* ext, Case& c, std::vector& out, std::string& err) { + DirectTex* t = nullptr; + for (auto& x : c.direct) + if (x.slot == "output") t = &x; + if (!t || !t->tex) return err = "no output texture", false; + ID3D12Device* dev = nullptr; + if (FAILED(ext->arena_resource()->GetDevice(__uuidof(ID3D12Device), (void**)&dev))) return err = "GetDevice", false; + const D3D12_RESOURCE_DESC rd = t->tex->GetDesc(); + D3D12_PLACED_SUBRESOURCE_FOOTPRINT fp = {}; + UINT rows = 0; + UINT64 row_bytes = 0, total = 0; + dev->GetCopyableFootprints(&rd, 0, 1, 0, &fp, &rows, &row_bytes, &total); + D3D12_HEAP_PROPERTIES hp = {}; + hp.Type = D3D12_HEAP_TYPE_READBACK; + D3D12_RESOURCE_DESC bd = {}; + bd.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER; + bd.Width = total; + bd.Height = 1; + bd.DepthOrArraySize = 1; + bd.MipLevels = 1; + bd.SampleDesc.Count = 1; + bd.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR; + ID3D12Resource* rb = nullptr; + const bool made = SUCCEEDED(dev->CreateCommittedResource(&hp, D3D12_HEAP_FLAG_NONE, &bd, D3D12_RESOURCE_STATE_COPY_DEST, + nullptr, __uuidof(ID3D12Resource), (void**)&rb)); + dev->Release(); + if (!made) return err = "readback buffer", false; + ID3D12Resource* tex = t->tex; + bool ok = ext->record_pass(D3D12_RESOURCE_STATE_UNORDERED_ACCESS, [&](ID3D12GraphicsCommandList* l, std::string&) { + D3D12_RESOURCE_BARRIER b = {}; + b.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION; + b.Transition.pResource = tex; + b.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES; + b.Transition.StateBefore = D3D12_RESOURCE_STATE_UNORDERED_ACCESS; + b.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; + l->ResourceBarrier(1, &b); + D3D12_TEXTURE_COPY_LOCATION dst = {}, src = {}; + dst.pResource = rb; + dst.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT; + dst.PlacedFootprint = fp; + src.pResource = tex; + src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX; + l->CopyTextureRegion(&dst, 0, 0, 0, &src, nullptr); + return true; + }, &err) && ext->sync(&err); + uint8_t* map = nullptr; + if (ok) ok = SUCCEEDED(rb->Map(0, nullptr, (void**)&map)); + if (ok) { + out.resize((size_t)row_bytes * rows); + for (UINT r = 0; r < rows; ++r) + std::memcpy(out.data() + (size_t)r * row_bytes, map + fp.Offset + (size_t)r * fp.Footprint.RowPitch, + (size_t)row_bytes); + D3D12_RANGE none = {0, 0}; + rb->Unmap(0, &none); + } else if (err.empty()) { + err = "readback"; + } + rb->Release(); + return ok; +} + void release_direct(Case& c) { for (auto& t : c.direct) { if (t.tex) t.tex->Release(); @@ -247,6 +313,14 @@ int run_case(Device* d, Case& c, const std::string& base) { release_direct(c); return 1; } + if (!c.direct_out.empty()) { + std::vector texels; + if (!read_direct_output(ext, c, texels, err) || !write_file(base + "/" + c.direct_out, texels)) { + std::printf("%s: direct output: %s\n", c.name.c_str(), err.c_str()); + release_direct(c); + return 1; + } + } } if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return 1; } release_direct(c); @@ -300,6 +374,8 @@ int main(int argc, char** argv) { c.direct.push_back(t); } else if (w == "direct_params") { s >> c.dparams.rx >> c.dparams.ry >> c.dparams.rw >> c.dparams.rh; + } else if (w == "direct_out") { + s >> c.direct_out; } else if (w == "arg") { Arg a; s >> a.kind; diff --git a/clean/dx12/tests/post_tf_d3d12_check.py b/clean/dx12/tests/post_tf_d3d12_check.py new file mode 100644 index 00000000..af96264e --- /dev/null +++ b/clean/dx12/tests/post_tf_d3d12_check.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: MIT +"""The analytic-filter D3D12 ports of Model M's fused output kernels (clean/dx12/kernels/post.hlsl, WG_LZ_ANALYTIC) +against HIP's post_w64.co on the same synthetic inputs, byte for byte; the original-filter port (post_w64__orig.co, +already proven on whole frames by clean/dx12/compare.py) as the Model M control on the same harness. + + python clean/dx12/tests/post_tf_d3d12_check.py --build --rocm "C:\\Program Files\\AMD\\ROCm\\7.2" --run + [--seeds N] --json out.json + + holds post_w64.co and post_w64__orig.co (clean/runner/build.py's builds of kernels/post.hip), the D3D12 ports +(clean/dx12/build_dx12.py --only w64/post_tf_f32_p12_diet_vsit,w64/post_tf4_dl_f32_p12_diet_vsit,native_fill: both +builds of each name) and kernel_replay.exe (clean/l/runner/l_d3d12_check.py prepare_replay). --run runs in one GPU slot. +The engine and rules are clean/l/runner/l_d3d12_ops_check.py's: the RAW downloaded bytes of every output (the full +history with its alpha, the reduced codes, the FP32 output image) are compared; each output buffer starts filled with +its back end's own sentinel (HIP 0xA5, D3D12 0x5A) and 64 slack bytes follow it, which no back end may write; a negative +control (one case again on D3D12 with half its workgroups) must be caught. Output: counts only, no timing. + +Kernels: +- w64/post_tf_f32_p12_diet_vsit (analytic port) vs HIP post_tf_f32_p12_diet_vsit of post_w64.co; +- w64/post_tf4_dl_f32_p12_diet_vsit (analytic port; D3D12 only, NW = 4 row pairs per step, 256 lanes) vs HIP + post_tf_f32_p12_diet_vsit (frames that its cap of 352 takes) or post_tf_dl_f32_p12 (native AA: D = 1.5 R), whose + every output pixel is the same function (the exposure pair holds the ratio and dec_scale that the non-diet kernel + computes itself); +- control: w64/post_tf_f32_p12_diet_vsit (original-filter port, five FP16 weights as arguments) vs post_w64__orig.co. + The same name as the first: the D3D12 device runs the port of the code object the case loads under "w64". +""" +import argparse +import json +import os +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +from clean.l.runner import l_d3d12_ops_check as OPS # noqa: E402 +from clean.reference import post as MP # noqa: E402 +from clean.runner import post as RP # noqa: E402 +from clean.runner import post_sizes as PZ # noqa: E402 + +SLACK = 64 +CPAIRS = 68 # runtime.cpp kTfCpairs +# (rw, rh, ow, oh, reset): Quality (D = 2.25 R), Performance (D = 3 R), odd sizes, reset, a wide frame of 64 strips whose +# chunk plan has k1 > 0 chunks of 68 row pairs and then chunks of 36 +TF_FRAMES = [(64, 36, 96, 54, False), (48, 27, 96, 54, False), (50, 29, 99, 57, True), (70, 41, 105, 61, False), + (1280, 187, 2560, 280, False)] +# post_tf4: Quality frames (HIP post_tf_f32_p12_diet_vsit takes them too) and native AA (D = 1.5 R: HIP post_tf_dl) +TF4_FRAMES = [(64, 36, 96, 54, False), (50, 29, 99, 57, True), (64, 36, 64, 36, False), (61, 33, 61, 33, False), + (96, 54, 96, 54, True)] +# FP16 bits of the original filter's five weights for the control (spec 06 draft 4 gives them from the model file; any +# FP16 set exercises the port) +ORIG_W = [np.float16(v).view(np.uint16) for v in (-0.0908, 0.1129, 0.6400, 0.4092, -0.0706)] + + +def _f32_bits(v): + return int(np.float32(v).view(np.uint32)) + + +def tf_case(kernel, rng, frame, co, hip_kernel, block, hip_block, taps, expo_pair): + rw, rh, ow, oh, reset = frame + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + x = MP.random_inputs(rng, rw, rh, dw, dh, reset=reset) + prm = RP.post_params(x) + data = RP.pack12_inputs(RP.post_device_inputs(x)) + k1, c2, nch = PZ.tf_chunks(ow, oh, CPAIRS) + e, ep = prm.e, prm.e_prev + dec_scale = float(np.float32(0.01 / e)) + ratio = float(np.float32(np.float64(e) / np.float64(ep))) + n_hist, n_red, n_out = dw * dh * 8, max(4, (dw // 2) * (dh // 2) * 4), ow * oh * 12 + red_def = np.zeros(n_red, bool) + red_def[:(dw // 2) * (dh // 2) * 4] = True + slack = np.zeros(SLACK, bool) + defined = [np.concatenate([np.ones(n_hist, bool), slack]), np.concatenate([red_def, slack]), + np.concatenate([np.ones(n_out, bool), slack])] + args = [("buf", data[k]) for k in ("color", "A")] + [("null",)] + args += [("buf", data[k]) for k in ("C", "hist_prev", "m_near", "m_center", "m_far")] + args += [("buf", RP.pack({"s": np.array(rng.uniform(0.5, 4.0))})), ("out", n_hist + SLACK), ("out", n_red + SLACK), + ("out", n_out + SLACK), ("bytes", bytes(prm))] + args += [("buf", np.array([e, ep, 0.0, ratio, dec_scale], np.float32).tobytes())] if expo_pair else [("null",)] + args += [("u32", ow), ("u32", oh), ("u32", _f32_bits(dec_scale)), ("u32", _f32_bits(1.0)), ("u32", 12 * ow), + ("u32", CPAIRS), ("u32", k1), ("u32", c2)] + if taps: + args += [("u32", int(w)) for w in ORIG_W] + return {"kernel": kernel, "co": co, "tag": "w64", "hip_kernel": hip_kernel, "grid": PZ.PS._cdiv(ow, 40) * nch, + "block": block, "hip_block": hip_block, "args": args, "defined": defined, + "label": f"{rw}x{rh}->{ow}x{oh} (D {dw}x{dh})" + (" reset" if reset else "") + (" expo" if expo_pair else "")} + + +def tf_cases(kernel, rng): + return [tf_case(kernel, rng, f, "post_w64.co", "post_tf_f32_p12_diet_vsit", 512, 512, False, i % 2 == 0) + for i, f in enumerate(TF_FRAMES)] + + +def tf4_cases(kernel, rng): + out = [] + for i, f in enumerate(TF4_FRAMES): + rw, rh, ow, oh, _ = f + dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh) + hk = "post_tf_f32_p12_diet_vsit" if RP.tf_supported(rw, rh, dw, dh) else "post_tf_dl_f32_p12" + assert RP.tf_supported(rw, rh, dw, dh, 4, RP.TF4_DL_CAP), f + # the non-diet HIP kernel computes the ratio and takes dec_scale from its argument: the exposure pair is given + out.append(tf_case(kernel, rng, f, "post_w64.co", hk, 256, 512, False, True if hk == "post_tf_dl_f32_p12" + else i % 2 == 0)) + return out + + +def control_cases(kernel, rng): + return [dict(tf_case(kernel, rng, f, "post_w64__orig.co", "post_tf_f32_p12_diet_vsit", 512, 512, True, i % 2 == 0), + kernel="w64/post_tf_f32_p12_diet_vsit") for i, f in enumerate(TF_FRAMES[:4])] + + +KINDS = {"w64/post_tf_f32_p12_diet_vsit": tf_cases, "w64/post_tf4_dl_f32_p12_diet_vsit": tf4_cases, + "orig_w64/post_tf_f32_p12_diet_vsit": control_cases} + +# ---- the direct forms (rs=direct: the colour from a texture, the output into a texture) ------------------------------- +# No HIP kernel of their own: form 3 is form 0 followed by the output store (post.hlsl). With color_pi 1, an RGBA32F +# colour texture holding the colour buffer's values and an RGBA32F output, the direct port's output texels must be +# HIP's FP32 output (form 0) with alpha 1, its history and reduced codes HIP's, and its out_p buffer untouched. +def orig_direct_cases(kernel, rng): + """Control: the original filter's direct port (post_w64__orig.co) against HIP's original form 0.""" + return [dict(c, kernel="w64/post_tf_f32_p12_diet_vsit_direct") for c in control_cases(kernel, rng)] + + +DIRECT = {"w64/post_tf_f32_p12_diet_vsit_direct": (tf_cases, "w64/post_tf_f32_p12_diet_vsit"), + "w64/post_tf4_dl_f32_p12_diet_vsit_direct": (tf4_cases, "w64/post_tf4_dl_f32_p12_diet_vsit"), + "orig_w64/post_tf_f32_p12_diet_vsit_direct": (orig_direct_cases, "w64/post_tf_f32_p12_diet_vsit")} +TEX_FILL = 0x5A5A5A5A + + +def direct_run(build, rocm, seeds): + import subprocess + from clean.l.runner import l_c32_check as C32 + g = C32.Gpu(build, rocm) + mods = {} + cdir = os.path.join(build, "replay-direct") + os.makedirs(cdir, exist_ok=True) + lines, todo, neg = [], [], None + for dk, (gen, _) in DIRECT.items(): + for si, seed in enumerate(seeds): + rng = np.random.default_rng(seed) + for ci, case in enumerate(gen(dk, rng)): + if case["co"] not in mods: + with open(os.path.join(build, case["co"]), "rb") as f: + mods[case["co"]] = g.hip.load_module(f.read()) + name = f"{dk.replace('/', '_')}_s{si}_{ci}" + want = OPS.hip_run(g, mods[case["co"]], case) + rw, rh, ow, oh = (int(v) for v in case["label"].split()[0].replace("->", "x").split("x")) + # kernel_replay direct lines: color RGBA32F (the colour buffer's values), depth and motion of zeros (not + # read by these kernels), the output RGBA32F filled with the sentinel word; render rectangle at (0, 0) + case = dict(case, direct=[ + ("color", OPS.DXGI_R32G32B32A32_FLOAT, rw, rh, case["args"][0][1]), + ("depth", OPS.DXGI_R32_FLOAT, rw, rh, bytes(rw * rh * 4)), + ("motion", OPS.DXGI_R32G32_FLOAT, rw, rh, bytes(rw * rh * 8)), + ("output", OPS.DXGI_R32G32B32A32_FLOAT, ow, oh, + np.full(ow * oh * 4, TEX_FILL, np.uint32).tobytes())], direct_params=(0, 0, rw, rh)) + body = OPS.replay_lines(name, case, cdir) + lines += body[:-1] + [f"direct_out {name}.tex", ""] + todo.append((name, dk, case, want, ow, oh)) + if neg is None and case["grid"] >= 4: + neg = (name, case, ow, oh) + nl = OPS.replay_lines(name, case, cdir, grid=case["grid"] // 2, prefix="negative_") + lines += nl[:-1] + [f"direct_out negative_{name}.tex", ""] + with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: + f.write("\n".join(lines)) + env = {k: v for k, v in os.environ.items() + if not any(s in k.upper() for s in ("HIP", "LLVM", "ROCM", "HSA")) or k.upper() in ("HIP_PLATFORM",)} + env["WG_D3D12_ARENA_MB"] = "512" + r = subprocess.run([os.path.join(build, "kernel_replay.exe"), build, os.path.join(cdir, "cases.txt")], env=env, + capture_output=True, text=True, timeout=150, creationflags=0x08000000) + + def read(path): + return open(path, "rb").read() if os.path.exists(path) else b"" + + def judge(name, case, want, ow, oh, prefix=""): + """(identical, counts): history and reduced as l_d3d12_ops_check (defined bytes equal, slack keeps the + sentinels); out_p untouched by the direct port; texture RGB = HIP's FP32 output, alpha = 1.0.""" + o = {} + bad = 0 + for oi in (0, 1): + got = read(os.path.join(cdir, f"{prefix}{name}.o{oi}")) + a, b = np.frombuffer(got, np.uint8), np.frombuffer(want[oi], np.uint8) + m = case["defined"][oi] + d = int(np.sum((a != b) & m)) if a.size == b.size else -1 + u = int(np.sum(~m & ((b != OPS.HIP_SENTINEL) | (a != OPS.D3D12_SENTINEL)))) if a.size == b.size else -1 + o[f"o{oi}"] = {"bytes": len(want[oi]), "differ": d, "unwritten_bad": u} + bad += d != 0 or u != 0 + outp = np.frombuffer(read(os.path.join(cdir, f"{prefix}{name}.o2")), np.uint8) + o["out_p_untouched"] = bool(outp.size and np.all(outp == OPS.D3D12_SENTINEL)) + tex = np.frombuffer(read(os.path.join(cdir, f"{prefix}{name}.tex")), np.uint32) + ref = np.frombuffer(want[2][:ow * oh * 12], np.uint32).reshape(-1, 3) + if tex.size == ow * oh * 4: + t = tex.reshape(-1, 4) + o["tex"] = {"texels": ow * oh, "rgb_bytes": ow * oh * 12, "rgb_words_differ": int(np.sum(t[:, :3] != ref)), + "alpha_not_one": int(np.sum(t[:, 3] != 0x3F800000))} + bad += o["tex"]["rgb_words_differ"] != 0 or o["tex"]["alpha_not_one"] != 0 + else: + o["tex"] = {"missing": True} + bad += 1 + bad += not o["out_p_untouched"] + return bad == 0, o + + res = {"replay_exit": r.returncode, "replay_tail": r.stdout[-600:] + r.stderr[-300:], "cases": []} + for name, dk, case, want, ow, oh in todo: + ok, o = judge(name, case, want, ow, oh) + res["cases"].append({"case": name, "kernel": dk, "label": case["label"], "identical": ok, "outputs": o}) + if neg: + name, case, ow, oh = neg + want = next(t[3] for t in todo if t[0] == name) + ok, o = judge(name, case, want, ow, oh, prefix="negative_") + res["negative_control"] = {"case": name, "half_grid": True, "caught": not ok, + "tex_rgb_words_differ": o["tex"].get("rgb_words_differ")} + res["per_kernel"] = {k: {"cases": sum(c["kernel"] == k for c in res["cases"]), + "identical": sum(c["identical"] for c in res["cases"] if c["kernel"] == k), + "tex_rgb_bytes": sum(c["outputs"]["tex"].get("rgb_bytes", 0) for c in res["cases"] + if c["kernel"] == k)} + for k in DIRECT} + res["pass"] = (r.returncode == 0 and bool(neg) and res["negative_control"]["caught"] + and all(v["cases"] > 0 and v["identical"] == v["cases"] for v in res["per_kernel"].values())) + return res + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", required=True) + ap.add_argument("--run", action="store_true") + ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--kernels", default=",".join(KINDS)) + ap.add_argument("--direct", action="store_true", help="the direct forms instead (see DIRECT)") + ap.add_argument("--json") + a = ap.parse_args(argv) + if not a.run: + return 0 + kernels = [k for k in a.kernels.split(",") if k] + seeds = tuple(range(89, 89 + a.seeds)) + res = direct_run(a.build, a.rocm, seeds) if a.direct else OPS.run(a.build, a.rocm, seeds, kernels, kinds=KINDS) + if a.json: + with open(a.json, "w") as f: + json.dump(res, f, indent=1) + print({k: v for k, v in res.items() if k not in ("cases", "replay_tail")}) + return 0 if res["pass"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 55c7c543..02915333 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -334,7 +334,8 @@ def hip_run(g, mod, case): elif a[0] == "bytes": args.append((ctypes.c_ubyte * len(a[1])).from_buffer_copy(a[1])) try: - h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"], case["block"], args) + h.launch(h.get_function(mod, case.get("hip_kernel", case["kernel"])), case["grid"], + case.get("hip_block", case["block"]), args) h.sync() return [bytes(h.download(p, n)) for p, n in outs] finally: @@ -371,7 +372,9 @@ def replay_lines(name, case, cdir, grid=None, prefix=""): return lines + [""] -def run(build, rocm, seeds, kernels): +def run(build, rocm, seeds, kernels, kinds=None): + """kinds: {kernel: case generator} (default this file's KINDS and CONTROL_KINDS). A case may name its HIP kernel + (hip_kernel) and HIP block size (hip_block) when the D3D12 port has none of its own.""" g = C32.Gpu(build, rocm) mods = {} @@ -381,7 +384,7 @@ def module(co): mods[co] = g.hip.load_module(f.read()) return mods[co] - kinds = dict(KINDS, **CONTROL_KINDS) + kinds = kinds or dict(KINDS, **CONTROL_KINDS) cdir = os.path.join(build, "replay-ops") os.makedirs(cdir, exist_ok=True) lines, hip_out, neg = [], {}, None diff --git a/clean/tests/test_post_tf_geometry.py b/clean/tests/test_post_tf_geometry.py index a7af950f..4f1b2355 100644 --- a/clean/tests/test_post_tf_geometry.py +++ b/clean/tests/test_post_tf_geometry.py @@ -161,7 +161,9 @@ def test_nw4_takes_every_ratio_from_1_to_1_5(self): def test_nw4_declarations_agree(self): """runtime.cpp's tf4_supported (cap, NW = 4 footprint), runner/post.py's TF4_DL_CAP and the four post_tf4_dl_* - kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same.""" + kernels of dx12/kernels/post.hlsl (NW = 4, the same cap, 256 lanes) say the same. A name may be declared once per + build it ports (the original filter's post_w64__orig.co, the analytic filter's post_w64.co): every declaration + line is checked.""" import re clean = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) rt = open(os.path.join(clean, "runtime", "runtime.cpp"), encoding="utf-8").read() @@ -169,7 +171,7 @@ def test_nw4_declarations_agree(self): self.assertIn("((long long)(63.0 * rw / dw) + 4) * ((long long)(6.0 * rh / dh) + 5) <= kTf4Cap", rt) hlsl = open(os.path.join(clean, "dx12", "kernels", "post.hlsl"), encoding="utf-8").read() decl = re.findall(r"(?m)^// @wg-kernel (\S*post_tf4_dl\S*) .*$", hlsl) - self.assertEqual(sorted(decl), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct", + self.assertEqual(sorted(set(decl)), ["pmo/post_tf4_dl_f32_p12_mo", "pmo/post_tf4_dl_f32_p12_mo_direct", "w64/post_tf4_dl_f32_p12_diet_vsit", "w64/post_tf4_dl_f32_p12_diet_vsit_direct"]) for line in re.findall(r"(?m)^// @wg-kernel \S*post_tf4_dl\S* .*$", hlsl): self.assertIn(" block=256 ", line)