Skip to content
Merged
85 changes: 85 additions & 0 deletions clean/dx12/kernels/post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,24 @@
// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448
// @wg-kernel pmo/post_tf4_dl_f32_p12_mo co=post_mo_w64__orig.co entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448
// @wg-kernel pmo/post_tf4_dl_f32_p12_mo_direct co=post_mo_w64__orig.co rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u,b40,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_POST_MO=1,WG_TF_NW=4,WG_TF_CAP=448
//
// Analytic output filter (WG_LZ_ANALYTIC = 1; HIP: post_w64.co, post.hip built with -mwavefrontsize64 and the default
// WG_LZ_PROFILE = 0, the build the runtime loads under "w64" when the model file has no output-filter weights, e.g.
// Model L's): the same kernels with spec 06's own Lanczos weights as FP32 constants (kLz0 ... kLz4): v = kLz0 t0, then
// one fused multiply-add per tap, clamp to the taps' range, every value FP32 (the horizontal results go to the ring as
// FP32, the vertical result straight into Dec). No weight arguments. Forms 0 and 3 (direct; without the FP16-indexed
// output table, which needs the original's FP16 filter result).
// @wg-kernel w64/post_tf_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_LZ_ANALYTIC=1
// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit__analytic entry=post_tf_f32_p12_diet_vsit block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1
// @wg-kernel w64/post_tf_f32_p12_diet_vsit_direct co=post_w64.co file=w64__post_tf_f32_p12_diet_vsit_direct__analytic rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=512 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_LZ_ANALYTIC=1
// @wg-kernel w64/post_tf4_dl_f32_p12_diet_vsit_direct co=post_w64.co file=w64__post_tf4_dl_f32_p12_diet_vsit_direct__analytic rs=direct entry=post_tf_f32_p12_diet_vsit_direct block=256 wave=64 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p,u,u,u,u,u,u,u,u defines=WG_DIRECT_TEXTURES=1,POST_OUT=3,WG_TF_NW=4,WG_TF_CAP=448,WG_LZ_ANALYTIC=1
#include "wg_platform.hlsli"
#ifndef WG_LZ_ANALYTIC
#define WG_LZ_ANALYTIC 0
#endif
#if WG_LZ_ANALYTIC && ((POST_OUT != 0 && POST_OUT != 3) || defined(WG_POST_MO))
#error "analytic output filter: forms 0 and 3 (direct) only"
#endif
#if defined(WG_DIRECT_TEXTURES)
#include "native_direct.hlsli"
#include "wg_f16lut.hlsli"
Expand Down Expand Up @@ -85,7 +102,9 @@ WG_ARGS_BEGIN
int mo_mw, mo_mh, mo_pitch, mo_wide; // WgMoParams (40 bytes)
float mo_sx, mo_sy, mo_bx, mo_by, mo_idw, mo_idh;
#endif
#if !WG_LZ_ANALYTIC
uint lz_w0, lz_w1, lz_w2, lz_w3, lz_w4; // FP16 bit patterns of the filter weights of taps -2 ... +2
#endif
WG_ARGS_END

// ---- constants (FP32 bit patterns of post.hip's constexpr values) ---------------------------------------------------
Expand Down Expand Up @@ -697,6 +716,26 @@ void mo_core(Gather g, float2 mv, int x, int y, int fx, int fy, float s, out flo
}
#endif

#if WG_LZ_ANALYTIC
// ---- output filter, profile 0 ("analytic", post_w64.co): FP32 constant weights of taps -2 ... +2 for an even output
// index (post.hip kLz0 ... kLz4; mirrored for an odd one), v = w0 t0, one fma per further tap, clamp to the taps' range
static const float kLz0 = asfloat(0xbdba3b4bu), kLz1 = asfloat(0x3de71ca9u), kLz2 = asfloat(0x3f23b70fu);
static const float kLz3 = asfloat(0x3ed182b5u), kLz4 = asfloat(0xbd90a4acu);
float lz5a(float t0, float t1, float t2, float t3, float t4, float w0, float w1, float w3, float w4)
{
float v = w0 * t0;
v = mad(w1, t1, v);
v = mad(kLz2, t2, v);
v = mad(w3, t3, v);
v = mad(w4, t4, v);
const float mid_lo = min(min(t1, t2), t3), mid_hi = max(max(t1, t2), t3);
const float lo = min(min(mid_lo, t0), t4), hi = max(max(mid_hi, t0), t4);
return min(max(v, lo), hi); // med3 (the taps are finite, lo <= hi)
}
// lz5<kEven> / lz5<false> on the FP32 taps t[o ... o + 4], as FP32 bits
uint lz5a_even(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz0, kLz1, kLz3, kLz4)); }
uint lz5a_odd(float t[10], int o) { return asuint(lz5a(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], kLz4, kLz3, kLz1, kLz0)); }
#else
// ---- output filter, profile 2 ("original"): FP16 weights, one FP16 fma per tap, clamp to the taps' range -------------
// Values are "boxes": FP16 bits in the low half of a dword.
half hbox(uint v) { return asfloat16((uint16_t)(v & 0xffffu)); }
Expand Down Expand Up @@ -726,6 +765,7 @@ uint lz5_odd(uint t[10], int o)
return lz5_original(t[o], t[o + 1], t[o + 2], t[o + 3], t[o + 4], hbox(lz_w4), hbox(lz_w3), hbox(lz_w2), hbox(lz_w1),
hbox(lz_w0));
}
#endif

// The reduced history's 10-bit code of one channel from its 2 x 2 sum (the chain of post.hip's reduced store)
uint rh_code(float sum)
Expand Down Expand Up @@ -774,6 +814,19 @@ void pd_vitem(int y, int xg, int xo, int urel, uint wa, uint wb, int p_next, boo
t[k][c] = lds[kRingBase + (uint)sl * kRowDw + (uint)(c * kTfOutX + xo)];
}
}
#if WG_LZ_ANALYTIC
// the analytic filter: wa = 1 for the even phase (pd_vsplit); FP32 results, so no FP16-indexed output table
const bool ev = wa != 0u;
const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1;
float vo[3], qv[3];
bool vsmall = false;
[unroll] for (int c = 0; c < 3; ++c) {
qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]),
w0, w1, w3, w4), k1em4, 3.0f);
vo[c] = exp2(qv[c] * kDecK) - 1.0f;
vsmall = or(vsmall, qv[c] < kDecPolyMax);
}
#else
const half w0 = hbox(wa), w4 = hbox(wa >> 16), w1 = hbox(wb), w3 = hbox(wb >> 16);
if (lut) { // the output values T[r] (wg_f16lut.hlsli, lev-f16lut d7149e2): the chain below, looked up
float3 res;
Expand All @@ -792,6 +845,7 @@ void pd_vitem(int y, int xg, int xo, int urel, uint wa, uint wb, int p_next, boo
vo[c] = exp2(qv[c] * kDecK) - 1.0f;
vsmall = or(vsmall, qv[c] < kDecPolyMax);
}
#endif
if (WaveActiveAnyTrue(vsmall)) { // Dec's polynomial only when some lane needs it (post.hip's branches)
[unroll] for (int c = 0; c < 3; ++c) vo[c] = qv[c] < kDecPolyMax ? dec_m1_poly(qv[c]) : vo[c];
}
Expand Down Expand Up @@ -823,8 +877,13 @@ void pd_vsplit(int it, out int yrel, out int xo, out int urel, out uint wa, out
yrel = 2 * pr + dy;
urel = 3 * pr + 2 * dy;
const bool ev = dy == 0;
#if WG_LZ_ANALYTIC
wa = ev ? 1u : 0u; // the phase only (pd_vitem selects the constant weights)
wb = 0u;
#else
wa = ((ev ? lz_w0 : lz_w4) & 0xffffu) | ((ev ? lz_w4 : lz_w0) << 16);
wb = ((ev ? lz_w1 : lz_w3) & 0xffffu) | ((ev ? lz_w3 : lz_w1) << 16);
#endif
}
#endif

Expand Down Expand Up @@ -928,7 +987,11 @@ WG_KERNEL(post_tf_p12_diet_vsit)
dec_scale = unif(wg_ldf(expo, 16u));
}
// the output table of this frame (wg_f16lut.hlsli; color: the table, 0 = none), wave-uniform
#if WG_LZ_ANALYTIC
const bool lut = false; // the table is indexed by an FP16 filter result (the original's); the analytic one is FP32
#else
const bool lut = WaveReadLaneFirst(f16lut_valid(color, dec_scale, nd_color_pi, nd_output_rne16) ? 1u : 0u) != 0u;
#endif
#endif

// this lane's D column (fixed over the walk)
Expand Down Expand Up @@ -1173,6 +1236,18 @@ WG_KERNEL(post_tf_p12_diet_vsit)
#else
const uint src = scr + (uint)hk * kRowDw + (uint)hc * 32u + (uint)hg * 3u;
#endif
#if WG_LZ_ANALYTIC
float t[10]; // the FP16 display taps as FP32 values (exact); results FP32 bits
[unroll] for (int i = 0; i < 5; ++i) {
const uint q = lds[src + (uint)i];
t[2 * i] = f16tof32(q);
t[2 * i + 1] = f16tof32(q >> 16);
}
hv[0] = lz5a_even(t, 0);
hv[1] = lz5a_odd(t, 2);
hv[2] = lz5a_even(t, 3);
hv[3] = lz5a_odd(t, 5);
#else
uint t[10];
[unroll] for (int i = 0; i < 5; ++i) {
const uint q = lds[src + (uint)i];
Expand All @@ -1183,6 +1258,7 @@ WG_KERNEL(post_tf_p12_diet_vsit)
hv[1] = lz5_odd(t, 2);
hv[2] = lz5_even(t, 3);
hv[3] = lz5_odd(t, 5);
#endif
}
tf_row_sync();
if (hact) {
Expand Down Expand Up @@ -1255,13 +1331,22 @@ WG_KERNEL(post_tf_p12_diet_vsit)
}
// emit (n = 1): weights of the lane's phase by selects (lz_wsel), lz5w, Dec, store 3 x f32
const bool ev = dy == 0;
#if WG_LZ_ANALYTIC
const float w0 = ev ? kLz0 : kLz4, w4 = ev ? kLz4 : kLz0, w1 = ev ? kLz1 : kLz3, w3 = ev ? kLz3 : kLz1;
#else
const half w0 = hbox(ev ? lz_w0 : lz_w4), w4 = hbox(ev ? lz_w4 : lz_w0);
const half w1 = hbox(ev ? lz_w1 : lz_w3), w3 = hbox(ev ? lz_w3 : lz_w1);
#endif
float vo[3], qv[3];
bool vsmall = false;
[unroll] for (int c = 0; c < 3; ++c) {
#if WG_LZ_ANALYTIC
qv[c] = clampf(lz5a(asfloat(t[0][c]), asfloat(t[1][c]), asfloat(t[2][c]), asfloat(t[3][c]), asfloat(t[4][c]),
w0, w1, w3, w4), k1em4, 3.0f);
#else
const uint r = lz5_original(t[0][c], t[1][c], t[2][c], t[3][c], t[4][c], w0, w1, hbox(lz_w2), w3, w4);
qv[c] = clampf(f16tof32(r), k1em4, 3.0f);
#endif
vo[c] = exp2(qv[c] * kDecK) - 1.0f;
vsmall = or(vsmall, qv[c] < kDecPolyMax);
}
Expand Down
76 changes: 76 additions & 0 deletions clean/dx12/tests/kernel_replay.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
// (NativeD3D12Ext::bind_direct_frame) around its launch.
// direct_params <rx> <ry> <rw> <rh> the frame's active render rectangle; motion scale 1, offset 0,
// color_pi 1, no exposure texture
// direct_out <file> after the launch, the output texture's texels are written to <file>,
// row by row (tightly packed); its initial texels come from the output
// slot's file (a sentinel fill, so that an unwritten texel shows)
#include "../../runtime/device.h"
#include "../../runtime/device_d3d12.h"

Expand Down Expand Up @@ -75,8 +78,71 @@ struct Case {
std::vector<Arg> args;
std::vector<DirectTex> direct;
NativeDirectParams dparams;
std::string direct_out; // the output texture's readback file ("" = none)
};

// The output texture (UNORDERED_ACCESS) copied to a readback buffer after the case's work, rows packed into `out`.
bool read_direct_output(NativeD3D12Ext* ext, Case& c, std::vector<uint8_t>& out, std::string& err) {
DirectTex* t = nullptr;
for (auto& x : c.direct)
if (x.slot == "output") t = &x;
if (!t || !t->tex) return err = "no output texture", false;
ID3D12Device* dev = nullptr;
if (FAILED(ext->arena_resource()->GetDevice(__uuidof(ID3D12Device), (void**)&dev))) return err = "GetDevice", false;
const D3D12_RESOURCE_DESC rd = t->tex->GetDesc();
D3D12_PLACED_SUBRESOURCE_FOOTPRINT fp = {};
UINT rows = 0;
UINT64 row_bytes = 0, total = 0;
dev->GetCopyableFootprints(&rd, 0, 1, 0, &fp, &rows, &row_bytes, &total);
D3D12_HEAP_PROPERTIES hp = {};
hp.Type = D3D12_HEAP_TYPE_READBACK;
D3D12_RESOURCE_DESC bd = {};
bd.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
bd.Width = total;
bd.Height = 1;
bd.DepthOrArraySize = 1;
bd.MipLevels = 1;
bd.SampleDesc.Count = 1;
bd.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
ID3D12Resource* rb = nullptr;
const bool made = SUCCEEDED(dev->CreateCommittedResource(&hp, D3D12_HEAP_FLAG_NONE, &bd, D3D12_RESOURCE_STATE_COPY_DEST,
nullptr, __uuidof(ID3D12Resource), (void**)&rb));
dev->Release();
if (!made) return err = "readback buffer", false;
ID3D12Resource* tex = t->tex;
bool ok = ext->record_pass(D3D12_RESOURCE_STATE_UNORDERED_ACCESS, [&](ID3D12GraphicsCommandList* l, std::string&) {
D3D12_RESOURCE_BARRIER b = {};
b.Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION;
b.Transition.pResource = tex;
b.Transition.Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES;
b.Transition.StateBefore = D3D12_RESOURCE_STATE_UNORDERED_ACCESS;
b.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE;
l->ResourceBarrier(1, &b);
D3D12_TEXTURE_COPY_LOCATION dst = {}, src = {};
dst.pResource = rb;
dst.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
dst.PlacedFootprint = fp;
src.pResource = tex;
src.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
l->CopyTextureRegion(&dst, 0, 0, 0, &src, nullptr);
return true;
}, &err) && ext->sync(&err);
uint8_t* map = nullptr;
if (ok) ok = SUCCEEDED(rb->Map(0, nullptr, (void**)&map));
if (ok) {
out.resize((size_t)row_bytes * rows);
for (UINT r = 0; r < rows; ++r)
std::memcpy(out.data() + (size_t)r * row_bytes, map + fp.Offset + (size_t)r * fp.Footprint.RowPitch,
(size_t)row_bytes);
D3D12_RANGE none = {0, 0};
rb->Unmap(0, &none);
} else if (err.empty()) {
err = "readback";
}
rb->Release();
return ok;
}

void release_direct(Case& c) {
for (auto& t : c.direct) {
if (t.tex) t.tex->Release();
Expand Down Expand Up @@ -247,6 +313,14 @@ int run_case(Device* d, Case& c, const std::string& base) {
release_direct(c);
return 1;
}
if (!c.direct_out.empty()) {
std::vector<uint8_t> texels;
if (!read_direct_output(ext, c, texels, err) || !write_file(base + "/" + c.direct_out, texels)) {
std::printf("%s: direct output: %s\n", c.name.c_str(), err.c_str());
release_direct(c);
return 1;
}
}
}
if (!d->sync()) { std::printf("%s: sync: %s\n", c.name.c_str(), d->err().c_str()); return 1; }
release_direct(c);
Expand Down Expand Up @@ -300,6 +374,8 @@ int main(int argc, char** argv) {
c.direct.push_back(t);
} else if (w == "direct_params") {
s >> c.dparams.rx >> c.dparams.ry >> c.dparams.rw >> c.dparams.rh;
} else if (w == "direct_out") {
s >> c.direct_out;
} else if (w == "arg") {
Arg a;
s >> a.kind;
Expand Down
Loading
Loading