Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,6 @@
"l_direct_color",
"-"
],
[
"l_direct_store",
"-"
],
[
"l_linear_32_32_f16",
"l_ops.co"
Expand All @@ -218,6 +214,10 @@
"l_linear_32_32_relu",
"l_ops.co"
],
[
"l_post_filter_direct",
"-"
],
[
"l_post_recon",
"l_post.co"
Expand Down
48 changes: 7 additions & 41 deletions clean/dx12/kernels/l_post_direct.hlsl
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
// SPDX-License-Identifier: MIT
// Model L's own post on the direct-resource route (runtime option l_post=own, native D3D12): the two D3D12-only steps
// around l_post_recon_p12 -> w64/post_filter_c1_s4, which read and write buffers (no direct forms). With them the direct
// frame gives the staged route's bytes: the staged route converts the game's textures into dense buffers (the
// Converter's input pass, color_divide when pi != 1), runs the same kernels, then scale_rgb when pi != 1 and the
// Converter's output pass into the game's texture.
// Model L's own post on the direct-resource route (runtime option l_post=own, native D3D12): the D3D12-only colour step
// before l_post_recon_p12, which reads its colour from a buffer (no direct form). With it the direct frame gives the
// staged route's bytes: the staged route converts the game's textures into dense buffers (the Converter's input pass,
// color_divide when pi != 1) and runs the same kernels. (The output: l_post_filter_direct, kernels/post_filter.hlsl,
// writes the game's texture as the staged route's scale_rgb and Converter output pass do.)
// l_direct_color: dense texel i < npix of the game's colour texture -> color_net (RGBA f32, 16 bytes):
// native_direct_normalized_color(i), the staged colour buffer's value (alpha +0; divided by pi as color_divide).
// l_direct_store: output pixel i < ow oh of the filter's FP32 RGB (out, 12 bytes a pixel) -> the game's output texture:
// times pi as scale_rgb (native_direct_scale) when pi != 1, then the typed store with alpha 1 as the Converter's
// output pass; with nd_output_rne16 (RGBA16F output-copy rows) the stored value is framebuffer_half's (+-65504 clamp,
// round to nearest even), as post.hlsl's direct forms store it (pd_out16).
// Grids: cdiv(npix, 256) and cdiv(ow oh, 256) workgroups of 256 lanes.
// Grid: cdiv(npix, 256) workgroups of 256 lanes.
//
// @wg-kernel l_direct_color co=- block=256 wave=32 args=p,u rs=direct defines=WG_DIRECT_TEXTURES=1,LPD_COLOR=1
// @wg-kernel l_direct_store co=- block=256 wave=32 args=p,u,u rs=direct defines=WG_DIRECT_TEXTURES=1,LPD_STORE=1
// @wg-kernel l_direct_color co=- block=256 wave=32 args=p,u rs=direct defines=WG_DIRECT_TEXTURES=1
#include "wg_platform.hlsli"
#include "native_direct.hlsli"

#if defined(LPD_COLOR)
WG_ARGS_BEGIN
uint color_net; // pointer
int npix;
Expand All @@ -30,31 +24,3 @@ WG_KERNEL(l_direct_color)
if ((int)i >= npix) return;
wg_st128(color_net, i * 16u, asuint(native_direct_normalized_color(i)));
}

#elif defined(LPD_STORE)
WG_ARGS_BEGIN
uint out_p; // pointer
int ow, oh;
WG_ARGS_END

// post.hip framebuffer_half's clamp (compares: a NaN passes), then wg_f2h (round to nearest even, NaN kept)
float lpd_half(float v)
{
const float c = select(v > 65504.0f, 65504.0f, select(v < -65504.0f, -65504.0f, v));
return f16tof32(wg_f2h(c));
}

WG_KERNEL(l_direct_store)
{
WG_SETUP();
const uint i = wg_block_id() * 256u + wg_lane_id();
if ((int)i >= ow * oh) return;
const uint y = i / (uint)ow, x = i - y * (uint)ow;
float3 v = float3(wg_ldf(out_p, i * 12u), wg_ldf(out_p, i * 12u + 4u), wg_ldf(out_p, i * 12u + 8u));
if (nd_color_pi != 1.0f)
v = float3(native_direct_scale(v.x, nd_color_pi), native_direct_scale(v.y, nd_color_pi),
native_direct_scale(v.z, nd_color_pi));
if (nd_output_rne16 != 0u) v = float3(lpd_half(v.x), lpd_half(v.y), lpd_half(v.z));
native_direct_output[uint2(x, y)] = float4(v, 1.0f);
}
#endif
34 changes: 34 additions & 0 deletions clean/dx12/kernels/post_filter.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,18 @@
// Dec (2^(kDecK q) - 1, the polynomial below kDecPolyMax), times dec_scale. The arithmetic of post.hlsl's analytic
// filter (WG_LZ_ANALYTIC), which is HIP's ISA.
//
// l_post_filter_direct (rs=direct; D3D12 only: Model L's own post on the direct-resource route, runtime option l_post=own):
// the same filter with the output in the game's output texture instead of out_p (not written), as the runtime's FP32
// output followed by l_direct_store stored it: times pi (native_direct_scale) when pi != 1, with output_rne16
// framebuffer_half's bytes (+-65504 clamp, round to nearest even), alpha 1. The lane of channel 0 of each output pixel
// gathers the other two channels from its lanes 3 g + 1, 3 g + 2 and stores the texel.
//
// @wg-kernel w64/post_filter_c1_s4 co=post_w64.co entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p
// @wg-kernel l_post_filter_direct co=- entry=post_filter_c1_s4 block=64 wave=64 args=p,p,u,u,u,u,u,p rs=direct defines=WG_DIRECT_TEXTURES=1
#include "wg_platform.hlsli"
#if defined(WG_DIRECT_TEXTURES)
#include "native_direct.hlsli"
#endif

WG_ARGS_BEGIN
uint display, out_p; // pointers
Expand Down Expand Up @@ -97,6 +107,15 @@ float4 row4(int pitch, int plane, int r, int m, bool fast, int c)
return float4(h[0], h[1], h[2], h[3]);
}

#if defined(WG_DIRECT_TEXTURES)
// framebuffer_half's clamp (compares: a NaN passes), then wg_f2h (round to nearest even, NaN kept)
float fd_half(float v)
{
const float c = select(v > 65504.0f, 65504.0f, select(v < -65504.0f, -65504.0f, v));
return f16tof32(wg_f2h(c));
}
#endif

#define kS 4 // row pairs per lane (post_filter_c1_s4)
#define kGroups 10 // column groups per 32-lane half

Expand Down Expand Up @@ -146,7 +165,22 @@ WG_KERNEL(post_filter_c1_s4)
const float r = dy == 0 ? lz5e(r0[i], r1[i], r2[i], r3[i], r4[i]) : lz5o(r2[i], r3[i], r4[i], r5[i], r6[i]);
const float qc = clampf(r, k1em4, 3.0f);
const float v = qc < kDecPolyMax ? dec_m1_poly(qc) : exp2(qc * kDecK) - 1.0f;
#if defined(WG_DIRECT_TEXTURES)
// channels 1, 2 of this pixel from the lanes after channel 0's (the same pixel, so the same loop exits)
const float val = opq(v) * dec_scale;
const uint l0 = (uint)(lane - c);
const float v1 = WaveReadLaneAt(val, l0 + 1u), v2 = WaveReadLaneAt(val, l0 + 2u);
if (c == 0 && i < ncols) {
float3 o = float3(val, v1, v2);
if (nd_color_pi != 1.0f)
o = float3(native_direct_scale(o.x, nd_color_pi), native_direct_scale(o.y, nd_color_pi),
native_direct_scale(o.z, nd_color_pi));
if (nd_output_rne16 != 0u) o = float3(fd_half(o.x), fd_half(o.y), fd_half(o.z));
native_direct_output[uint2((uint)(4 * m + i), (uint)y)] = float4(o, 1.0f);
}
#else
if (i < ncols) wg_stf(out_p, (o0 + (uint)i) * 12u + (uint)c * 4u, opq(v) * dec_scale);
#endif
}
}
}
Expand Down
69 changes: 44 additions & 25 deletions clean/dx12/tests/post_tf_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,11 @@
of post_w64.co: display image -> FP32 output, dec_scale from the argument or the exposure pair;
- control: w64/post_tf_f32_p12_diet_vsit (original-filter port, five FP16 weights as arguments) vs post_w64__orig.co.
The same name as the first: the D3D12 device runs the port of the code object the case loads under "w64".
--direct: the direct forms (DIRECT). --store: Model L's own post on the direct route, its output-copy store
(STORE_FRAMES): HIP's RGBA16F output-copy rows (post_filter_c1_s4_o16 / post_filter_h) against w64/post_filter_c1_s4
then l_direct_store with output_rne16 = 1 into an RGBA16F texture (filled with 0x5A beforehand: every texel must be
written), with FP16 subnormals, values past 65504, infinities, NaN and rounding ties among the stored values.
--direct: the direct forms (DIRECT). --store: Model L's own post on the direct route, the filter's direct form
l_post_filter_direct (STORE_FRAMES): HIP's RGBA16F output-copy rows (post_filter_c1_s4_o16 / post_filter_h) against its
RGBA16F texture with output_rne16 = 1 (filled with 0x5A beforehand: every texel must be written), with FP16 subnormals,
values past 65504, infinities, NaN and rounding ties among the stored values; an RGBA32F texture against HIP's FP32
output.
"""
import argparse
import json
Expand Down Expand Up @@ -264,17 +265,19 @@ def judge(name, case, want, ow, oh, prefix=""):
return res


# ---- Model L's own post on the direct route: the output-copy store (l_direct_store, output_rne16 = 1) --------------
# ---- Model L's own post on the direct route: the filter's direct form (l_post_filter_direct) ----------------------
# HIP's output copy of the same display image: post_filter_c1_s4_o16 (pi = 1) or post_filter_h (pi folded in, pitch 8 O)
# -> RGBA16F rows, framebuffer_half((Dec dec_scale) pi). D3D12: w64/post_filter_c1_s4 (FP32), then l_direct_store
# (dx12/kernels/l_post_direct.hlsl) on HIP's FP32 output into an RGBA16F output texture with color_pi and output_rne16 = 1.
# -> RGBA16F rows, framebuffer_half((Dec dec_scale) pi). D3D12: l_post_filter_direct (dx12/kernels/post_filter.hlsl) on
# the display image into an RGBA16F output texture with color_pi and output_rne16 = 1 (its out_p buffer untouched); also
# w64/post_filter_c1_s4 (FP32) against HIP's. Frames marked f32: an RGBA32F texture, output_rne16 = 0, pi 1: the texels
# are HIP's FP32 output with alpha 1.
# dec_scale / pi choices drive the stored values into FP16 subnormals, past 65504 (clamp), to +-infinity (clamp) and
# NaN (by the argument, with a payload, or from the exposure pair); ties of the FP16 rounding come from the many values.
# (ow, oh, pi, dec_scale: a float, or "expo_nan" = the exposure pair's e is NaN)
STORE_FRAMES = [(96, 54, 1.0, None), (99, 57, 1.375, None), (161, 83, 1.0, 2.0 ** -22), (161, 83, 1.375, 2.0 ** -22),
(1281, 140, 1.0, 2.0 ** -8), (1281, 140, 1.25, 2.0 ** -12), (40, 16, 1.0, float("inf")),
(40, 16, 1.375, float("-inf")), (7, 5, 1.0, "expo_nan"), (99, 57, 1.375, "nan_payload"),
(161, 83, 30000.0, None)]
(161, 83, 30000.0, None), (96, 54, 1.0, None, "f32"), (1281, 140, 1.0, 2.0 ** -12, "f32")]
TEX_FILL16 = 0x5A5A


Expand All @@ -285,7 +288,9 @@ def _s32(bits):
def store_cases(rng):
"""[(HIP FP32 filter case, HIP output-copy case, D3D12 filter case, pi, ow, oh, label)]"""
out = []
for ow, oh, pi, ds in STORE_FRAMES:
for fr in STORE_FRAMES:
ow, oh, pi, ds = fr[:4]
f32tex = len(fr) > 4 and fr[4] == "f32"
dw, dh = MP.recon_size_for(ow), MP.recon_size_for(oh)
disp = _display(rng, dw, dh)
e = float(np.float32(rng.uniform(0.5, 2.0)))
Expand Down Expand Up @@ -313,8 +318,9 @@ def store_cases(rng):
grid=PZ.PS._cdiv(ow, 128) * PZ.PS._cdiv(PZ.PS._cdiv(oh, 2), 16),
args=[base[0], ("out", n16 + SLACK)] + base[2:] +
[("u32", _s32(_f32_bits(np.float32(pi)))), ("u32", 8 * ow)])
label = f"O {ow}x{oh} pi {pi} dec_scale {ds if ds is not None else '0.01/e'} ({o16['hip_kernel']})"
out.append((f32, o16, pi, ow, oh, label))
label = f"O {ow}x{oh} pi {pi} dec_scale {ds if ds is not None else '0.01/e'} " + \
("(RGBA32F: post_filter_c1_s4)" if f32tex else f"({o16['hip_kernel']})")
out.append((f32, None if f32tex else o16, pi, ow, oh, label))
return out


Expand Down Expand Up @@ -347,16 +353,16 @@ def store_run(build, rocm, seeds):
for ci, (f32, o16, pi, ow, oh, label) in enumerate(store_cases(rng)):
name = f"store_s{si}_{ci}"
w32 = OPS.hip_run(g, mod, f32)[0]
w16 = OPS.hip_run(g, mod, o16)[0]
w16 = OPS.hip_run(g, mod, o16)[0] if o16 is not None else None
lines += OPS.replay_lines(f"{name}_filter", f32, cdir)
st = {"kernel": "l_direct_store", "co": "post_w64.co", "tag": "w64", "grid": PZ.PS._cdiv(ow * oh, 256),
"block": 256, "args": [("buf", w32[:ow * oh * 12]), ("u32", ow), ("u32", oh)],
"direct": [("color", OPS.DXGI_R32G32B32A32_FLOAT, 8, 8, bytes(8 * 8 * 16)),
("depth", OPS.DXGI_R32_FLOAT, 8, 8, bytes(8 * 8 * 4)),
("motion", OPS.DXGI_R32G32_FLOAT, 8, 8, bytes(8 * 8 * 8)),
("output", OPS.DXGI_R16G16B16A16_FLOAT, ow, oh,
np.full(ow * oh * 4, TEX_FILL16, np.uint16).tobytes())],
"direct_params": (0, 0, 8, 8, repr(float(np.float32(pi))), 1)}
f16 = o16 is not None
st = dict(f32, kernel="l_post_filter_direct", hip_kernel=None, args=f32["args"],
direct=[("color", OPS.DXGI_R32G32B32A32_FLOAT, 8, 8, bytes(8 * 8 * 16)),
("depth", OPS.DXGI_R32_FLOAT, 8, 8, bytes(8 * 8 * 4)),
("motion", OPS.DXGI_R32G32_FLOAT, 8, 8, bytes(8 * 8 * 8)),
("output", OPS.DXGI_R16G16B16A16_FLOAT if f16 else OPS.DXGI_R32G32B32A32_FLOAT, ow, oh,
bytes([0x5A]) * (ow * oh * (8 if f16 else 16)))],
direct_params=(0, 0, 8, 8, repr(float(np.float32(pi))), 1 if f16 else 0))
body = OPS.replay_lines(f"{name}_store", st, cdir)
lines += body[:-1] + [f"direct_out {name}_store.tex", ""]
if neg is None and st["grid"] >= 4:
Expand All @@ -375,7 +381,19 @@ def store_run(build, rocm, seeds):
def read(path):
return open(path, "rb").read() if os.path.exists(path) else b""

def tex_judge(path, w16, ow, oh):
def tex_judge(path, w16, ow, oh, w32=None, outp_path=None):
if outp_path is not None: # the direct form's out_p: untouched
a = np.frombuffer(read(outp_path), np.uint8)
if not (a.size and np.all(a == OPS.D3D12_SENTINEL)):
return False, {"out_p_written": True}
if w16 is None: # RGBA32F: HIP's FP32 output, alpha 1
t = np.frombuffer(read(path), np.uint32)
ref = np.frombuffer(w32[:ow * oh * 12], np.uint32).reshape(-1, 3)
if t.size != ow * oh * 4:
return False, {"bytes": ow * oh * 16, "missing": True}
t = t.reshape(-1, 4)
d = int(np.sum(t[:, :3] != ref)) + int(np.sum(t[:, 3] != 0x3F800000))
return d == 0, {"bytes": ow * oh * 16, "format": "RGBA32F", "differ_words": d}
n16 = ow * oh * 8
t = np.frombuffer(read(path), np.uint8)
h = np.frombuffer(w16, np.uint8)
Expand All @@ -396,7 +414,8 @@ def tex_judge(path, w16, ow, oh):
"differ": int(np.sum((got32 != b32) & m)) if got32.size == b32.size else -1,
"unwritten_bad": int(np.sum(~m & ((b32 != OPS.HIP_SENTINEL) | (got32 != OPS.D3D12_SENTINEL))))
if got32.size == b32.size else -1}
ok_t, tex = tex_judge(os.path.join(cdir, f"{name}_store.tex"), w16, ow, oh)
ok_t, tex = tex_judge(os.path.join(cdir, f"{name}_store.tex"), w16, ow, oh, w32,
os.path.join(cdir, f"{name}_store.o0"))
cls = _store_classes(np.frombuffer(w32[:ow * oh * 12], np.float32), pi)
for k, v in cls.items():
classes[k] = classes.get(k, 0) + v
Expand All @@ -405,11 +424,11 @@ def tex_judge(path, w16, ow, oh):
"stored_values": cls})
if neg:
t = next(x for x in todo if x[0] == neg)
ok, o = tex_judge(os.path.join(cdir, f"negative_{neg}_store.tex"), t[3], t[5], t[6])
ok, o = tex_judge(os.path.join(cdir, f"negative_{neg}_store.tex"), t[3], t[5], t[6], t[2])
res["negative_control"] = {"case": neg, "half_grid": True, "caught": not ok, "differ": o.get("differ")}
res["stored_value_classes"] = classes
res["summary"] = {"cases": len(res["cases"]), "identical": sum(c["identical"] for c in res["cases"]),
"rgba16f_bytes": sum(c["store_rgba16f"]["bytes"] for c in res["cases"]),
"texture_bytes": sum(c["store_rgba16f"].get("bytes", 0) for c in res["cases"]),
"filter_f32_bytes": sum(c["filter_f32"]["bytes"] for c in res["cases"])}
res["pass"] = (r.returncode == 0 and bool(neg) and res["negative_control"]["caught"] and
res["summary"]["identical"] == res["summary"]["cases"] > 0 and
Expand All @@ -425,7 +444,7 @@ def main(argv=None):
ap.add_argument("--seeds", type=int, default=2)
ap.add_argument("--kernels", default=",".join(KINDS))
ap.add_argument("--direct", action="store_true", help="the direct forms instead (see DIRECT)")
ap.add_argument("--store", action="store_true", help="Model L's output-copy store instead (see STORE_FRAMES)")
ap.add_argument("--store", action="store_true", help="Model L's direct filter instead (see STORE_FRAMES)")
ap.add_argument("--json")
a = ap.parse_args(argv)
if not a.run:
Expand Down
Loading
Loading