Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 83 additions & 5 deletions clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,17 @@ def kernel_decls(path):
if d["rs"] not in ("buffer", "tex", "direct"):
raise SystemExit(f"{path}: {name}: unsupported root-signature kind {d['rs']}")
d["arg_dwords"] = arg_dwords(d["args"], f"{path}: {name}")
# needs=fold | nofold: a port the device runs only when its start-up guard of the driver's conversion fold
# passed (fold) or failed (nofold); the two ports of one name and code object differ by this key
d["needs"] = kv.get("needs", "")
if d["needs"] not in ("", "fold", "nofold"):
raise SystemExit(f"{path}: {name}: needs={d['needs']} (fold or nofold)")
# isa=<rule>,<rule>: the port's RGA ISA must satisfy each rule, <op>[+<op>...]>=<n> or <op>==<n> (counts of
# instruction mnemonics; <op>@<literal>: only its x * literal + 0 sites, parse_isa_rules), checked after the
# build (isa_check; WG_DX12_ISA_CHECK=0 skips it)
d["isa"] = kv.get("isa", "")
if d["isa"]:
parse_isa_rules(d["isa"], f"{path}: {name}")
decls.append(d)
return decls

Expand Down Expand Up @@ -298,6 +309,65 @@ def rga_isa(dxil, isa_dir, stem):
shutil.copyfile(os.path.join(tmp, f), os.path.join(isa_dir, f"{stem}.{kind}"))


def parse_isa_rules(text, where):
"""isa=<rule>,...: [(terms, op, n)] with op '>=' or '=='; a term is (mnemonic, literal or None): <op> counts every
instruction of that mnemonic, <op>@<8 hex digits> only those whose literal dword is that value, whose last source
operand is 0 and that carry no op_sel modifier (an FP32 x * literal + 0: the sites of a conversion fold)."""
rules = []
for r in text.split(","):
m = re.fullmatch(r"([a-z0-9_]+(?:@[0-9a-f]{8})?(?:\+[a-z0-9_]+(?:@[0-9a-f]{8})?)*)(>=|==)(\d+)", r)
if not m:
raise SystemExit(f"{where}: isa rule {r!r} (<op>[@<literal>][+<op>[@<literal>]]>=<n> or ...==<n>)")
terms = [(t.split("@")[0], t.split("@")[1] if "@" in t else None) for t in m.group(1).split("+")]
rules.append((terms, m.group(2), int(m.group(3))))
return rules


def isa_count(lines, mnemonic, literal):
"""Instructions of `mnemonic` in RGA ISA lines (with literal: only the x * literal + 0 sites, parse_isa_rules)."""
c = 0
for line in lines:
code, _, comment = line.partition("//")
f = code.split()
if not f or f[0] != mnemonic:
continue
if literal is not None:
words = comment.split(":")[-1].split()
srcs = [o.strip() for o in " ".join(f[1:]).split(",")]
if not words or words[-1].lower() != literal or srcs[-1] != "0" or "op_sel" in code:
continue
c += 1
return c


def isa_check(built_isa):
"""built_isa: [(kernel name, dxil path, rules text)] of this build's ports with isa=. Each port's RGA ISA must
satisfy its rules (e.g. a driver fold the port's exactness depends on: a v_fma_mixlo_f16 count). Fails the build
otherwise; WG_DX12_ISA_CHECK=0 skips it (no RGA)."""
if not built_isa or os.environ.get("WG_DX12_ISA_CHECK", "1") == "0":
return
if not os.path.exists(RGA) and not RGA.endswith(".cmd"):
raise SystemExit(f"ISA check: RGA not found at {RGA} (WG_RGA, or WG_DX12_ISA_CHECK=0 to skip)")
bad = []
for name, dxil, text in built_isa:
with tempfile.TemporaryDirectory(prefix="wgisa_") as tmp:
rga_isa(dxil, tmp, "k")
path = os.path.join(tmp, "k.isa.txt")
if not os.path.exists(path):
bad.append(f"{name}: RGA gave no ISA")
continue
lines = open(path, encoding="utf-8", errors="replace").read().splitlines()
for terms, op, n in parse_isa_rules(text, name):
c = sum(isa_count(lines, mn, lit) for mn, lit in terms)
ok = c >= n if op == ">=" else c == n
label = "+".join(mn + (f"@{lit}" if lit else "") for mn, lit in terms)
print(f"ISA {name:40s} {label} = {c} (rule {op} {n}) {'ok' if ok else 'FAILED'}", flush=True)
if not ok:
bad.append(f"{name} ({os.path.basename(dxil)}): {label} = {c}, rule {op} {n}")
if bad:
raise SystemExit("ISA check failed:\n " + "\n ".join(bad))


VGPR_LIMITS = os.path.join(HERE, "vgpr_limits.json")


Expand Down Expand Up @@ -383,9 +453,12 @@ def main(argv=None):
old = {}
if only and os.path.exists(man_path): # partial build: keep the other kernels' entries
with open(man_path) as f:
old = {(k["name"], k.get("co", "")): k for k in json.load(f)["kernels"]}
kernels = dict(old) # (name, co) -> manifest entry
# every port of a rebuilt name is replaced (a name may gain or lose ports, e.g. needs=fold / nofold)
old = {(k["name"], k.get("co", ""), k.get("needs", "")): k for k in json.load(f)["kernels"]
if k["name"] not in only}
kernels = dict(old) # (name, co, needs) -> manifest entry
built = []
built_isa = [] # (name, dxil, isa rules) of the ports with isa=
# generated kernels (kernels/fused_*_gen.py: the fused forms of swin_block.hlsl / bneck.hlsl) are regenerated from
# the live sources first, so that no build compiles a stale copy
for fn in sorted(os.listdir(KERNEL_DIR)) if os.path.isdir(KERNEL_DIR) else []:
Expand All @@ -408,7 +481,7 @@ def main(argv=None):
continue
if not only and is_frame_gen(d):
continue
key = (d["name"], d["co"])
key = (d["name"], d["co"], d["needs"])
if key in kernels and key not in old:
raise SystemExit(f"kernel {d['name']} ({d['co']}) declared twice")
kernels[key] = None # placeholder: the declared-twice check sees it before any compile starts
Expand All @@ -433,6 +506,8 @@ def build_one(job):
r = {"name": d["name"], "file": os.path.basename(out), "co": d["co"], "entry": d["entry"],
"block": d["block"], "wave": d["wave"], "args": d["args"], "arg_dwords": d["arg_dwords"],
"source": d["source"], "sha256": sha, "rs": d["rs"]}
if d["needs"]:
r["needs"] = d["needs"]
if d["group"] != d["block"]:
r["group"] = d["group"]
r["gscale"] = d["gscale"]
Expand All @@ -448,18 +523,21 @@ def build_one(job):
futures = [pool.submit(build_one, j) for j in jobs]
try:
for (src, d, out, _k), fut in zip(jobs, futures):
kernels[(d["name"], d["co"])] = fut.result()
kernels[(d["name"], d["co"], d["needs"])] = fut.result()
built.append((d["name"], out))
if d["isa"]:
built_isa.append((d["name"], out, d["isa"]))
print(f"{d['name']:40s} -> {os.path.basename(out)} block {d['block']} wave {d['wave']} "
f"args {d['args']}", flush=True)
except BaseException:
for fut in futures:
fut.cancel() # kernels not started yet; the running ones finish their current child process
raise
names = {n for n, _ in kernels}
names = {k[0] for k in kernels}
if only and not (only <= names):
raise SystemExit(f"not declared in clean/dx12/kernels or clean/k/dx12/kernels: {sorted(only - names)}")
vgpr_check(built)
isa_check(built_isa)
with open(man_path, "w") as f:
json.dump({"format": FORMAT, "dxc": os.path.basename(os.path.dirname(os.path.dirname(DXC_DIR))),
"kernels": [kernels[k] for k in sorted(kernels)]}, f, indent=1)
Expand Down
57 changes: 57 additions & 0 deletions clean/dx12/hlsl/l_post_encode.hlsli
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// SPDX-License-Identifier: MIT
// Model L post: the history texel's FP16 encode through the driver's conversion fold, shared by
// clean/dx12/kernels/l_post.hlsl (l_post_recon_p12) and its start-up probe clean/dx12/kernels/l_fold_probe.hlsl, so
// that both compile the same expression. WrongGPU's own code. Include after wg_platform.hlsli.
//
// q_i = (_Float16)(kEncK x_i), one rounding of the exact product, as HIP's v_fma_mixlo_f16 x, kEncK, 0 (+0 for a zero
// product). Written as mad(x, kEncK, 0) feeding the driver's round-to-nearest-even matrix conversion, which the gfx1201
// D3D12 compiler folds into exactly that instruction (v_fma_mixlo_f16 / v_fma_mixhi_f16, as Model M's post). Without
// the fold the FP32 product would be rounded twice (a value whose FP32 product lands on an FP16 midpoint moves by one
// FP16 ulp). Guards: build_dx12.py's ISA check on the kernel (isa= in its declaration: the six sites, kEncK as the
// literal with a 0 addend) and two start-up steps of the native device, both of which must pass: the probe
// (l_fold_probe, clean/runtime/fold_probe.h: this expression on adversarial inputs) and the differential self-test of
// the real post shader (its fold port against its no-fold port on the crafted frame of clean/runtime/fold_selftest.h,
// raw bytes) with the check of the fold port's own values. Otherwise, or on any error, the device runs the no-fold
// port of the kernel.
#pragma once

#ifndef LP_MUT
#define LP_MUT 0
#endif
#ifndef LP_MUTSITE
#define LP_MUTSITE 0 // 1 ... 6: negative control (never shipped): site LP_MUTSITE - 1 = 3 row + channel rounded twice
#endif
static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f

// one history code's conversion input: mad(x, kEncK, 0), which the driver folds into the conversion (one rounding),
// or (twice, a compile-time constant of the negative controls) the FP32 product made opaque first (two roundings)
float lp_enc_in(float x, uint z, bool twice)
{
return twice ? asfloat(asuint(kEncK * x) ^ z) : mad(x, kEncK, 0.0f);
}

uint4 lp_encode_conv(float x0, float x1, float x2, float d0, float d1, float d2, uint z, int twice_ch)
{
v8f cv;
cv.v[0] = lp_enc_in(x0, z, LP_MUT == 3 || twice_ch == 0);
cv.v[1] = lp_enc_in(x1, z, LP_MUT == 3 || twice_ch == 1);
cv.v[2] = lp_enc_in(x2, z, LP_MUT == 3 || twice_ch == 2);
cv.v[3] = d0;
cv.v[4] = d1;
cv.v[5] = d2;
cv.v[6] = 0.0f;
cv.v[7] = 0.0f;
return wg_to_f16_ags(cv);
}

// the 8-value conversion of the history codes (x0 .. x2: the FP32 log2 arguments) and the three display values
// d0 .. d2 (already rounded to FP32, passed as opaque values by the caller); z = an opaque zero (the mutants only);
// row: the row of the pair (wave-uniform; the single-site mutants only)
uint4 lp_encode_fold(float x0, float x1, float x2, float d0, float d1, float d2, uint z, int row = 0)
{
#if LP_MUTSITE
[branch] if (row == (LP_MUTSITE - 1) / 3)
return lp_encode_conv(x0, x1, x2, d0, d1, d2, z, (LP_MUTSITE - 1) % 3);
#endif
return lp_encode_conv(x0, x1, x2, d0, d1, d2, z, -1);
}
4 changes: 4 additions & 0 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@
"l_direct_color",
"-"
],
[
"l_fold_probe",
"-"
],
[
"l_linear_32_32_f16",
"l_ops.co"
Expand Down
28 changes: 28 additions & 0 deletions clean/dx12/kernels/l_fold_probe.hlsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
// SPDX-License-Identifier: MIT
// Start-up probe of the driver's conversion fold used by Model L's l_post_recon_p12 (hlsl/l_post_encode.hlsli): the
// device runs it once (clean/runtime/fold_probe.h holds the inputs and the expected bytes, computed from the exact
// products) and takes the no-fold port of the kernel when any history code differs. WrongGPU's own code.
// Buffer: lane l reads six FP32 values at byte 24 l (x0 x1 x2 = log2 arguments of the history codes, d0 d1 d2 = display
// values) and writes the three converted dwords at byte 768 + 12 l (q0 | q1 << 16, q2 | h(d0) << 16, h(d1) | h(d2) << 16).
// @wg-kernel l_fold_probe co=- block=32 wave=32 args=p
#include "wg_platform.hlsli"
WG_ARGS_BEGIN
uint dst;
WG_ARGS_END
#include "l_post_encode.hlsli"

WG_KERNEL(l_fold_probe)
{
WG_SETUP();
const uint lane = wg_lane_id();
const uint z = wg_grid_ >> 31; // 0, opaque to the compiler
const uint in0 = lane * 24u;
const float x0 = wg_ldf(dst, in0), x1 = wg_ldf(dst, in0 + 4u), x2 = wg_ldf(dst, in0 + 8u);
const float d0 = asfloat(asuint(wg_ldf(dst, in0 + 12u)) ^ z), d1 = asfloat(asuint(wg_ldf(dst, in0 + 16u)) ^ z);
const float d2 = asfloat(asuint(wg_ldf(dst, in0 + 20u)) ^ z);
const uint4 hc = lp_encode_fold(x0, x1, x2, d0, d1, d2, z);
const uint o = 768u + lane * 12u;
wg_st32(dst, o, hc.x);
wg_st32(dst, o + 4u, hc.y);
wg_st32(dst, o + 8u, hc.z);
}
33 changes: 30 additions & 3 deletions clean/dx12/kernels/l_post.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@
// gives 0), the reduced codes rint(clamp(mean, 0, 1) 1023).
//
// @wg-kernel l_post_recon co=l_post.co block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p
// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1
// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_ENCFOLD=1 needs=fold isa=v_fma_mixlo_f16@3dddf184+v_fma_mixhi_f16@3dddf184==6,s_round_mode==0
// @wg-kernel l_post_recon_p12 co=l_post.co entry=l_post_recon_p12 file=l_post_recon_p12__nofold block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1 needs=nofold
// @wg-kernel l_post_recon_ldr co=l_post.co entry=l_post_recon_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_LDR=1
// @wg-kernel l_post_recon_p12_ldr co=l_post.co entry=l_post_recon_p12_ldr block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b64,p defines=LP_P12=1,LP_LDR=1
#include "wg_platform.hlsli"
Expand All @@ -49,14 +50,14 @@ WG_ARGS_BEGIN
uint expo;
WG_ARGS_END

#include "l_post_encode.hlsli" // kEncK and the history encode through the driver's conversion fold
#define kThreads 256
#define kTileX 32
#define kTilePairs 8
#define kStageCap 512
#define kLWin 4
#define kTexDw 6 // staged texel: f0 f1 | f2 f3 | f4 f5 | f6 f7 | f8 kA | kB kC (FP16 bits)

static const float kEncK = asfloat(0x3dddf184u); // 0.36f * 0.30102999566398120f
static const float kDecK = asfloat(0x4113a428u); // 3.32192809488736235f / 0.36f
static const float kDecLn = asfloat(0x40ccac9au); // 2.30258509299404568f / 0.36f
static const float kDecPolyMax = asfloat(0x3d60233du); // 0.35f / kDecLn
Expand All @@ -83,11 +84,14 @@ float opq(float x) { return asfloat(asuint(x) ^ g_z); }
half opqh(half x) { return asfloat16((uint16_t)(asuint16(x) ^ (uint16_t)g_z)); }
float mul_nz(float a, float b) { return mad(a, b, asfloat(g_z | 0x80000000u)); }
#ifndef LP_MUT
#define LP_MUT 0 // 1, 2, 4 ... 9: negative controls of the byte checks (never shipped)
#define LP_MUT 0 // 1..9: negative controls of the byte checks (never shipped)
#endif
#ifndef LP_DP
#define LP_DP 1
#endif
#ifndef LP_ENCFOLD
#define LP_ENCFOLD 0 // 1 (l_post_recon_p12's fold port): the history encode through the driver's conversion fold
#endif
#ifndef LP_FMANZ
#define LP_FMANZ 1
#endif
Expand Down Expand Up @@ -589,6 +593,24 @@ WG_KERNEL(l_post_recon)
if (second) g = g1;
float hh[3], oo[3];
finish_pixel(g, x, y0 + row, px, second ? py1 : py0, s, hh, oo);
#if !LP_LDR && LP_ENCFOLD
float xe[3], de[3];
[unroll] for (int ce = 0; ce < 3; ++ce) {
xe[ce] = enc_log(hh[ce]);
de[ce] = opq(clampf(enc_e(oo[ce]), 0.0f, kDisplayMax));
}
const uint4 hc = lp_encode_fold(xe[0], xe[1], xe[2], de[0], de[1], de[2], g_z, row);
// the device's start-up guard (P_pad1 = its side buffer; 0 in every frame): this pixel's FP32 conversion inputs,
// so the guard checks each stored history code against the exact one-rounding encode of its own input
[branch] if (P_pad1 != 0) {
if (and(active, or(!second, has1))) {
const uint so = (uint)((y0 + row) * P_dw + x) * 12u;
wg_st32((uint)P_pad1, so, asuint(xe[0]));
wg_st32((uint)P_pad1, so + 4u, asuint(xe[1]));
wg_st32((uint)P_pad1, so + 8u, asuint(xe[2]));
}
}
#else
v8f cv;
[unroll] for (int ci = 0; ci < 3; ++ci) {
#if LP_LDR
Expand All @@ -606,14 +628,19 @@ WG_KERNEL(l_post_recon)
cv.v[6] = 0.0f;
cv.v[7] = 0.0f;
const uint4 hc = wg_to_f16_ags(cv);
#endif
const int y = y0 + row;
if (and(active, or(!second, has1))) {
#if LP_LDR
const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu;
#else
#if LP_ENCFOLD
const uint q0 = hc.x & 0xffffu, q1 = hc.x >> 16, q2 = hc.y & 0xffffu; // the fold's +0 addend gives +0
#else
const uint q0 = mul_zero(kEncK, enc_log(hh[0])) ? 0u : (hc.x & 0xffffu);
const uint q1 = mul_zero(kEncK, enc_log(hh[1])) ? 0u : (hc.x >> 16);
const uint q2 = mul_zero(kEncK, enc_log(hh[2])) ? 0u : (hc.y & 0xffffu);
#endif
#endif
wg_st64(hist_out, (uint)(y * P_dw + x) * 8u, uint2(q0 | (q1 << 16), q2));
const int di = y * pitch + x;
Expand Down
1 change: 1 addition & 0 deletions clean/dx12/tests/kernel_replay.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,7 @@ int main(int argc, char** argv) {
auto d = make_d3d12_device(argv[1], &err);
if (!d) { std::printf("device: %s\n", err.c_str()); return 1; }
std::printf("device: %s\n", d->name().c_str());
if (!d->note().empty()) std::printf("device note: %s\n", d->note().c_str()); // e.g. the conversion-fold guard
std::ifstream f(argv[2]);
if (!f) { std::printf("cannot read %s\n", argv[2]); return 2; }
const std::string base = dir_of(argv[2]);
Expand Down
Loading
Loading