Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
}
}

#ifndef SW_SRC
#define SW_SRC 0
#endif
#ifndef SW_OUT_F16
#define SW_OUT_F16 0
#endif
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
#error "SW_SRC / SW_OUT_F16: plain blocks only"
#endif
#if SW_SRC == 1
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
// from the FP16 map [H + 4][W + 4][C]
void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
{
const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C;
[unroll] for (uint k = 0; k < KC; ++k) {
const uint4 q = wg_ld128(base, off + 16u * k);
f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y);
f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w);
}
}
#elif SW_SRC == 2
// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its
// sub-pixel; the byte offset of the lane's half of that pixel
uint sw_coarse_src(int pr, int pc, int H, int W, uint hh)
{
const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1);
return (uint)(r * W + c) * SW_C + hh * HALF;
}
#endif

// sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added
float norm_r(v8f a[KC])
{
Expand Down Expand Up @@ -677,6 +708,11 @@ WG_KERNEL(SW_ENTRY)
const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp)
const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16));
const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1));
#elif SW_SRC == 1
v8f xf[KC];
sw_load_halo_f16(x, pr, pc, W, hh, xf);
#elif SW_SRC == 2
load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa);
#elif SW_PM
// enc1's patch merge of this lane's (mirrored) token: lane half hh holds the 2 x 64 bytes of input row 2 mr + hh
// (TL | TR or BL | BR), 16 K groups, as patch_merge_c64
Expand Down Expand Up @@ -739,7 +775,11 @@ WG_KERNEL(SW_ENTRY)
v8f acc[KC];
{
[unroll] for (uint k = 0; k < KC; ++k) {
#if SW_SRC == 1
acc[k] = xf[k];
#else
acc[k] = sw_unpack_fp8(xa[k]);
#endif
#if SW_ADD
acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k]));
#endif
Expand Down Expand Up @@ -1085,6 +1125,17 @@ WG_KERNEL(SW_ENTRY)
[unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u);
#endif

#if SW_OUT_F16
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
if (dst >= 0) {
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
}
return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here
#endif

// Output q8(x2), real window members only.
uint2 o[KC];
#if SW_CLAMP_SKIP
Expand Down
11 changes: 8 additions & 3 deletions clean/dx12/kernels/l_swin.hlsl
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
// SPDX-License-Identifier: MIT
// D3D12 port of Model L's C = 32 Swin block (clean/l/kernels/l_swin.hip, l_swin_block_c32_nc of l_swin.co; issue #89):
// swin_block.hlsl instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernel's), with HIP's
// transcendental GELU chain (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit).
// D3D12 ports of Model L's C = 32 Swin blocks (clean/l/kernels/l_swin.hip, l_swin.co; issue #89): swin_block.hlsl
// instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernels'), with HIP's transcendental GELU chain
// (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit).
// l_swin_block_c32_nc FP8 map in, FP8 out, mirror border (#117)
// l_swin_block_c32_enc_nc the encoder block: FP16 halo map [H + 4][W + 4][32] in (SW_SRC 1), FP8 out
// l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16)
//
// @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc
// @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc
// @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc
#include "swin_block.hlsl"
56 changes: 56 additions & 0 deletions clean/dx12/kernels/swin_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
// SW_ADD 1: the block input is x + xadd (decoder blocks)
// SW_SAFE 1: the "_nc" kernels (inner FP8 packs without the +-448 clamp; the output pack clamps)
// SW_ECL >0: fused expand (the "_ex" kernels): x is the low-resolution map with SW_ECL channels
// SW_SRC gather source (HIP's SRC hook; plain blocks only): 0 the FP8 map at the mirrored member
// (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at
// (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid
// mirror, sub-pixel kept (LCoarseFp8Src)
// SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi)
// HIP ISA: <build>\dx12-m\isa\swin_block_c64_nc.s, swin_block_c96_nc.s, swin_block_c96_add_ex_nc.s.
// Every FP32 operation follows that ISA: the RMSNorm sums of squares pair as fma(a_even, a_even, a_odd * a_odd); the
// soft cap is fma(c^3, -1 / (3 cap^2), c), the max is over t * (tau log2 e) but the exponent is fma(t, tau log2 e, -max);
Expand Down Expand Up @@ -389,6 +394,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
}
}

#ifndef SW_SRC
#define SW_SRC 0
#endif
#ifndef SW_OUT_F16
#define SW_OUT_F16 0
#endif
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
#error "SW_SRC / SW_OUT_F16: plain blocks only"
#endif
#if SW_SRC == 1
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
// from the FP16 map [H + 4][W + 4][C]
void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
{
const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C;
[unroll] for (uint k = 0; k < KC; ++k) {
const uint4 q = wg_ld128(base, off + 16u * k);
f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y);
f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w);
}
}
#elif SW_SRC == 2
// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its
// sub-pixel; the byte offset of the lane's half of that pixel
uint sw_coarse_src(int pr, int pc, int H, int W, uint hh)
{
const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1);
return (uint)(r * W + c) * SW_C + hh * HALF;
}
#endif

// sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added
float norm_r(v8f a[KC])
{
Expand Down Expand Up @@ -539,6 +575,11 @@ WG_KERNEL(SW_ENTRY)
const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp)
const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16));
const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1));
#elif SW_SRC == 1
v8f xf[KC];
sw_load_halo_f16(x, pr, pc, W, hh, xf);
#elif SW_SRC == 2
load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa);
#else
load_frags_g(x, src, xa);
#endif
Expand Down Expand Up @@ -568,7 +609,11 @@ WG_KERNEL(SW_ENTRY)
v8f acc[KC];
{
[unroll] for (uint k = 0; k < KC; ++k) {
#if SW_SRC == 1
acc[k] = xf[k];
#else
acc[k] = sw_unpack_fp8(xa[k]);
#endif
#if SW_ADD
acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k]));
#endif
Expand Down Expand Up @@ -914,6 +959,17 @@ WG_KERNEL(SW_ENTRY)
[unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u);
#endif

#if SW_OUT_F16
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
if (dst >= 0) {
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
}
return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here
#endif

// Output q8(x2), real window members only.
uint2 o[KC];
#if SW_CLAMP_SKIP
Expand Down
2 changes: 2 additions & 0 deletions clean/dx12/vgpr_limits.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
{
"_doc": "build_dx12.py VGPR check: the RGA (gfx1201) USED_VGPRs bound of every Swin / bottleneck kernel (a name with a swin_ part). Each bound is the largest count that keeps the kernel's present occupancy: VGPR file 1536 per SIMD lane, granule 8, a 256-lane wave32 workgroup = 4 waves on each of a CU's 2 SIMDs, so 2 workgroups per CU need <= 192, 3 need <= 128 (a 128-lane workgroup = 2 waves per SIMD: 4 per CU need <= 192). A kernel over its bound fails the build (a merge or an edit lost a workgroup per CU: swin_block_c96_add_xe_* at 215 VGPRs ran 1 per CU, +38 us per frame at 1440p Q). A new Swin / bottleneck kernel needs an entry. WG_DX12_VGPR_CHECK=0 skips the check.",
"l_swin_block_c32_nc": 96,
"l_swin_block_c32_enc_nc": 96,
"l_swin_block_c32_dec_nc": 96,
"swin_block_c64_nc": 192,
"swin_block_c96_nc": 192,
"swin_block_c96_add_ex_nc": 192,
Expand Down
75 changes: 57 additions & 18 deletions clean/l/runner/l_d3d12_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,12 @@
decoded and encoded again), each output buffer filled beforehand with its own non-zero sentinel (HIP 0xA5, D3D12
0x5A), so a byte either path leaves unwritten shows up as a difference. Output: counts only, no timing.

Kernels: l_swin_block_c32_nc (the D3D12 port has only the `_nc` form, as Model M's ports).
Kernels (the D3D12 ports have only the `_nc` forms, as Model M's ports):
- l_swin_block_c32_nc: FP8 map in, FP8 out, mirror border;
- l_swin_block_c32_enc_nc: Model L's encoder block, the FP16 halo map [H + 4][W + 4][32] in, FP8 out, the "first"
window offsets (clean/l/FORMATS.md);
- l_swin_block_c32_dec_nc: Model L's decoder block, the FP8 map in with the token-grid mirror, FP16 out, the "second"
window offsets.
"""
import argparse
import ctypes
Expand All @@ -29,15 +34,20 @@
sys.path.insert(0, ROOT)

from clean.l.reference import oracle as L # noqa: E402
from clean.l.reference import table as T # noqa: E402
from clean.l.runner import l_c32_check as C32 # noqa: E402
from clean.reference.fp8 import fp8_encode, q8 # noqa: E402
from clean.runner import packing as P # noqa: E402

D3D12_KERNELS = ["l_swin_block_c32_nc", "native_fill"] # native_fill: the device's memset
L_KERNELS = ["l_swin_block_c32_nc", "l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"]
D3D12_KERNELS = L_KERNELS + ["native_fill"] # native_fill: the device's memset
# Control: Model M's swin_block_c96_nc, whose port is already proven byte for byte (clean/dx12/compare.py on whole
# frames): if it matches here too, the harness is sound.
CONTROL_KERNELS = ["swin_block_c96_nc"]
CASES = C32.CASES + [(96, 160, (2, 2)), (240, 432, (0, 2))]
# Model L's encoder / decoder blocks: grid sides multiples of 4, each with both window offsets of its kind. 4 x 8 and
# 12 x 4: fewer windows than a workgroup's 8.
L_BLOCK_GRIDS = C32.L_CASES + [(96, 160), (4, 8), (12, 4)]


def prepare(build, rocm):
Expand Down Expand Up @@ -79,18 +89,51 @@ def _params_c96(rng):
return P.pack_stage("enc2", MS.random_params("enc2", rng))[:P.stage_sections("enc2")[1]]


KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)}
KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "l_swin_block_c32_enc_nc": ("l_swin.co", 32, _params_c32),
"l_swin_block_c32_dec_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)}


def _nc(blob):
return bool(np.frombuffer(blob[:4], np.uint32)[0] & 1)


def block_cases(kernel, rng):
"""(H, W, offset, input bytes, output bytes, blob) of every case of `kernel` whose blob has the `_nc` flag (only
the _nc forms are ported); a blob without it draws no input."""
gen = KINDS[kernel][2]
if kernel in ("l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"):
enc = kernel == "l_swin_block_c32_enc_nc"
kind = "first" if enc else "second"
h = T.block_halo(T.TABLE, kind)
assert not enc or h == 2, "the ports read a 2-pixel halo"
for (H, W) in L_BLOCK_GRIDS:
for parity in (0, 1):
blob = gen(rng)
if not _nc(blob):
continue
off = T.block_offset(T.TABLE, kind, parity)
if enc:
xb = rng.normal(0, 1.5, (H + 2 * h, W + 2 * h, 32)).astype(np.float16).tobytes()
yield H, W, off, xb, H * W * 32, blob
else:
yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, 32)))).tobytes(), H * W * 64, blob
return
C = KINDS[kernel][1]
for (H, W, off) in CASES:
blob = gen(rng)
if not _nc(blob):
continue
yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, C)))).tobytes(), H * W * C, blob

# Each back end's output buffer starts filled with its own non-zero sentinel byte, different from the other's: a byte
# that one path (or both) never writes then differs between the two outputs instead of comparing equal as zeros.
HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A


def hip_block_raw(g, co, kernel, xb, blob, H, W, off, C):
"""The block on HIP; returns the output buffer's downloaded bytes as they are (never decoded and encoded again, so
for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL."""
def hip_block_raw(g, co, kernel, xb, blob, H, W, off, n):
"""The block on HIP; returns the output buffer's `n` downloaded bytes as they are (never decoded and encoded again,
so for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL."""
h = g.hip
n = H * W * C
px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(n)
try:
h.upload(px, xb)
Expand All @@ -115,30 +158,25 @@ def run(build, rocm, seeds, kernels):
lines, hip_out = [], {}
neg = None # (case name, kernel) of the negative control
for kernel in kernels:
co, C, gen = KINDS[kernel]
co = KINDS[kernel][0]
for si, seed in enumerate(seeds):
rng = np.random.default_rng(seed)
for ci, (H, W, off) in enumerate(CASES):
for ci, (H, W, off, xb, nout, blob) in enumerate(block_cases(kernel, rng)):
name = f"{kernel}_s{si}_{ci}"
blob = gen(rng)
if not np.frombuffer(blob[:4], np.uint32)[0] & 1:
continue # only the _nc forms are ported
x = q8(rng.normal(0, 1.5, (H, W, C)))
xb = fp8_encode(x).tobytes()
hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, C))
hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, nout))
_write(os.path.join(cdir, name + ".x"), xb)
_write(os.path.join(cdir, name + ".w"), blob)
ox, oy = off
nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4)
lines += [f"case {name}", f"module {co}", f"kernel {kernel} {-(-nwin // 8)} 256",
f"arg buf {name}.x", "arg null", f"arg out {H * W * C} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""]
if neg is None and nwin > 8:
# Negative control: this case again on D3D12 with half its workgroups. The windows never processed
# keep the D3D12 sentinel, so the comparison must find them: the check can fail.
neg = (name, kernel)
lines += [f"case negative_{name}", f"module {co}", f"kernel {kernel} {max(1, -(-nwin // 8) // 2)} 256",
f"arg buf {name}.x", "arg null", f"arg out {H * W * C} negative_{name}.out {D3D12_SENTINEL}",
f"arg buf {name}.x", "arg null", f"arg out {nout} negative_{name}.out {D3D12_SENTINEL}",
f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""]
with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f:
f.write("\n".join(lines))
Expand Down Expand Up @@ -184,14 +222,15 @@ def main(argv=None):
ap.add_argument("--prepare", action="store_true")
ap.add_argument("--run", action="store_true")
ap.add_argument("--seeds", type=int, default=2)
ap.add_argument("--kernels", default=",".join(L_KERNELS))
ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control")
ap.add_argument("--json")
a = ap.parse_args(argv)
os.makedirs(a.build, exist_ok=True)
if a.prepare:
print("built", prepare(a.build, a.rocm))
if a.run:
kernels = ["l_swin_block_c32_nc"] + ([] if a.no_control else CONTROL_KERNELS)
kernels = [k for k in a.kernels.split(",") if k] + ([] if a.no_control else CONTROL_KERNELS)
res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels)
if a.json:
with open(a.json, "w") as f:
Expand Down
Loading