diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 0576790d..8527eba0 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -441,6 +441,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) } } +#ifndef SW_SRC +#define SW_SRC 0 +#endif +#ifndef SW_OUT_F16 +#define SW_OUT_F16 0 +#endif +#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) +#error "SW_SRC / SW_OUT_F16: plain blocks only" +#endif +#if SW_SRC == 1 +// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo), +// from the FP16 map [H + 4][W + 4][C] +void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) +{ + const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C; + [unroll] for (uint k = 0; k < KC; ++k) { + const uint4 q = wg_ld128(base, off + 16u * k); + f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y); + f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w); + } +} +#elif SW_SRC == 2 +// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its +// sub-pixel; the byte offset of the lane's half of that pixel +uint sw_coarse_src(int pr, int pc, int H, int W, uint hh) +{ + const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1); + return (uint)(r * W + c) * SW_C + hh * HALF; +} +#endif + // sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added float norm_r(v8f a[KC]) { @@ -677,6 +708,11 @@ WG_KERNEL(SW_ENTRY) const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp) const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16)); const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1)); +#elif SW_SRC == 1 + v8f xf[KC]; + sw_load_halo_f16(x, pr, pc, W, hh, xf); +#elif SW_SRC == 2 + load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #elif SW_PM // enc1's patch merge of this lane's (mirrored) token: lane half hh holds the 2 x 64 bytes of input row 2 mr + hh // (TL | TR or BL | BR), 16 K groups, as patch_merge_c64 @@ -739,7 +775,11 @@ WG_KERNEL(SW_ENTRY) v8f acc[KC]; { [unroll] for (uint k = 0; k < KC; ++k) { +#if SW_SRC == 1 + acc[k] = xf[k]; +#else acc[k] = sw_unpack_fp8(xa[k]); +#endif #if SW_ADD acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k])); #endif @@ -1085,6 +1125,17 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u); #endif +#if SW_OUT_F16 + // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). + uint4 o16[KC]; + [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); + if (dst >= 0) { + const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u; + [unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]); + } + return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here +#endif + // Output q8(x2), real window members only. uint2 o[KC]; #if SW_CLAMP_SKIP diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index d4e0f8e2..660b6aed 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -1,7 +1,12 @@ // SPDX-License-Identifier: MIT -// D3D12 port of Model L's C = 32 Swin block (clean/l/kernels/l_swin.hip, l_swin_block_c32_nc of l_swin.co; issue #89): -// swin_block.hlsl instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernel's), with HIP's -// transcendental GELU chain (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit). +// D3D12 ports of Model L's C = 32 Swin blocks (clean/l/kernels/l_swin.hip, l_swin.co; issue #89): swin_block.hlsl +// instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernels'), with HIP's transcendental GELU chain +// (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit). +// l_swin_block_c32_nc FP8 map in, FP8 out, mirror border (#117) +// l_swin_block_c32_enc_nc the encoder block: FP16 halo map [H + 4][W + 4][32] in (SW_SRC 1), FP8 out +// l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16) // // @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc +// @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc +// @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc #include "swin_block.hlsl" diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index 5f8e6a01..cf79de3b 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -7,6 +7,11 @@ // SW_ADD 1: the block input is x + xadd (decoder blocks) // SW_SAFE 1: the "_nc" kernels (inner FP8 packs without the +-448 clamp; the output pack clamps) // SW_ECL >0: fused expand (the "_ex" kernels): x is the low-resolution map with SW_ECL channels +// SW_SRC gather source (HIP's SRC hook; plain blocks only): 0 the FP8 map at the mirrored member +// (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at +// (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid +// mirror, sub-pixel kept (LCoarseFp8Src) +// SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi) // HIP ISA: \dx12-m\isa\swin_block_c64_nc.s, swin_block_c96_nc.s, swin_block_c96_add_ex_nc.s. // Every FP32 operation follows that ISA: the RMSNorm sums of squares pair as fma(a_even, a_even, a_odd * a_odd); the // soft cap is fma(c^3, -1 / (3 cap^2), c), the max is over t * (tau log2 e) but the exponent is fma(t, tau log2 e, -max); @@ -389,6 +394,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC]) } } +#ifndef SW_SRC +#define SW_SRC 0 +#endif +#ifndef SW_OUT_F16 +#define SW_OUT_F16 0 +#endif +#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0) +#error "SW_SRC / SW_OUT_F16: plain blocks only" +#endif +#if SW_SRC == 1 +// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo), +// from the FP16 map [H + 4][W + 4][C] +void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) +{ + const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C; + [unroll] for (uint k = 0; k < KC; ++k) { + const uint4 q = wg_ld128(base, off + 16u * k); + f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y); + f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w); + } +} +#elif SW_SRC == 2 +// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its +// sub-pixel; the byte offset of the lane's half of that pixel +uint sw_coarse_src(int pr, int pc, int H, int W, uint hh) +{ + const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1); + return (uint)(r * W + c) * SW_C + hh * HALF; +} +#endif + // sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added float norm_r(v8f a[KC]) { @@ -539,6 +575,11 @@ WG_KERNEL(SW_ENTRY) const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp) const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16)); const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1)); +#elif SW_SRC == 1 + v8f xf[KC]; + sw_load_halo_f16(x, pr, pc, W, hh, xf); +#elif SW_SRC == 2 + load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa); #else load_frags_g(x, src, xa); #endif @@ -568,7 +609,11 @@ WG_KERNEL(SW_ENTRY) v8f acc[KC]; { [unroll] for (uint k = 0; k < KC; ++k) { +#if SW_SRC == 1 + acc[k] = xf[k]; +#else acc[k] = sw_unpack_fp8(xa[k]); +#endif #if SW_ADD acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k])); #endif @@ -914,6 +959,17 @@ WG_KERNEL(SW_ENTRY) [unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u); #endif +#if SW_OUT_F16 + // Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi). + uint4 o16[KC]; + [unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]); + if (dst >= 0) { + const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u; + [unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]); + } + return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here +#endif + // Output q8(x2), real window members only. uint2 o[KC]; #if SW_CLAMP_SKIP diff --git a/clean/dx12/vgpr_limits.json b/clean/dx12/vgpr_limits.json index d4251e04..c707c4b4 100644 --- a/clean/dx12/vgpr_limits.json +++ b/clean/dx12/vgpr_limits.json @@ -1,6 +1,8 @@ { "_doc": "build_dx12.py VGPR check: the RGA (gfx1201) USED_VGPRs bound of every Swin / bottleneck kernel (a name with a swin_ part). Each bound is the largest count that keeps the kernel's present occupancy: VGPR file 1536 per SIMD lane, granule 8, a 256-lane wave32 workgroup = 4 waves on each of a CU's 2 SIMDs, so 2 workgroups per CU need <= 192, 3 need <= 128 (a 128-lane workgroup = 2 waves per SIMD: 4 per CU need <= 192). A kernel over its bound fails the build (a merge or an edit lost a workgroup per CU: swin_block_c96_add_xe_* at 215 VGPRs ran 1 per CU, +38 us per frame at 1440p Q). A new Swin / bottleneck kernel needs an entry. WG_DX12_VGPR_CHECK=0 skips the check.", "l_swin_block_c32_nc": 96, + "l_swin_block_c32_enc_nc": 96, + "l_swin_block_c32_dec_nc": 96, "swin_block_c64_nc": 192, "swin_block_c96_nc": 192, "swin_block_c96_add_ex_nc": 192, diff --git a/clean/l/runner/l_d3d12_check.py b/clean/l/runner/l_d3d12_check.py index fee4f85d..e004d1c5 100644 --- a/clean/l/runner/l_d3d12_check.py +++ b/clean/l/runner/l_d3d12_check.py @@ -13,7 +13,12 @@ decoded and encoded again), each output buffer filled beforehand with its own non-zero sentinel (HIP 0xA5, D3D12 0x5A), so a byte either path leaves unwritten shows up as a difference. Output: counts only, no timing. -Kernels: l_swin_block_c32_nc (the D3D12 port has only the `_nc` form, as Model M's ports). +Kernels (the D3D12 ports have only the `_nc` forms, as Model M's ports): +- l_swin_block_c32_nc: FP8 map in, FP8 out, mirror border; +- l_swin_block_c32_enc_nc: Model L's encoder block, the FP16 halo map [H + 4][W + 4][32] in, FP8 out, the "first" + window offsets (clean/l/FORMATS.md); +- l_swin_block_c32_dec_nc: Model L's decoder block, the FP8 map in with the token-grid mirror, FP16 out, the "second" + window offsets. """ import argparse import ctypes @@ -29,15 +34,20 @@ sys.path.insert(0, ROOT) from clean.l.reference import oracle as L # noqa: E402 +from clean.l.reference import table as T # noqa: E402 from clean.l.runner import l_c32_check as C32 # noqa: E402 from clean.reference.fp8 import fp8_encode, q8 # noqa: E402 from clean.runner import packing as P # noqa: E402 -D3D12_KERNELS = ["l_swin_block_c32_nc", "native_fill"] # native_fill: the device's memset +L_KERNELS = ["l_swin_block_c32_nc", "l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"] +D3D12_KERNELS = L_KERNELS + ["native_fill"] # native_fill: the device's memset # Control: Model M's swin_block_c96_nc, whose port is already proven byte for byte (clean/dx12/compare.py on whole # frames): if it matches here too, the harness is sound. CONTROL_KERNELS = ["swin_block_c96_nc"] CASES = C32.CASES + [(96, 160, (2, 2)), (240, 432, (0, 2))] +# Model L's encoder / decoder blocks: grid sides multiples of 4, each with both window offsets of its kind. 4 x 8 and +# 12 x 4: fewer windows than a workgroup's 8. +L_BLOCK_GRIDS = C32.L_CASES + [(96, 160), (4, 8), (12, 4)] def prepare(build, rocm): @@ -79,18 +89,51 @@ def _params_c96(rng): return P.pack_stage("enc2", MS.random_params("enc2", rng))[:P.stage_sections("enc2")[1]] -KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)} +KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "l_swin_block_c32_enc_nc": ("l_swin.co", 32, _params_c32), + "l_swin_block_c32_dec_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)} + + +def _nc(blob): + return bool(np.frombuffer(blob[:4], np.uint32)[0] & 1) + + +def block_cases(kernel, rng): + """(H, W, offset, input bytes, output bytes, blob) of every case of `kernel` whose blob has the `_nc` flag (only + the _nc forms are ported); a blob without it draws no input.""" + gen = KINDS[kernel][2] + if kernel in ("l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"): + enc = kernel == "l_swin_block_c32_enc_nc" + kind = "first" if enc else "second" + h = T.block_halo(T.TABLE, kind) + assert not enc or h == 2, "the ports read a 2-pixel halo" + for (H, W) in L_BLOCK_GRIDS: + for parity in (0, 1): + blob = gen(rng) + if not _nc(blob): + continue + off = T.block_offset(T.TABLE, kind, parity) + if enc: + xb = rng.normal(0, 1.5, (H + 2 * h, W + 2 * h, 32)).astype(np.float16).tobytes() + yield H, W, off, xb, H * W * 32, blob + else: + yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, 32)))).tobytes(), H * W * 64, blob + return + C = KINDS[kernel][1] + for (H, W, off) in CASES: + blob = gen(rng) + if not _nc(blob): + continue + yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, C)))).tobytes(), H * W * C, blob # Each back end's output buffer starts filled with its own non-zero sentinel byte, different from the other's: a byte # that one path (or both) never writes then differs between the two outputs instead of comparing equal as zeros. HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A -def hip_block_raw(g, co, kernel, xb, blob, H, W, off, C): - """The block on HIP; returns the output buffer's downloaded bytes as they are (never decoded and encoded again, so - for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL.""" +def hip_block_raw(g, co, kernel, xb, blob, H, W, off, n): + """The block on HIP; returns the output buffer's `n` downloaded bytes as they are (never decoded and encoded again, + so for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL.""" h = g.hip - n = H * W * C px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(n) try: h.upload(px, xb) @@ -115,30 +158,25 @@ def run(build, rocm, seeds, kernels): lines, hip_out = [], {} neg = None # (case name, kernel) of the negative control for kernel in kernels: - co, C, gen = KINDS[kernel] + co = KINDS[kernel][0] for si, seed in enumerate(seeds): rng = np.random.default_rng(seed) - for ci, (H, W, off) in enumerate(CASES): + for ci, (H, W, off, xb, nout, blob) in enumerate(block_cases(kernel, rng)): name = f"{kernel}_s{si}_{ci}" - blob = gen(rng) - if not np.frombuffer(blob[:4], np.uint32)[0] & 1: - continue # only the _nc forms are ported - x = q8(rng.normal(0, 1.5, (H, W, C))) - xb = fp8_encode(x).tobytes() - hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, C)) + hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, nout)) _write(os.path.join(cdir, name + ".x"), xb) _write(os.path.join(cdir, name + ".w"), blob) ox, oy = off nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4) lines += [f"case {name}", f"module {co}", f"kernel {kernel} {-(-nwin // 8)} 256", - f"arg buf {name}.x", "arg null", f"arg out {H * W * C} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", + f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""] if neg is None and nwin > 8: # Negative control: this case again on D3D12 with half its workgroups. The windows never processed # keep the D3D12 sentinel, so the comparison must find them: the check can fail. neg = (name, kernel) lines += [f"case negative_{name}", f"module {co}", f"kernel {kernel} {max(1, -(-nwin // 8) // 2)} 256", - f"arg buf {name}.x", "arg null", f"arg out {H * W * C} negative_{name}.out {D3D12_SENTINEL}", + f"arg buf {name}.x", "arg null", f"arg out {nout} negative_{name}.out {D3D12_SENTINEL}", f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""] with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f: f.write("\n".join(lines)) @@ -184,6 +222,7 @@ def main(argv=None): ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) + ap.add_argument("--kernels", default=",".join(L_KERNELS)) ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control") ap.add_argument("--json") a = ap.parse_args(argv) @@ -191,7 +230,7 @@ def main(argv=None): if a.prepare: print("built", prepare(a.build, a.rocm)) if a.run: - kernels = ["l_swin_block_c32_nc"] + ([] if a.no_control else CONTROL_KERNELS) + kernels = [k for k in a.kernels.split(",") if k] + ([] if a.no_control else CONTROL_KERNELS) res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels) if a.json: with open(a.json, "w") as f: