Skip to content

Commit 37e6771

Browse files
Merge remote-tracking branch 'origin/main' into model-l/l-post-recon-d3d12
Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> # Conflicts: # clean/l/runner/l_d3d12_ops_check.py
2 parents 6008672 + 26eb21e commit 37e6771

7 files changed

Lines changed: 196 additions & 26 deletions

File tree

‎clean/dx12/kernels/fused_swin.hlsl‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
441441
}
442442
}
443443

444+
#ifndef SW_SRC
445+
#define SW_SRC 0
446+
#endif
447+
#ifndef SW_OUT_F16
448+
#define SW_OUT_F16 0
449+
#endif
450+
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
451+
#error "SW_SRC / SW_OUT_F16: plain blocks only"
452+
#endif
453+
#if SW_SRC == 1
454+
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
455+
// from the FP16 map [H + 4][W + 4][C]
456+
void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
457+
{
458+
const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C;
459+
[unroll] for (uint k = 0; k < KC; ++k) {
460+
const uint4 q = wg_ld128(base, off + 16u * k);
461+
f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y);
462+
f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w);
463+
}
464+
}
465+
#elif SW_SRC == 2
466+
// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its
467+
// sub-pixel; the byte offset of the lane's half of that pixel
468+
uint sw_coarse_src(int pr, int pc, int H, int W, uint hh)
469+
{
470+
const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1);
471+
return (uint)(r * W + c) * SW_C + hh * HALF;
472+
}
473+
#endif
474+
444475
// sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added
445476
float norm_r(v8f a[KC])
446477
{
@@ -677,6 +708,11 @@ WG_KERNEL(SW_ENTRY)
677708
const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp)
678709
const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16));
679710
const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1));
711+
#elif SW_SRC == 1
712+
v8f xf[KC];
713+
sw_load_halo_f16(x, pr, pc, W, hh, xf);
714+
#elif SW_SRC == 2
715+
load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa);
680716
#elif SW_PM
681717
// enc1's patch merge of this lane's (mirrored) token: lane half hh holds the 2 x 64 bytes of input row 2 mr + hh
682718
// (TL | TR or BL | BR), 16 K groups, as patch_merge_c64
@@ -739,7 +775,11 @@ WG_KERNEL(SW_ENTRY)
739775
v8f acc[KC];
740776
{
741777
[unroll] for (uint k = 0; k < KC; ++k) {
778+
#if SW_SRC == 1
779+
acc[k] = xf[k];
780+
#else
742781
acc[k] = sw_unpack_fp8(xa[k]);
782+
#endif
743783
#if SW_ADD
744784
acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k]));
745785
#endif
@@ -1085,6 +1125,17 @@ WG_KERNEL(SW_ENTRY)
10851125
[unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u);
10861126
#endif
10871127

1128+
#if SW_OUT_F16
1129+
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
1130+
uint4 o16[KC];
1131+
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
1132+
if (dst >= 0) {
1133+
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
1134+
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
1135+
}
1136+
return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here
1137+
#endif
1138+
10881139
// Output q8(x2), real window members only.
10891140
uint2 o[KC];
10901141
#if SW_CLAMP_SKIP

‎clean/dx12/kernels/l_ops_coef.hlsl‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,12 @@
1717
// The three 32-bit stores at the same per-lane offset into different buffers (A, B, latent) are the shape the driver was
1818
// seen to merge (README "drop a store"): the pointers are read through wg_uniform().
1919
//
20-
// @wg-kernel l_coef co=l_ops.co block=256 wave=32 args=p,p,p,p,p,p,b56
20+
// COEF_PACK12 (kernel l_coef_pack12, HIP l_coef_body<true>): the same values in Model M's dec0 "pack12" layout, read by
21+
// the output stage's _p12 kernels: image A, 8 bytes per pixel = A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per
22+
// pixel = C0 C1 (FP16); B unused (may be null); the latent as l_coef.
23+
//
24+
// @wg-kernel l_coef co=l_ops.co block=256 wave=32 args=p,p,p,p,p,p,b56 defines=COEF_PACK12=0
25+
// @wg-kernel l_coef_pack12 co=l_ops.co block=256 wave=32 args=p,p,p,p,p,p,b56 defines=COEF_PACK12=1
2126
#include "wg_platform.hlsli"
2227

2328
WG_ARGS_BEGIN
@@ -54,7 +59,11 @@ uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f)
5459
// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v)
5560
float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); }
5661

62+
#if COEF_PACK12
63+
WG_KERNEL(l_coef_pack12)
64+
#else
5765
WG_KERNEL(l_coef)
66+
#endif
5867
{
5968
WG_SETUP();
6069
g_z = wg_grid_ >> 31;
@@ -99,9 +108,14 @@ WG_KERNEL(l_coef)
99108
const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) |
100109
(snorm8_code(l_tanh(a12)) << 24);
101110
const uint i = (uint)(y * P_rw + x);
111+
#if COEF_PACK12
112+
wg_st64(pA, i * 8u, uint2(cA | (cB << 24), (cB >> 8) | (c1 << 16)));
113+
wg_st32(pC, i * 4u, c0);
114+
#else
102115
wg_st32(pA, i * 4u, cA);
103116
wg_st32(pB, i * 4u, cB);
104117
wg_st64(pC, i * 8u, uint2(c0, c1));
118+
#endif
105119
wg_st32(pL, i * 4u, cL);
106120
}
107121
}

‎clean/dx12/kernels/l_swin.hlsl‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
// SPDX-License-Identifier: MIT
2-
// D3D12 port of Model L's C = 32 Swin block (clean/l/kernels/l_swin.hip, l_swin_block_c32_nc of l_swin.co; issue #89):
3-
// swin_block.hlsl instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernel's), with HIP's
4-
// transcendental GELU chain (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit).
2+
// D3D12 ports of Model L's C = 32 Swin blocks (clean/l/kernels/l_swin.hip, l_swin.co; issue #89): swin_block.hlsl
3+
// instantiated at C = 32, 2 heads, MLP 128 (the same template as the HIP kernels'), with HIP's transcendental GELU chain
4+
// (SW_GELU 0, no table) and HIP's MLP unit cut (HPU 8 = all 8 hidden tiles in one unit).
5+
// l_swin_block_c32_nc FP8 map in, FP8 out, mirror border (#117)
6+
// l_swin_block_c32_enc_nc the encoder block: FP16 halo map [H + 4][W + 4][32] in (SW_SRC 1), FP8 out
7+
// l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16)
58
//
69
// @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc
10+
// @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc
11+
// @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc
712
#include "swin_block.hlsl"

‎clean/dx12/kernels/swin_block.hlsl‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
// SW_ADD 1: the block input is x + xadd (decoder blocks)
88
// SW_SAFE 1: the "_nc" kernels (inner FP8 packs without the +-448 clamp; the output pack clamps)
99
// SW_ECL >0: fused expand (the "_ex" kernels): x is the low-resolution map with SW_ECL channels
10+
// SW_SRC gather source (HIP's SRC hook; plain blocks only): 0 the FP8 map at the mirrored member
11+
// (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at
12+
// (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid
13+
// mirror, sub-pixel kept (LCoarseFp8Src)
14+
// SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi)
1015
// HIP ISA: <build>\dx12-m\isa\swin_block_c64_nc.s, swin_block_c96_nc.s, swin_block_c96_add_ex_nc.s.
1116
// Every FP32 operation follows that ISA: the RMSNorm sums of squares pair as fma(a_even, a_even, a_odd * a_odd); the
1217
// soft cap is fma(c^3, -1 / (3 cap^2), c), the max is over t * (tau log2 e) but the exponent is fma(t, tau log2 e, -max);
@@ -389,6 +394,37 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
389394
}
390395
}
391396

397+
#ifndef SW_SRC
398+
#define SW_SRC 0
399+
#endif
400+
#ifndef SW_OUT_F16
401+
#define SW_OUT_F16 0
402+
#endif
403+
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
404+
#error "SW_SRC / SW_OUT_F16: plain blocks only"
405+
#endif
406+
#if SW_SRC == 1
407+
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
408+
// from the FP16 map [H + 4][W + 4][C]
409+
void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
410+
{
411+
const uint off = (uint)(((pr + 2) * (W + 4) + pc + 2) * SW_C) * 2u + hh * SW_C;
412+
[unroll] for (uint k = 0; k < KC; ++k) {
413+
const uint4 q = wg_ld128(base, off + 16u * k);
414+
f[k].v[0] = wg_h2f_lo(q.x); f[k].v[1] = wg_h2f_hi(q.x); f[k].v[2] = wg_h2f_lo(q.y); f[k].v[3] = wg_h2f_hi(q.y);
415+
f[k].v[4] = wg_h2f_lo(q.z); f[k].v[5] = wg_h2f_hi(q.z); f[k].v[6] = wg_h2f_lo(q.w); f[k].v[7] = wg_h2f_hi(q.w);
416+
}
417+
}
418+
#elif SW_SRC == 2
419+
// LCoarseFp8Src: a member outside the grid takes the mirrored 2 x 2 group (mirror of the group index) and keeps its
420+
// sub-pixel; the byte offset of the lane's half of that pixel
421+
uint sw_coarse_src(int pr, int pc, int H, int W, uint hh)
422+
{
423+
const int r = 2 * sw_mirror(pr >> 1, H >> 1) + (pr & 1), c = 2 * sw_mirror(pc >> 1, W >> 1) + (pc & 1);
424+
return (uint)(r * W + c) * SW_C + hh * HALF;
425+
}
426+
#endif
427+
392428
// sum of squares of a token's channels in this lane (k = 0 .. KC - 1, in order), the other half of the wave added
393429
float norm_r(v8f a[KC])
394430
{
@@ -539,6 +575,11 @@ WG_KERNEL(SW_ENTRY)
539575
const uint lpw = WaveReadLaneAt(lowp, wave_id); // this wave's window (win: the same clamp)
540576
const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16));
541577
const uint eslot = (tid >> 5) * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1));
578+
#elif SW_SRC == 1
579+
v8f xf[KC];
580+
sw_load_halo_f16(x, pr, pc, W, hh, xf);
581+
#elif SW_SRC == 2
582+
load_frags_g(x, sw_coarse_src(pr, pc, H, W, hh), xa);
542583
#else
543584
load_frags_g(x, src, xa);
544585
#endif
@@ -568,7 +609,11 @@ WG_KERNEL(SW_ENTRY)
568609
v8f acc[KC];
569610
{
570611
[unroll] for (uint k = 0; k < KC; ++k) {
612+
#if SW_SRC == 1
613+
acc[k] = xf[k];
614+
#else
571615
acc[k] = sw_unpack_fp8(xa[k]);
616+
#endif
572617
#if SW_ADD
573618
acc[k] = sw_add8(acc[k], sw_unpack_fp8(xb[k]));
574619
#endif
@@ -914,6 +959,17 @@ WG_KERNEL(SW_ENTRY)
914959
[unroll] for (uint rk2 = 0; rk2 < KC; ++rk2) acc[rk2] = sw_unperm8(racc[rk2], 7u, 7u);
915960
#endif
916961

962+
#if SW_OUT_F16
963+
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
964+
uint4 o16[KC];
965+
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
966+
if (dst >= 0) {
967+
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
968+
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
969+
}
970+
return; // the q8 output below stays the file's last block (fused_swin_gen.py's anchor); not compiled here
971+
#endif
972+
917973
// Output q8(x2), real window members only.
918974
uint2 o[KC];
919975
#if SW_CLAMP_SKIP

‎clean/dx12/vgpr_limits.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
{
22
"_doc": "build_dx12.py VGPR check: the RGA (gfx1201) USED_VGPRs bound of every Swin / bottleneck kernel (a name with a swin_ part). Each bound is the largest count that keeps the kernel's present occupancy: VGPR file 1536 per SIMD lane, granule 8, a 256-lane wave32 workgroup = 4 waves on each of a CU's 2 SIMDs, so 2 workgroups per CU need <= 192, 3 need <= 128 (a 128-lane workgroup = 2 waves per SIMD: 4 per CU need <= 192). A kernel over its bound fails the build (a merge or an edit lost a workgroup per CU: swin_block_c96_add_xe_* at 215 VGPRs ran 1 per CU, +38 us per frame at 1440p Q). A new Swin / bottleneck kernel needs an entry. WG_DX12_VGPR_CHECK=0 skips the check.",
33
"l_swin_block_c32_nc": 96,
4+
"l_swin_block_c32_enc_nc": 96,
5+
"l_swin_block_c32_dec_nc": 96,
46
"swin_block_c64_nc": 192,
57
"swin_block_c96_nc": 192,
68
"swin_block_c96_add_ex_nc": 192,

‎clean/l/runner/l_d3d12_check.py‎

Lines changed: 57 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,12 @@
1313
decoded and encoded again), each output buffer filled beforehand with its own non-zero sentinel (HIP 0xA5, D3D12
1414
0x5A), so a byte either path leaves unwritten shows up as a difference. Output: counts only, no timing.
1515
16-
Kernels: l_swin_block_c32_nc (the D3D12 port has only the `_nc` form, as Model M's ports).
16+
Kernels (the D3D12 ports have only the `_nc` forms, as Model M's ports):
17+
- l_swin_block_c32_nc: FP8 map in, FP8 out, mirror border;
18+
- l_swin_block_c32_enc_nc: Model L's encoder block, the FP16 halo map [H + 4][W + 4][32] in, FP8 out, the "first"
19+
window offsets (clean/l/FORMATS.md);
20+
- l_swin_block_c32_dec_nc: Model L's decoder block, the FP8 map in with the token-grid mirror, FP16 out, the "second"
21+
window offsets.
1722
"""
1823
import argparse
1924
import ctypes
@@ -29,15 +34,20 @@
2934
sys.path.insert(0, ROOT)
3035

3136
from clean.l.reference import oracle as L # noqa: E402
37+
from clean.l.reference import table as T # noqa: E402
3238
from clean.l.runner import l_c32_check as C32 # noqa: E402
3339
from clean.reference.fp8 import fp8_encode, q8 # noqa: E402
3440
from clean.runner import packing as P # noqa: E402
3541

36-
D3D12_KERNELS = ["l_swin_block_c32_nc", "native_fill"] # native_fill: the device's memset
42+
L_KERNELS = ["l_swin_block_c32_nc", "l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"]
43+
D3D12_KERNELS = L_KERNELS + ["native_fill"] # native_fill: the device's memset
3744
# Control: Model M's swin_block_c96_nc, whose port is already proven byte for byte (clean/dx12/compare.py on whole
3845
# frames): if it matches here too, the harness is sound.
3946
CONTROL_KERNELS = ["swin_block_c96_nc"]
4047
CASES = C32.CASES + [(96, 160, (2, 2)), (240, 432, (0, 2))]
48+
# Model L's encoder / decoder blocks: grid sides multiples of 4, each with both window offsets of its kind. 4 x 8 and
49+
# 12 x 4: fewer windows than a workgroup's 8.
50+
L_BLOCK_GRIDS = C32.L_CASES + [(96, 160), (4, 8), (12, 4)]
4151

4252

4353
def prepare(build, rocm):
@@ -79,18 +89,51 @@ def _params_c96(rng):
7989
return P.pack_stage("enc2", MS.random_params("enc2", rng))[:P.stage_sections("enc2")[1]]
8090

8191

82-
KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)}
92+
KINDS = {"l_swin_block_c32_nc": ("l_swin.co", 32, _params_c32), "l_swin_block_c32_enc_nc": ("l_swin.co", 32, _params_c32),
93+
"l_swin_block_c32_dec_nc": ("l_swin.co", 32, _params_c32), "swin_block_c96_nc": ("swin.co", 96, _params_c96)}
94+
95+
96+
def _nc(blob):
97+
return bool(np.frombuffer(blob[:4], np.uint32)[0] & 1)
98+
99+
100+
def block_cases(kernel, rng):
101+
"""(H, W, offset, input bytes, output bytes, blob) of every case of `kernel` whose blob has the `_nc` flag (only
102+
the _nc forms are ported); a blob without it draws no input."""
103+
gen = KINDS[kernel][2]
104+
if kernel in ("l_swin_block_c32_enc_nc", "l_swin_block_c32_dec_nc"):
105+
enc = kernel == "l_swin_block_c32_enc_nc"
106+
kind = "first" if enc else "second"
107+
h = T.block_halo(T.TABLE, kind)
108+
assert not enc or h == 2, "the ports read a 2-pixel halo"
109+
for (H, W) in L_BLOCK_GRIDS:
110+
for parity in (0, 1):
111+
blob = gen(rng)
112+
if not _nc(blob):
113+
continue
114+
off = T.block_offset(T.TABLE, kind, parity)
115+
if enc:
116+
xb = rng.normal(0, 1.5, (H + 2 * h, W + 2 * h, 32)).astype(np.float16).tobytes()
117+
yield H, W, off, xb, H * W * 32, blob
118+
else:
119+
yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, 32)))).tobytes(), H * W * 64, blob
120+
return
121+
C = KINDS[kernel][1]
122+
for (H, W, off) in CASES:
123+
blob = gen(rng)
124+
if not _nc(blob):
125+
continue
126+
yield H, W, off, fp8_encode(q8(rng.normal(0, 1.5, (H, W, C)))).tobytes(), H * W * C, blob
83127

84128
# Each back end's output buffer starts filled with its own non-zero sentinel byte, different from the other's: a byte
85129
# that one path (or both) never writes then differs between the two outputs instead of comparing equal as zeros.
86130
HIP_SENTINEL, D3D12_SENTINEL = 0xA5, 0x5A
87131

88132

89-
def hip_block_raw(g, co, kernel, xb, blob, H, W, off, C):
90-
"""The block on HIP; returns the output buffer's downloaded bytes as they are (never decoded and encoded again, so
91-
for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL."""
133+
def hip_block_raw(g, co, kernel, xb, blob, H, W, off, n):
134+
"""The block on HIP; returns the output buffer's `n` downloaded bytes as they are (never decoded and encoded again,
135+
so for example the two FP8 NaN codes stay distinct). The output starts filled with HIP_SENTINEL."""
92136
h = g.hip
93-
n = H * W * C
94137
px, pw, po = h.malloc(len(xb)), h.malloc(len(blob)), h.malloc(n)
95138
try:
96139
h.upload(px, xb)
@@ -115,30 +158,25 @@ def run(build, rocm, seeds, kernels):
115158
lines, hip_out = [], {}
116159
neg = None # (case name, kernel) of the negative control
117160
for kernel in kernels:
118-
co, C, gen = KINDS[kernel]
161+
co = KINDS[kernel][0]
119162
for si, seed in enumerate(seeds):
120163
rng = np.random.default_rng(seed)
121-
for ci, (H, W, off) in enumerate(CASES):
164+
for ci, (H, W, off, xb, nout, blob) in enumerate(block_cases(kernel, rng)):
122165
name = f"{kernel}_s{si}_{ci}"
123-
blob = gen(rng)
124-
if not np.frombuffer(blob[:4], np.uint32)[0] & 1:
125-
continue # only the _nc forms are ported
126-
x = q8(rng.normal(0, 1.5, (H, W, C)))
127-
xb = fp8_encode(x).tobytes()
128-
hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, C))
166+
hip_out[name] = (kernel, hip_block_raw(g, co, kernel, xb, blob, H, W, off, nout))
129167
_write(os.path.join(cdir, name + ".x"), xb)
130168
_write(os.path.join(cdir, name + ".w"), blob)
131169
ox, oy = off
132170
nwin = -(-(W + ox) // 4) * -(-(H + oy) // 4)
133171
lines += [f"case {name}", f"module {co}", f"kernel {kernel} {-(-nwin // 8)} 256",
134-
f"arg buf {name}.x", "arg null", f"arg out {H * W * C} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
172+
f"arg buf {name}.x", "arg null", f"arg out {nout} {name}.out {D3D12_SENTINEL}", f"arg buf {name}.w",
135173
f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""]
136174
if neg is None and nwin > 8:
137175
# Negative control: this case again on D3D12 with half its workgroups. The windows never processed
138176
# keep the D3D12 sentinel, so the comparison must find them: the check can fail.
139177
neg = (name, kernel)
140178
lines += [f"case negative_{name}", f"module {co}", f"kernel {kernel} {max(1, -(-nwin // 8) // 2)} 256",
141-
f"arg buf {name}.x", "arg null", f"arg out {H * W * C} negative_{name}.out {D3D12_SENTINEL}",
179+
f"arg buf {name}.x", "arg null", f"arg out {nout} negative_{name}.out {D3D12_SENTINEL}",
142180
f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {ox}", f"arg u32 {oy}", ""]
143181
with open(os.path.join(cdir, "cases.txt"), "w", newline="\n") as f:
144182
f.write("\n".join(lines))
@@ -184,14 +222,15 @@ def main(argv=None):
184222
ap.add_argument("--prepare", action="store_true")
185223
ap.add_argument("--run", action="store_true")
186224
ap.add_argument("--seeds", type=int, default=2)
225+
ap.add_argument("--kernels", default=",".join(L_KERNELS))
187226
ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control")
188227
ap.add_argument("--json")
189228
a = ap.parse_args(argv)
190229
os.makedirs(a.build, exist_ok=True)
191230
if a.prepare:
192231
print("built", prepare(a.build, a.rocm))
193232
if a.run:
194-
kernels = ["l_swin_block_c32_nc"] + ([] if a.no_control else CONTROL_KERNELS)
233+
kernels = [k for k in a.kernels.split(",") if k] + ([] if a.no_control else CONTROL_KERNELS)
195234
res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), kernels)
196235
if a.json:
197236
with open(a.json, "w") as f:

0 commit comments

Comments
 (0)