Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -523,8 +523,9 @@ int wg_low0(int p0, int n)
}

#define EKC (SW_ECL / 16)
#if (4 * SW_C / 16) % 8 != 0
#error "fused_expand: MT = 4 C / 16 must be a multiple of the 8 waves"
#define FE_NW (THREADS / 32) // waves per workgroup (8; 4 in swin_c64w2.hlsl's 128-lane form)
#if (4 * SW_C / 16) % FE_NW != 0
#error "fused_expand: MT = 4 C / 16 must be a multiple of the waves"
#endif
// HIP fused_expand<C, CL>: the 8 windows' expanded block input e = q8(We y + be) into LDS records
// [window][low token][sub-pixel][C] at e_lds. tail = expand weights (bias [4C] f32, then the fragments, in global memory).
Expand All @@ -549,19 +550,19 @@ void fused_expand(uint tail, uint e_lds, int nwx, int nwin, uint tid, int blk, u
}
}
uint be = tail + hh * 32u;
// MT = 24 = 3 x 8 waves: every wave runs exactly MT / 8 tiles, unrolled with all weight fragments and biases issued
// first (one exposed global latency instead of one per tile)
// MT = 24 = 3 x 8 waves: every wave runs exactly MT / FE_NW tiles, unrolled with all weight fragments and biases
// issued first (one exposed global latency instead of one per tile)
const uint wv = (uint)wg_uniform((int)wave);
uint2 fa[MT / 8][EKC];
v8f ba[MT / 8];
uint2 fa[MT / FE_NW][EKC];
v8f ba[MT / FE_NW];
const uint fwb = tail + WE + wv * (EKC * 256u) + (lane * 8u + sw_zero_bits());
[unroll] for (uint it = 0; it < MT / 8; ++it) {
const uint mt = wv + 8u * it;
[unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (8u * it * EKC + kc) * 256u);
[unroll] for (uint it = 0; it < MT / FE_NW; ++it) {
const uint mt = wv + FE_NW * it;
[unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (FE_NW * it * EKC + kc) * 256u);
ba[it] = wg_load8f(be, mt * 64u);
}
[unroll] for (uint it2 = 0; it2 < MT / 8; ++it2) {
const uint mt = wv + 8u * it2;
[unroll] for (uint it2 = 0; it2 < MT / FE_NW; ++it2) {
const uint mt = wv + FE_NW * it2;
uint k = mt / G, g = mt - k * G;
v8f acc0 = ba[it2], acc1 = ba[it2];
[unroll] for (uint kc2 = 0; kc2 < EKC; ++kc2) {
Expand Down
23 changes: 12 additions & 11 deletions clean/dx12/kernels/swin_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -483,8 +483,9 @@ int wg_low0(int p0, int n)
}

#define EKC (SW_ECL / 16)
#if (4 * SW_C / 16) % 8 != 0
#error "fused_expand: MT = 4 C / 16 must be a multiple of the 8 waves"
#define FE_NW (THREADS / 32) // waves per workgroup (8; 4 in swin_c64w2.hlsl's 128-lane form)
#if (4 * SW_C / 16) % FE_NW != 0
#error "fused_expand: MT = 4 C / 16 must be a multiple of the waves"
#endif
// HIP fused_expand<C, CL>: the 8 windows' expanded block input e = q8(We y + be) into LDS records
// [window][low token][sub-pixel][C] at e_lds. tail = expand weights (bias [4C] f32, then the fragments, in global memory).
Expand All @@ -509,19 +510,19 @@ void fused_expand(uint tail, uint e_lds, int nwx, int nwin, uint tid, int blk, u
}
}
uint be = tail + hh * 32u;
// MT = 24 = 3 x 8 waves: every wave runs exactly MT / 8 tiles, unrolled with all weight fragments and biases issued
// first (one exposed global latency instead of one per tile)
// MT = 24 = 3 x 8 waves: every wave runs exactly MT / FE_NW tiles, unrolled with all weight fragments and biases
// issued first (one exposed global latency instead of one per tile)
const uint wv = (uint)wg_uniform((int)wave);
uint2 fa[MT / 8][EKC];
v8f ba[MT / 8];
uint2 fa[MT / FE_NW][EKC];
v8f ba[MT / FE_NW];
const uint fwb = tail + WE + wv * (EKC * 256u) + (lane * 8u + sw_zero_bits());
[unroll] for (uint it = 0; it < MT / 8; ++it) {
const uint mt = wv + 8u * it;
[unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (8u * it * EKC + kc) * 256u);
[unroll] for (uint it = 0; it < MT / FE_NW; ++it) {
const uint mt = wv + FE_NW * it;
[unroll] for (uint kc = 0; kc < EKC; ++kc) fa[it][kc] = wg_ld64(fwb, (FE_NW * it * EKC + kc) * 256u);
ba[it] = wg_load8f(be, mt * 64u);
}
[unroll] for (uint it2 = 0; it2 < MT / 8; ++it2) {
const uint mt = wv + 8u * it2;
[unroll] for (uint it2 = 0; it2 < MT / FE_NW; ++it2) {
const uint mt = wv + FE_NW * it2;
uint k = mt / G, g = mt - k * G;
v8f acc0 = ba[it2], acc1 = ba[it2];
[unroll] for (uint kc2 = 0; kc2 < EKC; ++kc2) {
Expand Down
15 changes: 11 additions & 4 deletions clean/dx12/kernels/swin_c64ex.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,15 @@
// Model L's frame (issue #89) launches it for dec1 (default swin_fuse_stages dec1,dec2: the L decoder reads dec1's map).
// swin_block.hlsl instantiated at C = 64, 2 heads, MLP 256 (its C = 64 path, as swin_block_c64_nc) with the block input
// x + xadd (SW_ADD) and the fused expand from the 96-channel low-resolution map (SW_ECL 96), as
// swin_block_c96_add_ex_nc is at C = 96. 256 lanes, one window per wave; x = dec2's output (96 channels at half size),
// xadd = the skip map, aux unused.
// swin_block_c96_add_ex_nc is at C = 96. x = dec2's output (96 channels at half size), xadd = the skip map, aux unused.
//
// @wg-kernel swin_block_c64_add_ex_nc co=swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=96,SW_ENTRY_OVERRIDE=swin_block_c64_add_ex_nc
#include "swin_block.hlsl"
// Two windows per wave in 128-lane workgroups (swin_c64w2.hlsl, the launch shape of Model M's enc1 block): the launch
// keeps HIP's geometry (8 windows per launched workgroup), each wave does two windows interleaved, every staged weight
// fragment read from LDS once for both, and a unit barrier waits for 4 waves instead of 8. Every window's arithmetic is
// that of swin_block.hlsl's C = 64 path (the same operations on the same operands in the same order).
//
// @wg-kernel swin_block_c64_add_ex_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u,p defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=1,SW_SAFE=1,SW_ECL=96,SW_W2=1,SW_W2_NAME=swin_block_c64_add_ex_nc
//
// Launch-shape switch: the one-window-per-wave kernel of swin_block.hlsl (256-lane workgroups, HIP's shape) is the same
// line without "group=128 gscale=1", with SW_W2=0 and SW_ENTRY_OVERRIDE=swin_block_c64_add_ex_nc in place of SW_W2_NAME.
#include "swin_c64w2.hlsl"
44 changes: 41 additions & 3 deletions clean/dx12/kernels/swin_c64w2.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
// swin_block_c64_nc (enc1's Swin block) with TWO windows per wave in 128-lane workgroups (tracks lev3-swin, r3-c64w2).
//
// @wg-kernel swin_block_c64_nc co=swin.co block=256 group=128 gscale=1 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=64,SW_HEADS=2,SW_MLP=256,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_W2=1
// Model L's dec1 block swin_block_c64_add_ex_nc (swin_c64ex.hlsl) runs this kernel too (SW_W2_NAME), with the block
// input x + xadd (SW_ADD) and the fused expand (SW_ECL: swin_block.hlsl's fused_expand with the 4 waves; each wave then
// reads its two windows' records).
//
// Launch-shape switch: the line above runs this file's kernel (SW_W2 1: 128-lane workgroups, gscale 1). The one-window-
// per-wave kernel of swin_block.hlsl (256-lane workgroups, HIP's shape) is the same line without "group=128 gscale=1"
Expand Down Expand Up @@ -47,9 +50,12 @@
#include "swin_block.hlsl"

#if SW_W2
#if SW_C != 64 || THREADS != 128 || SW_ADD || SW_ECL > 0 || (GROUP != 2 && GROUP != 1) || SW_GELU != 3 || SW_NORM2_EARLY || SW_HEAD_PAIRS
#if SW_C != 64 || THREADS != 128 || (GROUP != 2 && GROUP != 1) || SW_GELU != 3 || SW_NORM2_EARLY || SW_HEAD_PAIRS || SW_SRC != 0 || SW_OUT_F16
#error "swin_c64w2.hlsl: the c64 block, 128 lanes"
#endif
#ifndef SW_W2_NAME
#define SW_W2_NAME swin_block_c64_nc
#endif
#define NWW 2 // windows per wave
#ifndef SW_W2_MLP_STEP
#define SW_W2_MLP_STEP 4 // MLP units per loop pass (NMLP = 4: the whole MLP unrolled)
Expand All @@ -74,7 +80,7 @@
#define SW_W2_COMMIT SW_COMMIT_EARLY
#endif

WG_KERNEL(swin_block_c64_nc)
WG_KERNEL(SW_W2_NAME)
{
WG_SETUP();
const uint tid = wg_lane_id();
Expand All @@ -91,6 +97,15 @@ WG_KERNEL(swin_block_c64_nc)
// Gather: token t of each window, mirrored at the borders.
int dst[NWW];
uint2 xa[NWW][KC];
#if SW_ADD
uint2 xb[NWW][KC];
#endif
#if SW_ECL > 0
// the 8 windows' expanded input into the second staging buffer (swin_block.hlsl), read back after the barrier
const uint lowp = sw_win_low0(nwx, nwin, lane, (int)wg_block_id());
fused_expand(w + L_TAIL, MAXU, nwx, nwin, tid, (int)wg_block_id(), lowp);
uint eslot[NWW];
#endif
[unroll] for (uint i = 0; i < NWW; ++i) {
const int win0 = (int)(wg_block_id() * (THREADS / 32 * NWW) + wave_id * NWW + i);
const int win = win0 < nwin ? win0 : nwin - 1;
Expand All @@ -99,7 +114,17 @@ WG_KERNEL(swin_block_c64_nc)
const bool real = win0 < nwin && pr >= 0 && pr < H && pc >= 0 && pc < W;
dst[i] = real ? pr * W + pc : -1;
const int mr = sw_mirror(pr, H), mc = sw_mirror(pc, W);
#if SW_ECL > 0
const uint wb = wave_id * NWW + i; // the window in the workgroup (win: the same clamp)
const uint lpw = WaveReadLaneAt(lowp, wb);
const int low = ((mr >> 1) - (int)(lpw & 0xffffu)) * 2 + ((mc >> 1) - (int)(lpw >> 16));
eslot[i] = wb * 16u + (uint)low * 4u + (uint)(((mr & 1) << 1) | (mc & 1));
#else
load_frags_g(x, (uint)(mr * W + mc) * SW_C + hh * HALF, xa[i]);
#endif
#if SW_ADD
load_frags_g(xadd, (uint)(mr * W + mc) * SW_C + hh * HALF, xb[i]);
#endif
}
v8f ng[KC], nb[KC];
{
Expand All @@ -108,13 +133,26 @@ WG_KERNEL(swin_block_c64_nc)
}
st_commit(stage, 0u, HEAD_BYTES / 16, tid);
wg_barrier_lds();
#if SW_ECL > 0
[unroll] for (uint ie = 0; ie < NWW; ++ie)
[unroll] for (uint je = 0; je < KC / 2; ++je) {
const uint4 v = sw_lds_ld128(MAXU + eslot[ie] * SW_C + hh * HALF + 16u * je);
xa[ie][2 * je] = v.xy; xa[ie][2 * je + 1] = v.zw;
}
wg_barrier_lds(); // read before the buffer they live in is refilled
#endif
uint buf = 0u;

// Steps 1-2: a = q8(RMSNorm(x; gamma1)); the residual accumulator starts as x + bp.
uint2 af[NWW][KC];
v8f acc[NWW][KC];
[unroll] for (uint i1 = 0; i1 < NWW; ++i1) {
[unroll] for (uint k = 0; k < KC; ++k) acc[i1][k] = sw_unpack_fp8(xa[i1][k]);
[unroll] for (uint k = 0; k < KC; ++k) {
acc[i1][k] = sw_unpack_fp8(xa[i1][k]);
#if SW_ADD
acc[i1][k] = sw_add8(acc[i1][k], sw_unpack_fp8(xb[i1][k]));
#endif
}
const float r = norm_r(acc[i1]);
[unroll] for (uint k2 = 0; k2 < KC; ++k2) {
v8f d;
Expand Down
2 changes: 1 addition & 1 deletion clean/dx12/vgpr_limits.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"l_swin_block_c32_dec_nc": 96,
"l_swin_block_c32_dec_coef12_nc": 96,
"swin_block_c64_nc": 192,
"swin_block_c64_add_ex_nc": 128,
"swin_block_c64_add_ex_nc": 192,
"swin_block_c96_nc": 192,
"swin_block_c96_add_ex_nc": 192,
"swin_block_c96_pm_nc": 192,
Expand Down
Loading