Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clean/dx12/build_dx12.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ def kernel_decls(path):

# Kernels of Model M's code objects that only Model L's frame launches (#89): dec1 as a standalone block (#139;
# Model M runs dec1 fused with dec0).
MODEL_L_ONLY = {"swin_block_c64_add_ex_nc"}
MODEL_L_ONLY = {"swin_block_c64_add_ex_nc", "lev/input_encoder_q20", "lev/input_encoder_direct_q20"}


def is_model_l(decl):
Expand Down
8 changes: 8 additions & 0 deletions clean/dx12/kernel_sets.json
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,14 @@
"lev/input_encoder_direct",
"l_enc_values.co"
],
[
"lev/input_encoder_direct_q20",
"-"
],
[
"lev/input_encoder_q20",
"-"
],
[
"swin_block_c64_add_ex_nc",
"swin.co"
Expand Down
18 changes: 18 additions & 0 deletions clean/dx12/kernels/encoder.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,22 @@
// form). Not with motion at output resolution.
// @wg-kernel lev/input_encoder co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1
// @wg-kernel lev/input_encoder_direct co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1
// The q20 forms (D3D12 only; WG_ENC_Q20): the same values, each halo pixel's 20 value bytes without the 12 zero
// bytes, in two planes: bytes 0..15 of every pixel ([fh + 4][fw + 4][16]), then bytes 16..19 of every pixel
// ([fh + 4][fw + 4][4]): the map the encoder block with the linears as its prologue reads (l_swin_block_c32_enc_lin_nc,
// runtime fusion le).
// @wg-kernel lev/input_encoder_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1
// @wg-kernel lev/input_encoder_direct_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1
#include "wg_platform.hlsli"
#ifndef WG_ENC_VALUES_OUT
#define WG_ENC_VALUES_OUT 0
#endif
#ifndef WG_ENC_Q20
#define WG_ENC_Q20 0
#endif
#if WG_ENC_Q20 && !WG_ENC_VALUES_OUT
#error "WG_ENC_Q20: Model L's value kernel only"
#endif
#if WG_ENC_VALUES_OUT && defined(WG_ENC_MO)
#error "WG_ENC_VALUES_OUT: motion at render resolution only"
#endif
Expand Down Expand Up @@ -540,9 +552,15 @@ void encoder_main()
if (tok < ntok) {
const uint mw = (uint)(P_fw + 4);
[unroll] for (uint dx3 = 0; dx3 < 2; ++dx3) {
#if WG_ENC_Q20 // two planes: bytes 0..15, then bytes 16..19 of every pixel
const uint hp = (uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2);
wg_st128(features, hp * 16u, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3]));
wg_st32(features, mw * (uint)(P_fh + 4) * 16u + hp * 4u, q[dx3][4]);
#else
const uint ho = ((uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2)) * 32u;
wg_st128(features, ho, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3]));
wg_st128(features, ho + 16u, uint4(q[dx3][4], 0u, 0u, 0u));
#endif
}
}
#else
Expand Down
49 changes: 47 additions & 2 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -227,9 +227,16 @@ uint sw_gelu_m(uint g)
return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u;
}
struct SwGeluPre { uint2 e[GROUP][8]; };
#ifndef TT_GELU
#define TT_GELU 0
#endif
SwGeluPre sw_gelu_pre(v8f h[GROUP])
{
SwGeluPre r;
#if TT_GELU
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0);
return r;
#endif
[unroll] for (uint g = 0; g < GROUP; ++g) {
if (sw_gelu_m(g) == 1u) {
const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]);
Expand All @@ -251,6 +258,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP])
}
void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP])
{
#if TT_GELU == 3
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) {
const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes
uint b[8];
[unroll] for (uint i0 = 0; i0 < 4; ++i0) {
b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u);
b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u);
}
q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24),
(b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24));
}
return;
#elif TT_GELU == 4
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) {
const uint2 c = wg_pack_q8_nc_ags(h[g0]);
const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u);
const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u);
q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16));
}
return;
#endif
[unroll] for (uint g = 0; g < GROUP; ++g) {
v8f y;
if (sw_gelu_m(g) != 0u) {
Expand Down Expand Up @@ -479,16 +507,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
#include "l_lin_px.hlsli"
// LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel
// (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that
// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow)
// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's
// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the
// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map.
void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC])
{
const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2);
uint2 xq[2];
[unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc);
const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u;
const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19
xq[0] = uint2(v.x, hh != 0u ? 0u : v.y);
xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw;
v8f b1[2], b2[2];
uint2 f1[2][2], f2[2][2];
lin_blob(linW1, lane, hh, b1, f1);
lin_blob(linW2, lane, hh, b2, f2);
#if defined(TT_LINF32) && TT_LINF32
{ // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32)
uint2 x1[2];
[unroll] for (uint m1 = 0; m1 < 2u; ++m1) {
v8f r = lin_acc(b1[m1], f1[m1], xq);
[unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f);
x1[m1] = wg_pack_q8_ags(r);
}
[unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1);
return;
}
#endif
uint4 y[2];
lin_pair(xq, b1, f1, b2, f2, y);
[unroll] for (uint k = 0; k < KC; ++k) {
Expand Down
3 changes: 2 additions & 1 deletion clean/dx12/kernels/l_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
// l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16)
// l_swin_block_c32_enc_lin_nc D3D12 only: the encoder block with the two input linears as its prologue (SW_SRC 3;
// byte for byte the pair l_linear_32_32_fused -> _enc_nc, which stays the fallback route):
// the FP8 q halo map in, x2 neither written nor read. Arguments: _enc_nc's (x = q), then
// the FP8 q halo map in its 20-byte form (lev/input_encoder_q20) in, x2 neither
// written nor read. Arguments: _enc_nc's (x = q), then
// enc0.Win1, enc0.Win2
//
// @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc
Expand Down
57 changes: 52 additions & 5 deletions clean/dx12/kernels/swin_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,11 @@
// (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at
// (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid
// mirror, sub-pixel kept (LCoarseFp8Src); 3 (D3D12 only) Model L's encoder block with
// its two input linears as the prologue: the FP8 q halo map [H + 4][W + 4][32] at the same
// position through the linears (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2,
// in registers (the x2 map is neither written nor read)
// its two input linears as the prologue: the FP8 q halo map in its 20-byte form (two
// planes [H + 4][W + 4][16], [H + 4][W + 4][4]; lev/input_encoder_q20) at the same position
// through the linears
// (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, in registers (the x2 map is
// neither written nor read)
// SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi)
// SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue:
// the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those
Expand Down Expand Up @@ -193,9 +195,16 @@ uint sw_gelu_m(uint g)
return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u;
}
struct SwGeluPre { uint2 e[GROUP][8]; };
#ifndef TT_GELU
#define TT_GELU 0
#endif
SwGeluPre sw_gelu_pre(v8f h[GROUP])
{
SwGeluPre r;
#if TT_GELU
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0);
return r;
#endif
[unroll] for (uint g = 0; g < GROUP; ++g) {
if (sw_gelu_m(g) == 1u) {
const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]);
Expand All @@ -217,6 +226,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP])
}
void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP])
{
#if TT_GELU == 3
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) {
const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes
uint b[8];
[unroll] for (uint i0 = 0; i0 < 4; ++i0) {
b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u);
b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u);
}
q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24),
(b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24));
}
return;
#elif TT_GELU == 4
[unroll] for (uint g0 = 0; g0 < GROUP; ++g0) {
const uint2 c = wg_pack_q8_nc_ags(h[g0]);
const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u);
const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u);
q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16));
}
return;
#endif
[unroll] for (uint g = 0; g < GROUP; ++g) {
v8f y;
if (sw_gelu_m(g) != 0u) {
Expand Down Expand Up @@ -445,16 +475,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC])
#include "l_lin_px.hlsli"
// LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel
// (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that
// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow)
// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's
// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the
// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map.
void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC])
{
const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2);
uint2 xq[2];
[unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc);
const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u;
const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19
xq[0] = uint2(v.x, hh != 0u ? 0u : v.y);
xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw;
v8f b1[2], b2[2];
uint2 f1[2][2], f2[2][2];
lin_blob(linW1, lane, hh, b1, f1);
lin_blob(linW2, lane, hh, b2, f2);
#if defined(TT_LINF32) && TT_LINF32
{ // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32)
uint2 x1[2];
[unroll] for (uint m1 = 0; m1 < 2u; ++m1) {
v8f r = lin_acc(b1[m1], f1[m1], xq);
[unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f);
x1[m1] = wg_pack_q8_ags(r);
}
[unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1);
return;
}
#endif
uint4 y[2];
lin_pair(xq, b1, f1, b2, f2, y);
[unroll] for (uint k = 0; k < KC; ++k) {
Expand Down
3 changes: 2 additions & 1 deletion clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle
| Map | Shape | Element | Written by | Read by |
|---|---|---|---|---|
| `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` |
| `q` (20-byte form) | two planes: `[fh + 4][fw + 4][16]` (bytes 0..15 of every pixel), then `[fh + 4][fw + 4][4]` (bytes 16..19) | the same values without the zero bytes (D3D12 only, fusion `le`) | `lev/input_encoder_q20` | `l_swin_block_c32_enc_lin_nc` |
| `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused` or `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` |
| `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel; not stored by `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` |
| `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` |
Expand Down Expand Up @@ -80,7 +81,7 @@ workgroup, in raster order of their map.
| `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` |
| `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run |
| `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` |
| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` (FP8 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run |
| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` in its 20-byte form (from `lev/input_encoder_q20`; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run |
| `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` |
| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
| `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` |
Expand Down
17 changes: 14 additions & 3 deletions clean/l/runner/l_enclin_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@
--run (GPU, one GPU slot): every case is an encoder block case of clean/l/runner/l_d3d12_check.py (its grids, fewer
windows than a workgroup's 8 among them, both window offsets, the block blob) with an FP8 q halo map [H + 4][W + 4][32]
in place of x2 and the two linear blobs (enc0.Win1, enc0.Win2) at clean/l/runner/l_linfuse_check.py's weight scales,
special FP8 codes in q in every other case. On the same input bytes:
special FP8 codes in q in every other case; q's bytes 20..31 zero (the map's format: 20 values, 12 zero bytes), and
the fused kernel reads q's 20-byte form (lev/input_encoder_q20's layout: bytes 0..15 of every pixel, then bytes 16..19
of every pixel). On the same input bytes:
- the sequence on D3D12 (kernel_replay, two passes: l_linear_32_32_fused writes x2, then l_swin_block_c32_enc_nc reads
that file);
- the sequence on HIP (in this process: l_linear_32_32_relu, l_linear_32_32_f16, l_swin_block_c32_enc_nc);
Expand Down Expand Up @@ -55,9 +57,18 @@ def cases(rng):
if ci % 2:
pick = rng.random(q.size) < 0.1
q[pick] = rng.choice(LF.SPECIAL, int(pick.sum()))
q = q.reshape(npix, 32)
q[:, 20:] = 0
yield H, W, off, q.tobytes(), LF._blob(rng, s, b), LF._blob(rng, s, b), blob, nout


def q20(q):
"""The 20-byte form of a 32-byte q map: bytes 0..15 of every pixel, then bytes 16..19 of every pixel; 12 bytes of
slack after it (the last pixel's upper lanes load 16 bytes at its place in the second plane)."""
a = np.frombuffer(q, np.uint8).reshape(-1, 32)
return a[:, :16].tobytes() + a[:, 16:20].tobytes() + bytes(12)


def hip_sequence(g, ops, c):
"""relu, f16, then the encoder block, on HIP: x3's raw bytes (the output starts at HIP_SENTINEL)."""
H, W, off, q, w1, w2, blob, nout = c
Expand Down Expand Up @@ -90,7 +101,7 @@ def buf(data):
def enclin_lines(name, c, prefix="", grid=None):
H, W, off, _q, _w1, _w2, _blob, nout = c
return [f"case {prefix}{name}_enclin", f"kernel {ENCLIN} {grid or -(-nwin_of(H, W, off) // 8)} 256",
f"arg buf {name}.q", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}",
f"arg buf {name}.q20", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}",
f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}",
f"arg buf {name}.w1", f"arg buf {name}.w2", ""]

Expand All @@ -109,7 +120,7 @@ def run(build, rocm, seeds):
H, W, off, q, w1, w2, blob, nout = c
npix = (H + 4) * (W + 4)
all_cases[name] = (nout, hip_sequence(g, ops, c))
for ext, data in ((".q", q), (".w1", w1), (".w2", w2), (".w", blob)):
for ext, data in ((".q", q), (".q20", q20(q)), (".w1", w1), (".w2", w2), (".w", blob)):
write(os.path.join(cdir, name + ext), data)
grid = -(-nwin_of(H, W, off) // 8)
# pass 1: the D3D12 fused linears (x2) and the fused block
Expand Down
Loading
Loading