diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 1327395d..e1e9ac90 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -178,7 +178,7 @@ def kernel_decls(path): # Kernels of Model M's code objects that only Model L's frame launches (#89): dec1 as a standalone block (#139; # Model M runs dec1 fused with dec0). -MODEL_L_ONLY = {"swin_block_c64_add_ex_nc"} +MODEL_L_ONLY = {"swin_block_c64_add_ex_nc", "lev/input_encoder_q20", "lev/input_encoder_direct_q20"} def is_model_l(decl): diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 2a7febe3..2b651688 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -274,6 +274,14 @@ "lev/input_encoder_direct", "l_enc_values.co" ], + [ + "lev/input_encoder_direct_q20", + "-" + ], + [ + "lev/input_encoder_q20", + "-" + ], [ "swin_block_c64_add_ex_nc", "swin.co" diff --git a/clean/dx12/kernels/encoder.hlsl b/clean/dx12/kernels/encoder.hlsl index 88f4f769..a5e0fd18 100644 --- a/clean/dx12/kernels/encoder.hlsl +++ b/clean/dx12/kernels/encoder.hlsl @@ -38,10 +38,22 @@ // form). Not with motion at output resolution. // @wg-kernel lev/input_encoder co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1 // @wg-kernel lev/input_encoder_direct co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1 +// The q20 forms (D3D12 only; WG_ENC_Q20): the same values, each halo pixel's 20 value bytes without the 12 zero +// bytes, in two planes: bytes 0..15 of every pixel ([fh + 4][fw + 4][16]), then bytes 16..19 of every pixel +// ([fh + 4][fw + 4][4]): the map the encoder block with the linears as its prologue reads (l_swin_block_c32_enc_lin_nc, +// runtime fusion le). +// @wg-kernel lev/input_encoder_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 +// @wg-kernel lev/input_encoder_direct_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 #include "wg_platform.hlsli" #ifndef WG_ENC_VALUES_OUT #define WG_ENC_VALUES_OUT 0 #endif +#ifndef WG_ENC_Q20 +#define WG_ENC_Q20 0 +#endif +#if WG_ENC_Q20 && !WG_ENC_VALUES_OUT +#error "WG_ENC_Q20: Model L's value kernel only" +#endif #if WG_ENC_VALUES_OUT && defined(WG_ENC_MO) #error "WG_ENC_VALUES_OUT: motion at render resolution only" #endif @@ -540,9 +552,15 @@ void encoder_main() if (tok < ntok) { const uint mw = (uint)(P_fw + 4); [unroll] for (uint dx3 = 0; dx3 < 2; ++dx3) { +#if WG_ENC_Q20 // two planes: bytes 0..15, then bytes 16..19 of every pixel + const uint hp = (uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2); + wg_st128(features, hp * 16u, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3])); + wg_st32(features, mw * (uint)(P_fh + 4) * 16u + hp * 4u, q[dx3][4]); +#else const uint ho = ((uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2)) * 32u; wg_st128(features, ho, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3])); wg_st128(features, ho + 16u, uint4(q[dx3][4], 0u, 0u, 0u)); +#endif } } #else diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 34bce87f..d3157eea 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -227,9 +227,16 @@ uint sw_gelu_m(uint g) return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u; } struct SwGeluPre { uint2 e[GROUP][8]; }; +#ifndef TT_GELU +#define TT_GELU 0 +#endif SwGeluPre sw_gelu_pre(v8f h[GROUP]) { SwGeluPre r; +#if TT_GELU + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0); + return r; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { if (sw_gelu_m(g) == 1u) { const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]); @@ -251,6 +258,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP]) } void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP]) { +#if TT_GELU == 3 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes + uint b[8]; + [unroll] for (uint i0 = 0; i0 < 4; ++i0) { + b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u); + b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u); + } + q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24), + (b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24)); + } + return; +#elif TT_GELU == 4 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); + const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u); + const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u); + q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16)); + } + return; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { v8f y; if (sw_gelu_m(g) != 0u) { @@ -479,16 +507,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) #include "l_lin_px.hlsli" // LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel // (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that -// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's +// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the +// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map. void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) { const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); uint2 xq[2]; - [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u; + const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19 + xq[0] = uint2(v.x, hh != 0u ? 0u : v.y); + xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw; v8f b1[2], b2[2]; uint2 f1[2][2], f2[2][2]; lin_blob(linW1, lane, hh, b1, f1); lin_blob(linW2, lane, hh, b2, f2); +#if defined(TT_LINF32) && TT_LINF32 + { // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32) + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + v8f r = lin_acc(b1[m1], f1[m1], xq); + [unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f); + x1[m1] = wg_pack_q8_ags(r); + } + [unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1); + return; + } +#endif uint4 y[2]; lin_pair(xq, b1, f1, b2, f2, y); [unroll] for (uint k = 0; k < KC; ++k) { diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index bd7fb81f..3f14f168 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -18,7 +18,8 @@ // l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16) // l_swin_block_c32_enc_lin_nc D3D12 only: the encoder block with the two input linears as its prologue (SW_SRC 3; // byte for byte the pair l_linear_32_32_fused -> _enc_nc, which stays the fallback route): -// the FP8 q halo map in, x2 neither written nor read. Arguments: _enc_nc's (x = q), then +// the FP8 q halo map in its 20-byte form (lev/input_encoder_q20) in, x2 neither +// written nor read. Arguments: _enc_nc's (x = q), then // enc0.Win1, enc0.Win2 // // @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index bced9859..d119627e 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -11,9 +11,11 @@ // (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at // (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid // mirror, sub-pixel kept (LCoarseFp8Src); 3 (D3D12 only) Model L's encoder block with -// its two input linears as the prologue: the FP8 q halo map [H + 4][W + 4][32] at the same -// position through the linears (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, -// in registers (the x2 map is neither written nor read) +// its two input linears as the prologue: the FP8 q halo map in its 20-byte form (two +// planes [H + 4][W + 4][16], [H + 4][W + 4][4]; lev/input_encoder_q20) at the same position +// through the linears +// (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, in registers (the x2 map is +// neither written nor read) // SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi) // SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue: // the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those @@ -193,9 +195,16 @@ uint sw_gelu_m(uint g) return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u; } struct SwGeluPre { uint2 e[GROUP][8]; }; +#ifndef TT_GELU +#define TT_GELU 0 +#endif SwGeluPre sw_gelu_pre(v8f h[GROUP]) { SwGeluPre r; +#if TT_GELU + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0); + return r; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { if (sw_gelu_m(g) == 1u) { const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]); @@ -217,6 +226,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP]) } void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP]) { +#if TT_GELU == 3 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes + uint b[8]; + [unroll] for (uint i0 = 0; i0 < 4; ++i0) { + b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u); + b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u); + } + q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24), + (b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24)); + } + return; +#elif TT_GELU == 4 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); + const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u); + const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u); + q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16)); + } + return; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { v8f y; if (sw_gelu_m(g) != 0u) { @@ -445,16 +475,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) #include "l_lin_px.hlsli" // LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel // (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that -// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's +// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the +// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map. void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) { const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); uint2 xq[2]; - [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u; + const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19 + xq[0] = uint2(v.x, hh != 0u ? 0u : v.y); + xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw; v8f b1[2], b2[2]; uint2 f1[2][2], f2[2][2]; lin_blob(linW1, lane, hh, b1, f1); lin_blob(linW2, lane, hh, b2, f2); +#if defined(TT_LINF32) && TT_LINF32 + { // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32) + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + v8f r = lin_acc(b1[m1], f1[m1], xq); + [unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f); + x1[m1] = wg_pack_q8_ags(r); + } + [unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1); + return; + } +#endif uint4 y[2]; lin_pair(xq, b1, f1, b2, f2, y); [unroll] for (uint k = 0; k < KC; ++k) { diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index 74bff879..430d8a3a 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -23,6 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle | Map | Shape | Element | Written by | Read by | |---|---|---|---|---| | `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | +| `q` (20-byte form) | two planes: `[fh + 4][fw + 4][16]` (bytes 0..15 of every pixel), then `[fh + 4][fw + 4][4]` (bytes 16..19) | the same values without the zero bytes (D3D12 only, fusion `le`) | `lev/input_encoder_q20` | `l_swin_block_c32_enc_lin_nc` | | `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused` or `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | | `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel; not stored by `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | | `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | @@ -80,7 +81,7 @@ workgroup, in raster order of their map. | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | | `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | -| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` (FP8 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run | +| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` in its 20-byte form (from `lev/input_encoder_q20`; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run | | `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | | `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | diff --git a/clean/l/runner/l_enclin_check.py b/clean/l/runner/l_enclin_check.py index 5895c346..c80d8cb6 100644 --- a/clean/l/runner/l_enclin_check.py +++ b/clean/l/runner/l_enclin_check.py @@ -12,7 +12,9 @@ --run (GPU, one GPU slot): every case is an encoder block case of clean/l/runner/l_d3d12_check.py (its grids, fewer windows than a workgroup's 8 among them, both window offsets, the block blob) with an FP8 q halo map [H + 4][W + 4][32] in place of x2 and the two linear blobs (enc0.Win1, enc0.Win2) at clean/l/runner/l_linfuse_check.py's weight scales, -special FP8 codes in q in every other case. On the same input bytes: +special FP8 codes in q in every other case; q's bytes 20..31 zero (the map's format: 20 values, 12 zero bytes), and +the fused kernel reads q's 20-byte form (lev/input_encoder_q20's layout: bytes 0..15 of every pixel, then bytes 16..19 +of every pixel). On the same input bytes: - the sequence on D3D12 (kernel_replay, two passes: l_linear_32_32_fused writes x2, then l_swin_block_c32_enc_nc reads that file); - the sequence on HIP (in this process: l_linear_32_32_relu, l_linear_32_32_f16, l_swin_block_c32_enc_nc); @@ -55,9 +57,18 @@ def cases(rng): if ci % 2: pick = rng.random(q.size) < 0.1 q[pick] = rng.choice(LF.SPECIAL, int(pick.sum())) + q = q.reshape(npix, 32) + q[:, 20:] = 0 yield H, W, off, q.tobytes(), LF._blob(rng, s, b), LF._blob(rng, s, b), blob, nout +def q20(q): + """The 20-byte form of a 32-byte q map: bytes 0..15 of every pixel, then bytes 16..19 of every pixel; 12 bytes of + slack after it (the last pixel's upper lanes load 16 bytes at its place in the second plane).""" + a = np.frombuffer(q, np.uint8).reshape(-1, 32) + return a[:, :16].tobytes() + a[:, 16:20].tobytes() + bytes(12) + + def hip_sequence(g, ops, c): """relu, f16, then the encoder block, on HIP: x3's raw bytes (the output starts at HIP_SENTINEL).""" H, W, off, q, w1, w2, blob, nout = c @@ -90,7 +101,7 @@ def buf(data): def enclin_lines(name, c, prefix="", grid=None): H, W, off, _q, _w1, _w2, _blob, nout = c return [f"case {prefix}{name}_enclin", f"kernel {ENCLIN} {grid or -(-nwin_of(H, W, off) // 8)} 256", - f"arg buf {name}.q", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}", + f"arg buf {name}.q20", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}", f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", f"arg buf {name}.w1", f"arg buf {name}.w2", ""] @@ -109,7 +120,7 @@ def run(build, rocm, seeds): H, W, off, q, w1, w2, blob, nout = c npix = (H + 4) * (W + 4) all_cases[name] = (nout, hip_sequence(g, ops, c)) - for ext, data in ((".q", q), (".w1", w1), (".w2", w2), (".w", blob)): + for ext, data in ((".q", q), (".q20", q20(q)), (".w1", w1), (".w2", w2), (".w", blob)): write(os.path.join(cdir, name + ext), data) grid = -(-nwin_of(H, W, off) // 8) # pass 1: the D3D12 fused linears (x2) and the fused block diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index c702a087..41f56ddb 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -195,6 +195,8 @@ def base(kernel): whose arguments it shares.""" k = kernel.replace("_nc", "") post = ("l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr") # pack12 and LDR forms + if k.startswith("lev/input_encoder") and k.endswith("_q20"): # the q20 forms (fusion le): the same arguments + return k[:-len("_q20")] return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k in post else k @@ -638,7 +640,8 @@ def test_frame(self): (NO_ENCLIN, full_ports(), True, True, False), (None, without(ENCLIN), True, True, False), (None, without(ENCLIN, LIN), True, False, False), - (None, without(LIN), True, False, True)): + (None, without(LIN), True, False, True), + (None, without("lev/input_encoder_q20"), True, True, False)): with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), lin_in_manifest=any(kc[0] == LIN for kc in ports), enclin_in_manifest=any(kc[0] == ENCLIN for kc in ports)): diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index ba3b6879..58fae1da 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -343,7 +343,9 @@ struct Runtime { // l_swin.hlsl SW_SRC 3; byte for byte the linears -> l_swin_block_c32_enc_nc): one launch less, and x2 (64 bytes // a pixel of the halo map) is neither stored nor read back: the block reads q (32 bytes) from l_narrow. Over ll // when both are on. Without le, or with a manifest that lacks the kernel, the linears and the block run - bool fz_le = false, l_le_built = false; + // The block reads q in its 20-byte form, which the q20 input encoder writes (lev/input_encoder_q20 / + // _direct_q20): le needs that kernel too + bool fz_le = false, l_le_built = false, l_q20_built = false, l_q20d_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; @@ -561,6 +563,8 @@ struct Runtime { l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end(); l_le_built = std::find(have.begin(), have.end(), "l_swin_block_c32_enc_lin_nc") != have.end(); + l_q20_built = std::find(have.begin(), have.end(), "lev/input_encoder_q20") != have.end(); + l_q20d_built = std::find(have.begin(), have.end(), "lev/input_encoder_direct_q20") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1587,14 +1591,15 @@ struct Runtime { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) - const bool in_block = fz_le && l_le_built && l_nc[0]; // fusion le: the linears in the block (its _nc form) + // fusion le: the linears in the block (its _nc form), q in the 20-byte form from the q20 input encoder + const bool in_block = fz_le && l_le_built && l_nc[0] && (native_direct_io ? l_q20d_built : l_q20_built); DevPtr& q = fused || in_block ? l_narrow : l_wide; // q's map (fused / in the block: not under x2) if (in_block) { DevPtr none = 0, wb = wdev["enc0_block"]; int ox = 0, oy = 0; l_offsets(false, f, &ox, &oy); const int nwin = ((fw + ox + 3) / 4) * ((fh + oy + 3) / 4); - return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + return launch(native_direct_io ? "lev/input_encoder_direct_q20" : "lev/input_encoder_q20", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, &expo, &wv, &p}) && diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 165954b6..5a3e98a8 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -71,6 +71,7 @@ struct Signature { }; const Signature kSignatures[] = { {"input_encoder", "ppppppppppppE"}, // Model M's encoder and Model L's values (q in place of the features) + {"input_encoder_q20", "ppppppppppppE"}, // Model L's values, q in its 20-byte form (native D3D12, fusion le) {"coef_decoder", "ppppppppD"}, // Model M's decoder (also its pack12 form) {"swin_block_*", "ppppiiii"}, // Model M's blocks: x, skip, out, w, H, W, ox, oy {"l_swin_block_c32_dec_coef12_nc", "ppppiiiippppii"}, // the decoder block with l_coef_pack12 folded in: then