From d5b0648125fa495aa0cc1212bed8ae8299ad7edf Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:44:35 -0500 Subject: [PATCH] Model L D3D12: the q map in a 20-byte form on the le route (the same bytes out) lev/input_encoder_q20 and lev/input_encoder_direct_q20 (encoder.hlsl WG_ENC_Q20) write each halo pixel's 20 value bytes without the 12 zero bytes, in two planes (bytes 0..15 of every pixel, then bytes 16..19), and the encoder block with the linears as its prologue (l_swin_block_c32_enc_lin_nc, SW_SRC 3) reads that form: lane half 1 takes bytes 16..19 from the second plane and zeros for 20..31, the fragments it read from the 32-byte map. 12 of q's 32 bytes per halo pixel are neither written nor read; every load and store stays aligned. Fusion le now needs the q20 input encoder too; without it the 32-byte route (input encoder, linears, plain block) runs. The 32-byte input encoders and every other existing DXIL are unchanged. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/build_dx12.py | 2 +- clean/dx12/kernel_sets.json | 8 +++ clean/dx12/kernels/encoder.hlsl | 18 ++++++ clean/dx12/kernels/fused_swin.hlsl | 49 +++++++++++++++- clean/dx12/kernels/l_swin.hlsl | 3 +- clean/dx12/kernels/swin_block.hlsl | 57 +++++++++++++++++-- clean/l/FORMATS.md | 3 +- clean/l/runner/l_enclin_check.py | 17 +++++- clean/l/runtime/tests/test_l_runtime_frame.py | 5 +- clean/runtime/runtime.cpp | 11 +++- clean/tests/runtime_frame_trace.cpp | 1 + 11 files changed, 157 insertions(+), 17 deletions(-) diff --git a/clean/dx12/build_dx12.py b/clean/dx12/build_dx12.py index 1327395d..e1e9ac90 100644 --- a/clean/dx12/build_dx12.py +++ b/clean/dx12/build_dx12.py @@ -178,7 +178,7 @@ def kernel_decls(path): # Kernels of Model M's code objects that only Model L's frame launches (#89): dec1 as a standalone block (#139; # Model M runs dec1 fused with dec0). -MODEL_L_ONLY = {"swin_block_c64_add_ex_nc"} +MODEL_L_ONLY = {"swin_block_c64_add_ex_nc", "lev/input_encoder_q20", "lev/input_encoder_direct_q20"} def is_model_l(decl): diff --git a/clean/dx12/kernel_sets.json b/clean/dx12/kernel_sets.json index 2a7febe3..2b651688 100644 --- a/clean/dx12/kernel_sets.json +++ b/clean/dx12/kernel_sets.json @@ -274,6 +274,14 @@ "lev/input_encoder_direct", "l_enc_values.co" ], + [ + "lev/input_encoder_direct_q20", + "-" + ], + [ + "lev/input_encoder_q20", + "-" + ], [ "swin_block_c64_add_ex_nc", "swin.co" diff --git a/clean/dx12/kernels/encoder.hlsl b/clean/dx12/kernels/encoder.hlsl index 88f4f769..a5e0fd18 100644 --- a/clean/dx12/kernels/encoder.hlsl +++ b/clean/dx12/kernels/encoder.hlsl @@ -38,10 +38,22 @@ // form). Not with motion at output resolution. // @wg-kernel lev/input_encoder co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1 // @wg-kernel lev/input_encoder_direct co=l_enc_values.co entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1 +// The q20 forms (D3D12 only; WG_ENC_Q20): the same values, each halo pixel's 20 value bytes without the 12 zero +// bytes, in two planes: bytes 0..15 of every pixel ([fh + 4][fw + 4][16]), then bytes 16..19 of every pixel +// ([fh + 4][fw + 4][4]): the map the encoder block with the linears as its prologue reads (l_swin_block_c32_enc_lin_nc, +// runtime fusion le). +// @wg-kernel lev/input_encoder_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 defines=WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 +// @wg-kernel lev/input_encoder_direct_q20 co=- entry=input_encoder block=256 wave=32 args=p,p,p,p,p,p,p,p,p,p,p,p,b116 rs=direct defines=WG_DIRECT_TEXTURES=1,WG_ENC_VALUES_OUT=1,WG_ENC_Q20=1 #include "wg_platform.hlsli" #ifndef WG_ENC_VALUES_OUT #define WG_ENC_VALUES_OUT 0 #endif +#ifndef WG_ENC_Q20 +#define WG_ENC_Q20 0 +#endif +#if WG_ENC_Q20 && !WG_ENC_VALUES_OUT +#error "WG_ENC_Q20: Model L's value kernel only" +#endif #if WG_ENC_VALUES_OUT && defined(WG_ENC_MO) #error "WG_ENC_VALUES_OUT: motion at render resolution only" #endif @@ -540,9 +552,15 @@ void encoder_main() if (tok < ntok) { const uint mw = (uint)(P_fw + 4); [unroll] for (uint dx3 = 0; dx3 < 2; ++dx3) { +#if WG_ENC_Q20 // two planes: bytes 0..15, then bytes 16..19 of every pixel + const uint hp = (uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2); + wg_st128(features, hp * 16u, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3])); + wg_st32(features, mw * (uint)(P_fh + 4) * 16u + hp * 4u, q[dx3][4]); +#else const uint ho = ((uint)(Y + 2) * mw + (uint)(X0 + (int)dx3 + 2)) * 32u; wg_st128(features, ho, uint4(q[dx3][0], q[dx3][1], q[dx3][2], q[dx3][3])); wg_st128(features, ho + 16u, uint4(q[dx3][4], 0u, 0u, 0u)); +#endif } } #else diff --git a/clean/dx12/kernels/fused_swin.hlsl b/clean/dx12/kernels/fused_swin.hlsl index 34bce87f..d3157eea 100644 --- a/clean/dx12/kernels/fused_swin.hlsl +++ b/clean/dx12/kernels/fused_swin.hlsl @@ -227,9 +227,16 @@ uint sw_gelu_m(uint g) return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u; } struct SwGeluPre { uint2 e[GROUP][8]; }; +#ifndef TT_GELU +#define TT_GELU 0 +#endif SwGeluPre sw_gelu_pre(v8f h[GROUP]) { SwGeluPre r; +#if TT_GELU + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0); + return r; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { if (sw_gelu_m(g) == 1u) { const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]); @@ -251,6 +258,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP]) } void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP]) { +#if TT_GELU == 3 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes + uint b[8]; + [unroll] for (uint i0 = 0; i0 < 4; ++i0) { + b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u); + b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u); + } + q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24), + (b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24)); + } + return; +#elif TT_GELU == 4 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); + const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u); + const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u); + q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16)); + } + return; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { v8f y; if (sw_gelu_m(g) != 0u) { @@ -479,16 +507,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) #include "l_lin_px.hlsli" // LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel // (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that -// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's +// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the +// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map. void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) { const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); uint2 xq[2]; - [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u; + const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19 + xq[0] = uint2(v.x, hh != 0u ? 0u : v.y); + xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw; v8f b1[2], b2[2]; uint2 f1[2][2], f2[2][2]; lin_blob(linW1, lane, hh, b1, f1); lin_blob(linW2, lane, hh, b2, f2); +#if defined(TT_LINF32) && TT_LINF32 + { // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32) + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + v8f r = lin_acc(b1[m1], f1[m1], xq); + [unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f); + x1[m1] = wg_pack_q8_ags(r); + } + [unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1); + return; + } +#endif uint4 y[2]; lin_pair(xq, b1, f1, b2, f2, y); [unroll] for (uint k = 0; k < KC; ++k) { diff --git a/clean/dx12/kernels/l_swin.hlsl b/clean/dx12/kernels/l_swin.hlsl index bd7fb81f..3f14f168 100644 --- a/clean/dx12/kernels/l_swin.hlsl +++ b/clean/dx12/kernels/l_swin.hlsl @@ -18,7 +18,8 @@ // l_swin_block_c32_dec_nc the decoder block: FP8 map in with the token-grid mirror (SW_SRC 2), FP16 out (SW_OUT_F16) // l_swin_block_c32_enc_lin_nc D3D12 only: the encoder block with the two input linears as its prologue (SW_SRC 3; // byte for byte the pair l_linear_32_32_fused -> _enc_nc, which stays the fallback route): -// the FP8 q halo map in, x2 neither written nor read. Arguments: _enc_nc's (x = q), then +// the FP8 q halo map in its 20-byte form (lev/input_encoder_q20) in, x2 neither +// written nor read. Arguments: _enc_nc's (x = q), then // enc0.Win1, enc0.Win2 // // @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc diff --git a/clean/dx12/kernels/swin_block.hlsl b/clean/dx12/kernels/swin_block.hlsl index bced9859..d119627e 100644 --- a/clean/dx12/kernels/swin_block.hlsl +++ b/clean/dx12/kernels/swin_block.hlsl @@ -11,9 +11,11 @@ // (default); 1 Model L's encoder block, an FP16 halo map [H + 4][W + 4][C] read directly at // (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid // mirror, sub-pixel kept (LCoarseFp8Src); 3 (D3D12 only) Model L's encoder block with -// its two input linears as the prologue: the FP8 q halo map [H + 4][W + 4][32] at the same -// position through the linears (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, -// in registers (the x2 map is neither written nor read) +// its two input linears as the prologue: the FP8 q halo map in its 20-byte form (two +// planes [H + 4][W + 4][16], [H + 4][W + 4][4]; lev/input_encoder_q20) at the same position +// through the linears +// (hlsl/l_lin_px.hlsli), the FP16 values mode 1 reads from x2, in registers (the x2 map is +// neither written nor read) // SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi) // SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue: // the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those @@ -193,9 +195,16 @@ uint sw_gelu_m(uint g) return (SW_GELU_PAT / (ix == 0u ? 1000u : ix == 1u ? 100u : ix == 2u ? 10u : 1u)) % 10u; } struct SwGeluPre { uint2 e[GROUP][8]; }; +#ifndef TT_GELU +#define TT_GELU 0 +#endif SwGeluPre sw_gelu_pre(v8f h[GROUP]) { SwGeluPre r; +#if TT_GELU + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) [unroll] for (uint i0 = 0; i0 < 8; ++i0) r.e[g0][i0] = uint2(0, 0); + return r; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { if (sw_gelu_m(g) == 1u) { const wg_gelu_lut_e a = wg_gelu_lut_fetch8(h[g]); @@ -217,6 +226,27 @@ SwGeluPre sw_gelu_pre(v8f h[GROUP]) } void sw_gelu_fin(v8f h[GROUP], SwGeluPre p, out uint2 q[GROUP]) { +#if TT_GELU == 3 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); // the 8 codes + uint b[8]; + [unroll] for (uint i0 = 0; i0 < 4; ++i0) { + b[i0] = wg_mem.Load(x + ((c.x >> (8u * i0)) & 255u) * 4u); + b[4 + i0] = wg_mem.Load(x + ((c.y >> (8u * i0)) & 255u) * 4u); + } + q[g0] = uint2((b[0] & 255u) | ((b[1] & 255u) << 8) | ((b[2] & 255u) << 16) | (b[3] << 24), + (b[4] & 255u) | ((b[5] & 255u) << 8) | ((b[6] & 255u) << 16) | (b[7] << 24)); + } + return; +#elif TT_GELU == 4 + [unroll] for (uint g0 = 0; g0 < GROUP; ++g0) { + const uint2 c = wg_pack_q8_nc_ags(h[g0]); + const uint b0 = wg_mem.Load(x + (c.x & 65535u) * 4u), b1 = wg_mem.Load(x + (c.x >> 16) * 4u); + const uint b2 = wg_mem.Load(x + (c.y & 65535u) * 4u), b3 = wg_mem.Load(x + (c.y >> 16) * 4u); + q[g0] = uint2((b0 & 65535u) | (b1 << 16), (b2 & 65535u) | (b3 << 16)); + } + return; +#endif [unroll] for (uint g = 0; g < GROUP; ++g) { v8f y; if (sw_gelu_m(g) != 0u) { @@ -445,16 +475,33 @@ void sw_load_halo_f16(uint base, int pr, int pc, int W, uint hh, out v8f f[KC]) #include "l_lin_px.hlsli" // LHaloLinSrc: the values of LHaloF16Src without the x2 map: the member at grid position (pr, pc) is pixel // (pr + 2) (W + 4) + pc + 2 of the FP8 q halo map, through l_linear_32_32_fused's arithmetic (lane (t, h) = that -// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow) +// kernel's lane of the pixel; every lane runs it, also for outside members: wave-uniform control flow). The map's +// 20-byte form (two planes): lane half h = 0 loads bytes 0..15 from the first plane, h = 1 bytes 16..19 from the +// second and takes zeros for 20..31 (the 32-byte form's zero bytes): the same fragments as from the 32-byte map. void sw_load_halo_lin(uint base, int pr, int pc, int W, uint lane, uint hh, out v8f f[KC]) { const uint pix = (uint)((pr + 2) * (W + 4) + pc + 2); uint2 xq[2]; - [unroll] for (uint kc = 0; kc < 2u; ++kc) xq[kc] = wg_ld64(base, pix * 32u + hh * 16u + 8u * kc); + const uint plane1 = (uint)((H + 4) * (W + 4)) * 16u; + const uint4 v = wg_ld128(base, hh != 0u ? plane1 + pix * 4u : pix * 16u); // h = 1: .x = bytes 16..19 + xq[0] = uint2(v.x, hh != 0u ? 0u : v.y); + xq[1] = hh != 0u ? uint2(0u, 0u) : v.zw; v8f b1[2], b2[2]; uint2 f1[2][2], f2[2][2]; lin_blob(linW1, lane, hh, b1, f1); lin_blob(linW2, lane, hh, b2, f2); +#if defined(TT_LINF32) && TT_LINF32 + { // no FP16 rounding points: x1 = q8(relu(acc)), x2 = acc (FP32) + uint2 x1[2]; + [unroll] for (uint m1 = 0; m1 < 2u; ++m1) { + v8f r = lin_acc(b1[m1], f1[m1], xq); + [unroll] for (uint j = 0; j < 8u; ++j) r.v[j] = wg_fmax(r.v[j], 0.0f); + x1[m1] = wg_pack_q8_ags(r); + } + [unroll] for (uint k8 = 0; k8 < KC; ++k8) f[k8] = lin_acc(b2[k8], f2[k8], x1); + return; + } +#endif uint4 y[2]; lin_pair(xq, b1, f1, b2, f2, y); [unroll] for (uint k = 0; k < KC; ++k) { diff --git a/clean/l/FORMATS.md b/clean/l/FORMATS.md index 74bff879..430d8a3a 100644 --- a/clean/l/FORMATS.md +++ b/clean/l/FORMATS.md @@ -23,6 +23,7 @@ Every map is `[rows][cols][C]`, channel fastest, as Model M's feature maps (`cle | Map | Shape | Element | Written by | Read by | |---|---|---|---|---| | `q` | `[fh + 4][fw + 4][32]` halo | FP8: values 0..19, bytes 20..31 zero | `lev/input_encoder` | `l_linear_32_32_relu` | +| `q` (20-byte form) | two planes: `[fh + 4][fw + 4][16]` (bytes 0..15 of every pixel), then `[fh + 4][fw + 4][4]` (bytes 16..19) | the same values without the zero bytes (D3D12 only, fusion `le`) | `lev/input_encoder_q20` | `l_swin_block_c32_enc_lin_nc` | | `x1` | `[fh + 4][fw + 4][32]` halo | FP8 (not stored by `l_linear_32_32_fused` or `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_relu` | `l_linear_32_32_f16` | | `x2` | `[fh + 4][fw + 4][32]` halo | **FP16** (64 bytes per pixel; not stored by `l_swin_block_c32_enc_lin_nc`) | `l_linear_32_32_f16` | `l_swin_block_c32_enc` | | `x3` | `[fh][fw][32]` | FP8 | `l_swin_block_c32_enc` | `l_project_gather` | @@ -80,7 +81,7 @@ workgroup, in raster order of their map. | `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` | | `l_linear_32_32_fused` | exists (GPU-checked), D3D12 only | `x, y, w1, w2, npix` | as above | the two above in one kernel, `x1` kept in registers: the same bytes in `y` (`x2`), `x1` not stored. x = `q`, y = `x2` in two different buffers. w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the two (fusion `ll`, on by default); without `ll` or without the kernel in the build, the two run | | `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` | -| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` (FP8 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run | +| `l_swin_block_c32_enc_lin_nc` | exists (GPU-checked), D3D12 only | the encoder block's, then `w1, w2` | as the encoder block | the encoder block with the two input linears as its prologue: the same bytes in `x3`, `x1` and `x2` not stored. x = `q` in its 20-byte form (from `lev/input_encoder_q20`; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2), the linears run on it in registers). w1 = `enc0.Win1`, w2 = `enc0.Win2`. The native D3D12 frame runs it in place of the linears and the block (fusion `le`, on by default); without `le` or without the kernel in the build, the linears (fused with `ll`) and the block run | | `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` | | `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` | | `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` | diff --git a/clean/l/runner/l_enclin_check.py b/clean/l/runner/l_enclin_check.py index 5895c346..c80d8cb6 100644 --- a/clean/l/runner/l_enclin_check.py +++ b/clean/l/runner/l_enclin_check.py @@ -12,7 +12,9 @@ --run (GPU, one GPU slot): every case is an encoder block case of clean/l/runner/l_d3d12_check.py (its grids, fewer windows than a workgroup's 8 among them, both window offsets, the block blob) with an FP8 q halo map [H + 4][W + 4][32] in place of x2 and the two linear blobs (enc0.Win1, enc0.Win2) at clean/l/runner/l_linfuse_check.py's weight scales, -special FP8 codes in q in every other case. On the same input bytes: +special FP8 codes in q in every other case; q's bytes 20..31 zero (the map's format: 20 values, 12 zero bytes), and +the fused kernel reads q's 20-byte form (lev/input_encoder_q20's layout: bytes 0..15 of every pixel, then bytes 16..19 +of every pixel). On the same input bytes: - the sequence on D3D12 (kernel_replay, two passes: l_linear_32_32_fused writes x2, then l_swin_block_c32_enc_nc reads that file); - the sequence on HIP (in this process: l_linear_32_32_relu, l_linear_32_32_f16, l_swin_block_c32_enc_nc); @@ -55,9 +57,18 @@ def cases(rng): if ci % 2: pick = rng.random(q.size) < 0.1 q[pick] = rng.choice(LF.SPECIAL, int(pick.sum())) + q = q.reshape(npix, 32) + q[:, 20:] = 0 yield H, W, off, q.tobytes(), LF._blob(rng, s, b), LF._blob(rng, s, b), blob, nout +def q20(q): + """The 20-byte form of a 32-byte q map: bytes 0..15 of every pixel, then bytes 16..19 of every pixel; 12 bytes of + slack after it (the last pixel's upper lanes load 16 bytes at its place in the second plane).""" + a = np.frombuffer(q, np.uint8).reshape(-1, 32) + return a[:, :16].tobytes() + a[:, 16:20].tobytes() + bytes(12) + + def hip_sequence(g, ops, c): """relu, f16, then the encoder block, on HIP: x3's raw bytes (the output starts at HIP_SENTINEL).""" H, W, off, q, w1, w2, blob, nout = c @@ -90,7 +101,7 @@ def buf(data): def enclin_lines(name, c, prefix="", grid=None): H, W, off, _q, _w1, _w2, _blob, nout = c return [f"case {prefix}{name}_enclin", f"kernel {ENCLIN} {grid or -(-nwin_of(H, W, off) // 8)} 256", - f"arg buf {name}.q", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}", + f"arg buf {name}.q20", "arg null", f"arg out {nout + SLACK} {prefix}{name}.fx {FOLD_SENTINEL}", f"arg buf {name}.w", f"arg u32 {H}", f"arg u32 {W}", f"arg u32 {off[0]}", f"arg u32 {off[1]}", f"arg buf {name}.w1", f"arg buf {name}.w2", ""] @@ -109,7 +120,7 @@ def run(build, rocm, seeds): H, W, off, q, w1, w2, blob, nout = c npix = (H + 4) * (W + 4) all_cases[name] = (nout, hip_sequence(g, ops, c)) - for ext, data in ((".q", q), (".w1", w1), (".w2", w2), (".w", blob)): + for ext, data in ((".q", q), (".q20", q20(q)), (".w1", w1), (".w2", w2), (".w", blob)): write(os.path.join(cdir, name + ext), data) grid = -(-nwin_of(H, W, off) // 8) # pass 1: the D3D12 fused linears (x2) and the fused block diff --git a/clean/l/runtime/tests/test_l_runtime_frame.py b/clean/l/runtime/tests/test_l_runtime_frame.py index c702a087..41f56ddb 100644 --- a/clean/l/runtime/tests/test_l_runtime_frame.py +++ b/clean/l/runtime/tests/test_l_runtime_frame.py @@ -195,6 +195,8 @@ def base(kernel): whose arguments it shares.""" k = kernel.replace("_nc", "") post = ("l_post_recon_p12", "l_post_recon_ldr", "l_post_recon_p12_ldr") # pack12 and LDR forms + if k.startswith("lev/input_encoder") and k.endswith("_q20"): # the q20 forms (fusion le): the same arguments + return k[:-len("_q20")] return "l_coef" if k == "l_coef_pack12" else "l_post_recon" if k in post else k @@ -638,7 +640,8 @@ def test_frame(self): (NO_ENCLIN, full_ports(), True, True, False), (None, without(ENCLIN), True, True, False), (None, without(ENCLIN, LIN), True, False, False), - (None, without(LIN), True, False, True)): + (None, without(LIN), True, False, True), + (None, without("lev/input_encoder_q20"), True, True, False)): with self.subTest(fusion=env, fold_in_manifest=any(kc[0] == FOLD for kc in ports), lin_in_manifest=any(kc[0] == LIN for kc in ports), enclin_in_manifest=any(kc[0] == ENCLIN for kc in ports)): diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index ba3b6879..58fae1da 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -343,7 +343,9 @@ struct Runtime { // l_swin.hlsl SW_SRC 3; byte for byte the linears -> l_swin_block_c32_enc_nc): one launch less, and x2 (64 bytes // a pixel of the halo map) is neither stored nor read back: the block reads q (32 bytes) from l_narrow. Over ll // when both are on. Without le, or with a manifest that lacks the kernel, the linears and the block run - bool fz_le = false, l_le_built = false; + // The block reads q in its 20-byte form, which the q20 input encoder writes (lev/input_encoder_q20 / + // _direct_q20): le needs that kernel too + bool fz_le = false, l_le_built = false, l_q20_built = false, l_q20d_built = false; long long xd_max_pixels = [] { const char* e = std::getenv("WG_D3D12_XD_MAX_PIXELS"); return e && *e ? std::atoll(e) : kXdMaxPixels; @@ -561,6 +563,8 @@ struct Runtime { l_fold_built = std::find(have.begin(), have.end(), "l_swin_block_c32_dec_coef12_nc") != have.end(); l_lin_built = std::find(have.begin(), have.end(), "l_linear_32_32_fused") != have.end(); l_le_built = std::find(have.begin(), have.end(), "l_swin_block_c32_enc_lin_nc") != have.end(); + l_q20_built = std::find(have.begin(), have.end(), "lev/input_encoder_q20") != have.end(); + l_q20d_built = std::find(have.begin(), have.end(), "lev/input_encoder_direct_q20") != have.end(); } #endif if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); @@ -1587,14 +1591,15 @@ struct Runtime { DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; const bool fused = fz_ll && l_lin_built; // (native D3D12 only: l_lin_built) - const bool in_block = fz_le && l_le_built && l_nc[0]; // fusion le: the linears in the block (its _nc form) + // fusion le: the linears in the block (its _nc form), q in the 20-byte form from the q20 input encoder + const bool in_block = fz_le && l_le_built && l_nc[0] && (native_direct_io ? l_q20d_built : l_q20_built); DevPtr& q = fused || in_block ? l_narrow : l_wide; // q's map (fused / in the block: not under x2) if (in_block) { DevPtr none = 0, wb = wdev["enc0_block"]; int ox = 0, oy = 0; l_offsets(false, f, &ox, &oy); const int nwin = ((fw + ox + 3) / 4) * ((fh + oy + 3) / 4); - return launch(native_direct_io ? "lev/input_encoder_direct" : "lev/input_encoder", + return launch(native_direct_io ? "lev/input_encoder_direct_q20" : "lev/input_encoder_q20", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &q, &mnear[nxt], &mfar, &mcenter, &expo, &wv, &p}) && diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 165954b6..5a3e98a8 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -71,6 +71,7 @@ struct Signature { }; const Signature kSignatures[] = { {"input_encoder", "ppppppppppppE"}, // Model M's encoder and Model L's values (q in place of the features) + {"input_encoder_q20", "ppppppppppppE"}, // Model L's values, q in its 20-byte form (native D3D12, fusion le) {"coef_decoder", "ppppppppD"}, // Model M's decoder (also its pack12 form) {"swin_block_*", "ppppiiii"}, // Model M's blocks: x, skip, out, w, H, W, ox, oy {"l_swin_block_c32_dec_coef12_nc", "ppppiiiippppii"}, // the decoder block with l_coef_pack12 folded in: then