Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion clean/dx12/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
"expand_c96", "d1p12/swin_block_c64_add_dec0_nc", "w64/post_tf_f32_p12_diet_vsit"}
# The native backend (--backend d3d12) runs the D3D12 fusions of WG_D3D12_FUSION (default: runtime.cpp's
# kDefaultD3d12Fusion, mirrored here), which replace some launches (kernels/fused_swin.hlsl, fused_bneck.hlsl, fused_dec.hlsl).
DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd"
DEFAULT_D3D12_FUSION = "pm,xe,mb,rv,xo,xd,lc" # lc: Model L only


def d3d12_fusion():
Expand Down
75 changes: 75 additions & 0 deletions clean/dx12/hlsl/l_coef_px.hlsli
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// SPDX-License-Identifier: MIT
// Model L's per-pixel coefficient functions (issue #89; HIP l_coef_body of clean/l/kernels/l_ops.hip), shared by
// kernels/l_ops_coef.hlsl (l_coef, l_coef_pack12) and the decoder block that ends in them (kernels/l_swin.hlsl,
// l_swin_block_c32_dec_coef12_nc): one pixel's channels 0..15 of the decoder block's FP16 output (y2) -> A, B, C and
// the latent, stored at pixel index i of the render rectangle. Every FP32 operation follows HIP's ISA of l_coef (see
// l_ops_coef.hlsl). Needs COEF_PACK12 (0: Model M's base layout, 1: pack12) and wg_platform.hlsli; uses g_z, an opaque
// zero the kernel sets from its grid size (g_z = wg_grid_ >> 31).

static uint g_z; // 0, opaque to the compiler (set from the grid size)
float opq(float x) { return asfloat(asuint(x) ^ g_z); }
float bits(uint b) { return asfloat(b); }

// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp
float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); }

// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd
// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure).
float l_tanh(float t)
{
const float a = abs(t);
float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f);
big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask
big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign
const float t2 = t * t;
const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu));
const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque
const float small = mad(p2, t, t);
return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial)
}

uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32
uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; }

// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v)
float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); }

// One pixel: lo / hi = its channels 0..7 / 8..15 (four dwords each, two FP16 per dword, channel 2k in the low half);
// Model L's 13 values are channels a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py
// L_DEC_VALUES). S, T: the coefficient blob's scalars (dec0.coef, bytes 320 and 324). pA, pB, pC, pL: the output
// buffers (pB unused with COEF_PACK12); i: the pixel's index in the render rectangle (y rw + x).
void l_coef_px(uint4 lo, uint4 hi, float S, float T, uint pA, uint pB, uint pC, uint pL, uint i)
{
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
// A: unorm codes of RN16(sigma(a0..a2))
const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) |
(unorm8_code(wg_rn16(l_sigma(a2))) << 16);
// B: softmax of a3..a5
const float m = max(max(a3, a4), a5);
const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)),
e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu));
const float rs = wg_rcp(e2 + (e0 + e1));
const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16);
// C: spreads and correlation
const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7)));
const float sx = S * g0, sy = S * g1;
const float sxy = sx * sy;
const float rho = l_tanh(a8 * T);
const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16);
const uint c1 = wg_mul_f16(sxy, rho);
// latent: snorm codes of tanh(a9..a12)
const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) |
(snorm8_code(l_tanh(a12)) << 24);
#if COEF_PACK12
wg_st64(pA, i * 8u, uint2(cA | (cB << 24), (cB >> 8) | (c1 << 16)));
wg_st32(pC, i * 4u, c0);
#else
wg_st32(pA, i * 4u, cA);
wg_st32(pB, i * 4u, cB);
wg_st64(pC, i * 8u, uint2(c0, c1));
#endif
wg_st32(pL, i * 4u, cL);
}
22 changes: 22 additions & 0 deletions clean/dx12/kernels/fused_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,16 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
#error "SW_SRC / SW_OUT_F16: plain blocks only"
#endif
#ifndef SW_OUT_COEF12
#define SW_OUT_COEF12 0
#endif
#if SW_OUT_COEF12
#if !SW_OUT_F16 || SW_C != 32
#error "SW_OUT_COEF12: Model L's decoder block only (SW_OUT_F16, C = 32)"
#endif
#define COEF_PACK12 1
#include "l_coef_px.hlsli"
#endif
#if SW_SRC == 1
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
// from the FP16 map [H + 4][W + 4][C]
Expand Down Expand Up @@ -1129,6 +1139,18 @@ WG_KERNEL(SW_ENTRY)
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
#if SW_OUT_COEF12
// l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's
// bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them
if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) {
g_z = wg_grid_ >> 31;
const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u);
const uint pA = (uint)wg_uniform((int)coefA), pC = (uint)wg_uniform((int)coefC),
pL = (uint)wg_uniform((int)coefL);
l_coef_px(o16[0], o16[1], S, T, pA, 0u, pC, pL, (uint)(pr * coefRw + pc));
}
return;
#endif
if (dst >= 0) {
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
Expand Down
6 changes: 5 additions & 1 deletion clean/dx12/kernels/fused_swin_gen.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@
#if SW_ECL > 0
uint aux; // unused by the expand kernels (null)
#endif
#if SW_OUT_COEF12
uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused)
int coefRw, coefRh; // the render rectangle
#endif
WG_ARGS_END"""
ARGS_NEW = """#ifndef SW_PM
#define SW_PM 0
Expand Down Expand Up @@ -352,7 +356,7 @@ def main():
if "--check" in sys.argv:
print("fused_swin.hlsl is STALE (run fused_swin_gen.py)")
return 1
with open(DST, "w", encoding="utf-8") as f:
with open(DST, "w", encoding="utf-8", newline="\n") as f: # LF on every OS (.gitattributes: no conversion)
f.write(s)
print("fused_swin.hlsl regenerated")
return 0
Expand Down
67 changes: 5 additions & 62 deletions clean/dx12/kernels/l_ops_coef.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -31,33 +31,9 @@ WG_ARGS_BEGIN
float P_dw, P_dh, P_kx, P_ky, P_jx, P_jy, P_pjx, P_pjy;
WG_ARGS_END

static uint g_z; // 0, opaque to the compiler (set from the grid size)
float opq(float x) { return asfloat(asuint(x) ^ g_z); }
float bits(uint b) { return asfloat(b); }

// sigma(x) = 1 / (1 + 2^(x * -log2 e)): v_mul, v_exp, v_add 1.0, v_rcp
float l_sigma(float x) { return wg_rcp(1.0f + wg_exp2(x * bits(0xBFB8AA3Bu))); }

// tanh(t): for |t| >= 0.6 (and not NaN), 1 - 2 / (2^(2 |t| log2 e) + 1), 1 from about 9.01 on, sign of t; below, the odd
// polynomial. Both sides are computed; the ISA's branch selects one, select() here (both are pure).
float l_tanh(float t)
{
const float a = abs(t);
float big = mad(wg_rcp(wg_exp2(a * bits(0x4038AA3Bu)) + 1.0f), -2.0f, 1.0f);
big = select(bits(0x41102CB4u) > a || isnan(a), big, 1.0f); // v_cmp_nle 9.01, |t|; v_cndmask
big = asfloat((asuint(big) & 0x7fffffffu) | (asuint(t) & 0x80000000u)); // v_bfi copysign
const float t2 = t * t;
const float p = mad(mad(mad(bits(0x3C80F082u), t2, bits(0xBD563CAEu)), t2, bits(0x3E085941u)), t2, bits(0xBEAAA9EDu));
const float p2 = mad(p, t2, asfloat(g_z)); // fma(p, t2, +0), the zero opaque
const float small = mad(p2, t, t);
return select(a >= bits(0x3F19999Au), big, small); // v_cmp_le 0.6, |t| (false for a NaN: the polynomial)
}

uint unorm8_code(float v) { return wg_f2u(wg_rint(v * 255.0f)); } // v in [0, 1]: v_mul 255, v_rndne, v_cvt_u32
uint snorm8_code(float v) { return (uint)wg_f2i(wg_rint(wg_med3f(v, -1.0f, 1.0f) * 127.0f)) & 0xffu; }

// the clamp of a spread input (an FP16 value) to +-20, NaN kept: HIP's v >= 20 ? 20 : (v <= -20 ? -20 : v)
float clamp20(float v) { return select(v >= 20.0f, 20.0f, select(v <= -20.0f, -20.0f, v)); }
// the per-pixel functions (sigma, tanh, the codes, the clamp, the stores): hlsl/l_coef_px.hlsli, shared with the decoder
// block that ends in this step (l_swin.hlsl, l_swin_block_c32_dec_coef12_nc)
#include "l_coef_px.hlsli"

#if COEF_PACK12
WG_KERNEL(l_coef_pack12)
Expand All @@ -80,42 +56,9 @@ WG_KERNEL(l_coef)
[unroll] for (int dx = 0; dx < 2; ++dx) {
const int x = 2 * c + dx;
if (x >= P_rw) continue;
// channels 0..15 of the pixel (8 dwords, two FP16 each); Model L's 13 values are channels
// a0..a12 = 0, 1, 6, 7, 8, 9, 10, 11, 12, 2, 3, 4, 5 (clean/l/reference/table.py L_DEC_VALUES)
// channels 0..15 of the pixel (8 dwords, two FP16 each)
const uint off = (uint)(y * P_fw + x) * 64u;
const uint4 lo = wg_ld128(y2, off), hi = wg_ld128(y2, off + 16u);
const float a0 = wg_h2f_lo(lo.x), a1 = wg_h2f_hi(lo.x), a2 = wg_h2f_lo(lo.w), a3 = wg_h2f_hi(lo.w);
const float a4 = wg_h2f_lo(hi.x), a5 = wg_h2f_hi(hi.x), a6 = wg_h2f_lo(hi.y), a7 = wg_h2f_hi(hi.y);
const float a8 = wg_h2f_lo(hi.z);
const float a9 = wg_h2f_lo(lo.y), a10 = wg_h2f_hi(lo.y), a11 = wg_h2f_lo(lo.z), a12 = wg_h2f_hi(lo.z);
// A: unorm codes of RN16(sigma(a0..a2))
const uint cA = unorm8_code(wg_rn16(l_sigma(a0))) | (unorm8_code(wg_rn16(l_sigma(a1))) << 8) |
(unorm8_code(wg_rn16(l_sigma(a2))) << 16);
// B: softmax of a3..a5
const float m = max(max(a3, a4), a5);
const float e0 = wg_exp2((a3 - m) * bits(0x3FB8AA3Bu)), e1 = wg_exp2((a4 - m) * bits(0x3FB8AA3Bu)),
e2 = wg_exp2((a5 - m) * bits(0x3FB8AA3Bu));
const float rs = wg_rcp(e2 + (e0 + e1));
const uint cB = unorm8_code(opq(e0 * rs)) | (unorm8_code(opq(e1 * rs)) << 8) | (unorm8_code(opq(e2 * rs)) << 16);
// C: spreads and correlation
const float g0 = wg_rn16(l_sigma(clamp20(a6))), g1 = wg_rn16(l_sigma(clamp20(a7)));
const float sx = S * g0, sy = S * g1;
const float sxy = sx * sy;
const float rho = l_tanh(a8 * T);
const uint c0 = wg_mul_f16(sx, sx) | (wg_mul_f16(sy, sy) << 16);
const uint c1 = wg_mul_f16(sxy, rho);
// latent: snorm codes of tanh(a9..a12)
const uint cL = snorm8_code(l_tanh(a9)) | (snorm8_code(l_tanh(a10)) << 8) | (snorm8_code(l_tanh(a11)) << 16) |
(snorm8_code(l_tanh(a12)) << 24);
const uint i = (uint)(y * P_rw + x);
#if COEF_PACK12
wg_st64(pA, i * 8u, uint2(cA | (cB << 24), (cB >> 8) | (c1 << 16)));
wg_st32(pC, i * 4u, c0);
#else
wg_st32(pA, i * 4u, cA);
wg_st32(pB, i * 4u, cB);
wg_st64(pC, i * 8u, uint2(c0, c1));
#endif
wg_st32(pL, i * 4u, cL);
l_coef_px(lo, hi, S, T, pA, pB, pC, pL, (uint)(y * P_rw + x));
}
}
5 changes: 5 additions & 0 deletions clean/dx12/kernels/l_swin.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -12,4 +12,9 @@
// @wg-kernel l_swin_block_c32_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=0,SW_ENTRY_OVERRIDE=l_swin_block_c32_nc
// @wg-kernel l_swin_block_c32_enc_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_enc_nc
// @wg-kernel l_swin_block_c32_dec_nc co=l_swin.co block=256 wave=32 args=p,p,p,p,u,u,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_nc
// l_swin_block_c32_dec_coef12_nc D3D12 only: the decoder block with l_coef_pack12 as its epilogue (SW_OUT_COEF12;
// byte for byte the pair _dec_nc -> l_coef_pack12, which stays the fallback route);
// the FP16 map is not written. Arguments: _dec_nc's (out unused), then A, C, latent,
// dec0.coef, rw, rh
// @wg-kernel l_swin_block_c32_dec_coef12_nc co=- block=256 wave=32 args=p,p,p,p,u,u,u,u,p,p,p,p,u,u defines=SW_C=32,SW_HEADS=2,SW_MLP=128,SW_ADD=0,SW_SAFE=1,SW_ECL=0,SW_HPU=8,SW_GELU=1,SW_SRC=2,SW_OUT_F16=1,SW_OUT_COEF12=1,SW_ENTRY_OVERRIDE=l_swin_block_c32_dec_coef12_nc
#include "swin_block.hlsl"
29 changes: 29 additions & 0 deletions clean/dx12/kernels/swin_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
// (pr + 2, pc + 2) (LHaloF16Src); 2 Model L's decoder block, the FP8 map with the token-grid
// mirror, sub-pixel kept (LCoarseFp8Src)
// SW_OUT_F16 1: the output is RN16(x2) as an FP16 map [H][W][C] (Model L's decoder block, LStoreF16Epi)
// SW_OUT_COEF12 1 (with SW_OUT_F16, C = 32): instead of the FP16 map, l_coef_pack12 of it as the epilogue:
// the lane that holds a pixel's channels 0..15 (hh = 0) runs hlsl/l_coef_px.hlsli on those
// FP16 values, the bytes the map would hold, for the pixels inside the render rectangle
// HIP ISA: <build>\dx12-m\isa\swin_block_c64_nc.s, swin_block_c96_nc.s, swin_block_c96_add_ex_nc.s.
// Every FP32 operation follows that ISA: the RMSNorm sums of squares pair as fma(a_even, a_even, a_odd * a_odd); the
// soft cap is fma(c^3, -1 / (3 cap^2), c), the max is over t * (tau log2 e) but the exponent is fma(t, tau log2 e, -max);
Expand All @@ -30,6 +33,10 @@ WG_ARGS_BEGIN
#if SW_ECL > 0
uint aux; // unused by the expand kernels (null)
#endif
#if SW_OUT_COEF12
uint coefA, coefC, coefL, coefW; // pointers: l_coef_pack12's A, C, latent and coefficient blob (out_p unused)
int coefRw, coefRh; // the render rectangle
#endif
WG_ARGS_END

// ---- BlockLayout<C, HEADS, MLP> (FORMATS.md, "Swin block section") ---------------------------------------------------
Expand Down Expand Up @@ -403,6 +410,16 @@ void load_frags_g(uint base, uint off, out uint2 f[KC])
#if (SW_SRC != 0 || SW_OUT_F16) && (SW_ADD || SW_ECL > 0)
#error "SW_SRC / SW_OUT_F16: plain blocks only"
#endif
#ifndef SW_OUT_COEF12
#define SW_OUT_COEF12 0
#endif
#if SW_OUT_COEF12
#if !SW_OUT_F16 || SW_C != 32
#error "SW_OUT_COEF12: Model L's decoder block only (SW_OUT_F16, C = 32)"
#endif
#define COEF_PACK12 1
#include "l_coef_px.hlsli"
#endif
#if SW_SRC == 1
// LHaloF16Src: the lane's channels hh * C/2 + 8 k + j of the member at grid position (pr, pc) (inside the 2-pixel halo),
// from the FP16 map [H + 4][W + 4][C]
Expand Down Expand Up @@ -963,6 +980,18 @@ WG_KERNEL(SW_ENTRY)
// Output RN16(x2) as FP16, real window members only (HIP LStoreF16Epi).
uint4 o16[KC];
[unroll] for (uint kf = 0; kf < KC; ++kf) o16[kf] = wg_to_f16(acc[kf]);
#if SW_OUT_COEF12
// l_coef_pack12 of the map: o16[0] / o16[1] of the hh = 0 lane are the pixel's channels 0..7 / 8..15 (the map's
// bytes dst * 64 + 0 .. 31, its lo / hi loads); Model L's 13 values are all among them
if (dst >= 0 && hh == 0u && pr < coefRh && pc < coefRw) {
g_z = wg_grid_ >> 31;
const float S = wg_ldf(coefW, 320u), T = wg_ldf(coefW, 324u);
const uint pA = (uint)wg_uniform((int)coefA), pC = (uint)wg_uniform((int)coefC),
pL = (uint)wg_uniform((int)coefL);
l_coef_px(o16[0], o16[1], S, T, pA, 0u, pC, pL, (uint)(pr * coefRw + pc));
}
return;
#endif
if (dst >= 0) {
const uint ob16 = ((uint)dst * SW_C + hh * HALF) * 2u;
[unroll] for (uint kg = 0; kg < KC; ++kg) wg_st128(out_p, ob16 + 16u * kg, o16[kg]);
Expand Down
3 changes: 2 additions & 1 deletion clean/dx12/tests/native_direct_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
POST='w64/post_tf_f32_p12_diet_vsit'


# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd): enc2's block
# The neural launches with the native D3D12 fusions of runtime.cpp kDefaultD3d12Fusion (pm,xe,mb,rv,xo,xd; its lc is
# Model L's): enc2's block
# with enc1's merge in reverse order (pm, rv), the bottleneck with enc2's merge and the overlap counters (mb, xo), dec2's
# block with dec1's expand (xe) launched as xd on frames of at most kXdMaxPixels render pixels (WG_D3D12_XD_MAX_PIXELS)
# else xo, and the fused dec1 + dec0 in its xd form on those frames. Set by main from --shape (COMMON above: the
Expand Down
1 change: 1 addition & 0 deletions clean/dx12/vgpr_limits.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"l_swin_block_c32_nc": 96,
"l_swin_block_c32_enc_nc": 96,
"l_swin_block_c32_dec_nc": 96,
"l_swin_block_c32_dec_coef12_nc": 96,
"swin_block_c64_nc": 192,
"swin_block_c64_add_ex_nc": 128,
"swin_block_c96_nc": 192,
Expand Down
3 changes: 3 additions & 0 deletions clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ workgroup, in raster order of their map.
| `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` |
| `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` |
| `l_coef_pack12` | exists (GPU-checked) | as `l_coef` (`B` unused, may be null) | as `l_coef` | `l_coef`'s values in Model M's dec0 "pack12" layout (`clean/kernels/decoder.hip` `WG_DEC_PACK12`, read by the output stage's `_p12` kernels): image A, 8 bytes per pixel = A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); latent as `l_coef`. Only padding bytes are left out: unpacked, byte-identical to `l_coef` (`l_ops_check.py`) |
| `l_swin_block_c32_dec_coef12_nc` | exists (GPU-checked), D3D12 only | the decoder block's, `out` unused (null), then `A, C, latent, w_coef, rw, rh` | as the decoder block | the decoder block with `l_coef_pack12` as its epilogue: the same bytes in `A`, `C` and `latent`, `y2` not written. w = `dec0_block`, w_coef = `dec0.coef`. The native D3D12 frame runs it in place of the two (fusion `lc`, on by default); without `lc` or without the kernel in the build, the two run |

## Launch order of one frame

Expand Down Expand Up @@ -156,4 +157,6 @@ the frame parity, as Model M's.
- `clean/l/runner/l_ops_check.py`: the linears, both projections and `l_coef` against the oracle steps.
- `clean/l/runner/l_enc_check.py`: `lev/input_encoder` (halo map and motion surfaces) against the oracle.
- `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte.
- `clean/l/runner/l_fold_check.py`: `l_swin_block_c32_dec_coef12_nc` against `l_swin_block_c32_dec_nc` ->
`l_coef_pack12` on D3D12 and on HIP, byte for byte.
- One check per new kernel follows with that kernel.
Loading
Loading