Skip to content
Merged
53 changes: 47 additions & 6 deletions clean/kernels/encoder.hip
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,15 @@
#ifndef WG_ENC_AB_STORE
#define WG_ENC_AB_STORE 0 // no motion-surface stores
#endif
// Model L (clean/l/FORMATS.md, `lev/input_encoder`): no projection. The kernel runs on the padded grid plus a halo of
// WG_ENC_HALO pixels per side and stores each pixel's 20 dithered FP8 values into the halo map `features`
// [fh + 2 halo][fw + 2 halo][32] (bytes 20..31 zero). The values come from the mirrored render position, and the dither
// from the unmirrored one (index |y| mod 16, |x| mod 16). Motion surfaces are written only for pixels inside R. Built
// as its own code object; off by default: this file's kernel is unchanged.
#ifndef WG_ENC_VALUES_OUT
#define WG_ENC_VALUES_OUT 0
#endif
#define WG_ENC_HALO (WG_ENC_VALUES_OUT ? 2 : 0)

// Per-frame constants, passed by value (all 4-byte fields; must match EncParams in clean/runner/enc0.py).
struct WgEncParams {
Expand Down Expand Up @@ -390,12 +399,16 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
const u8* __restrict__ w, const WgEncParams& P, const MoT mo = MoT()) {
const u32 tid = wg_lane_id();
const u32 lane = tid & 31u, t = lane & 15u, hh = lane >> 4;
#if WG_ENC_VALUES_OUT
const int Wt = (P.fw + 2 * WG_ENC_HALO) >> 1, ntok = ((P.fh + 2 * WG_ENC_HALO) >> 1) * Wt;
#else
const int Wt = P.fw >> 1, ntok = (P.fh >> 1) * Wt;
#endif
const int tok = (int)(wg_block_id() * (kEncThreads / WG_WAVE) + (tid >> 5)) * kTokensPerWave + (int)t;
const int tk = tok < ntok ? tok : ntok - 1;
const int r = tk / Wt, cc = tk - r * Wt;

#if !WG_ENC_FRAG_GLOBAL
#if !WG_ENC_FRAG_GLOBAL && !WG_ENC_VALUES_OUT
// Stage the projection weights in LDS; the copy overlaps the per-pixel work, the barrier comes after it.
WG_LDS __attribute__((aligned(16))) u8 lds[kFragBytes];
for (int i = (int)tid; i < kFragBytes / 16; i += kEncThreads)
Expand All @@ -422,7 +435,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
}

const int rw = P.rw, rh = P.rh;
const int X0 = 2 * cc, Y = 2 * r + (int)hh;
const int X0 = 2 * cc - WG_ENC_HALO, Y = 2 * r + (int)hh - WG_ENC_HALO;
const int y = reflect(Y, rh);
const int x0 = reflect(X0, rw), x1 = reflect(X0 + 1, rw);
const bool rev = x0 > x1; // right padding: slot 0 is pixel B
Expand Down Expand Up @@ -456,17 +469,33 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri

// Per slot (sub-pixel dx of this lane): render column, near / far pixel indices, centre motion, colour, dither.
const u32 xs[2] = {(u32)x0, (u32)x1};
const u32 ins[2] = {rev ? inB : inA, rev ? inA : inB}, ifs[2] = {rev ? ifB : ifA, rev ? ifA : ifB};
#if WG_ENC_VALUES_OUT
// Halo positions far right of a small render size can mirror (and clamp) onto one render column: x0 == x1 (then 0).
// Both slots are that pixel: slot 1 takes slot 0's depth neighbourhood and centre motion (it keeps its own dither).
const bool same = x0 == x1;
#else
constexpr bool same = false;
#endif
const u32 ins[2] = {rev ? inB : inA, same ? inA : (rev ? inA : inB)};
const u32 ifs[2] = {rev ? ifB : ifA, same ? ifA : (rev ? ifA : ifB)};
#if WG_ENC_MOTION_F16
const v2u mh = wg_ld64(f.motion, (rc + (u32)xa) * 4u);
const v4f mcab = {wg_h2f_lo(mh[0]), wg_h2f_hi(mh[0]), wg_h2f_lo(mh[1]), wg_h2f_hi(mh[1])};
#else
const v4f mcab = __builtin_bit_cast(v4f, wg_ld128(f.motion, (rc + (u32)xa) * 8u));
#endif
const float mcs[2][2] = {{rev ? mcab[2] : mcab[0], rev ? mcab[3] : mcab[1]},
{rev ? mcab[0] : mcab[2], rev ? mcab[1] : mcab[3]}};
{same ? mcab[0] : (rev ? mcab[0] : mcab[2]), same ? mcab[1] : (rev ? mcab[1] : mcab[3])}};
#if WG_ENC_VALUES_OUT
// Dither word per pixel at the unmirrored position: |y| mod 16, |x| mod 16 (X0 may be negative here).
const u8* trow = w + kOffDither + 256u * (u32)P.frame + (((u32)(Y < 0 ? -Y : Y) & 15u) << 4);
const u32 t2 = (u32)trow[(u32)(X0 < 0 ? -X0 : X0) & 15u] |
((u32)trow[(u32)(X0 + 1 < 0 ? -(X0 + 1) : X0 + 1) & 15u] << 8);
const bool store = tok < ntok && Y >= 0 && Y < rh && X0 >= 0;
#else
const u32 t2 = wg_ld16(w, (u32)kOffDither + 256u * (u32)P.frame + (((u32)Y & 15u) << 4) + ((u32)X0 & 15u));
const bool store = tok < ntok && Y < rh;
#endif
const u32 so = (u32)(Y * rw + X0) * 4u;

// Motions of both slots, and the motion surfaces of the real pixels: one 64-bit store per surface when both pixels
Expand Down Expand Up @@ -524,7 +553,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
else
pixel_values<kHistRaw>(f, P, U, (int)xs[dx], y, c, px);
quantize(px.v, dx ? t2 >> 8 : t2 & 0xffu, q[dx]);
#if !WG_ENC_AB_PROJ
#if !WG_ENC_AB_PROJ && !WG_ENC_VALUES_OUT
// Values 8..15 go to the partner lane (the other lane half of the same token).
x8[dx][0] = wg_xor16_u(q[dx][2]);
x8[dx][1] = wg_xor16_u(q[dx][3]);
Expand All @@ -534,7 +563,19 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
#endif
}

#if WG_ENC_AB_PROJ
#if WG_ENC_VALUES_OUT
// Pixel (X0 + dx, Y) of the halo map: 20 value bytes and 12 zero bytes.
if (tok < ntok) {
const u32 mw = (u32)(P.fw + 2 * WG_ENC_HALO);
#pragma unroll
for (int dx = 0; dx < 2; ++dx) {
const u32 o = ((u32)(Y + WG_ENC_HALO) * mw + (u32)(X0 + dx + WG_ENC_HALO)) * 32u;
wg_st128(features, o, (v4u){q[dx][0], q[dx][1], q[dx][2], q[dx][3]});
wg_st128(features, o + 16u, (v4u){q[dx][4], 0u, 0u, 0u});
}
}
(void)x8;
#elif WG_ENC_AB_PROJ
if (tok < ntok) {
v4u o0 = {q[0][0], q[0][1], q[0][2], q[0][3]}, o1 = {q[1][0], q[1][1], q[1][2], q[1][3] ^ q[0][4] ^ q[1][4]};
wg_st128(features, (u32)tok * kFeat + hh * 32u, o0);
Expand Down
27 changes: 21 additions & 6 deletions clean/kernels/swin.hip
Original file line number Diff line number Diff line change
Expand Up @@ -452,8 +452,16 @@ struct SwinStoreEpi {
static constexpr bool kFused = false;
};

// Gather source hook (SRC): the default reads the FP8 map x [H][W][C] at the mirrored member position. A custom SRC
// (kCustom; Model L, clean/l/kernels/l_swin.hip) provides SRC::load<C, KC>(x, pr, pc, H, W, hh, f): the FP32 values
// of the lane's channels hh * C/2 + 8 k + j (k < KC) of the member at grid position (pr, pc), which may lie outside
// the grid. Plain blocks only (no ADD, merge or expand).
struct SwinFp8Src {
static constexpr bool kCustom = false;
};

template <int C, int HEADS, int MLP, int WPW, bool ADD, int GROUP, bool SAFE, int THREADS, int MCO = 0, int ECL = 0,
class EPI = SwinStoreEpi, bool RESIDENT = false>
class EPI = SwinStoreEpi, bool RESIDENT = false, class SRC = SwinFp8Src>
WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8* __restrict__ out,
const u8* __restrict__ w, int H, int W, int ox, int oy, u8* __restrict__ aux WG_PROF_ARG_DECL,
const void* epi = nullptr, int nwg = 0) {
Expand All @@ -464,6 +472,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
// copy. Same arithmetic per window as the staged kernel (bit-identical).
constexpr int RES = L::SIZE - L::STREAM;
static_assert(!RESIDENT || (MCO == 0 && ECL == 0 && !EPI::kFused && WPW == 1), "resident: plain block kernels only");
static_assert(!SRC::kCustom || (MCO == 0 && ECL == 0 && !ADD), "custom gather source: plain block kernels only");
WG_LDS __attribute__((aligned(16))) u8 lds[RESIDENT ? (RES + 15) / 16 * 16 : 2 * MAXU];
Stager<MAXU, THREADS> stage;
const u32 tid = wg_lane_id();
Expand Down Expand Up @@ -493,6 +502,7 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
int dst[WPW];
int eslot[WPW]; // fused expand: LDS record of this token
v2i xa[WPW][KC], xb[WPW][ADD ? KC : 1];
v8f xf[WPW][SRC::kCustom ? KC : 1];
if constexpr (ECL > 0) fused_expand<C, ECL>(x, w + TAIL, lds + MAXU, H, W, nwx, nwin, ox, oy, tid, (int)wg_block_id());
#pragma unroll
for (int i = 0; i < WPW; ++i) {
Expand All @@ -518,8 +528,12 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
if (ADD) xb[i][k] = xa[i][k];
}
#else
if constexpr (ECL == 0) load_frags<KC>(x + src, xa[i]);
if (ADD) load_frags<KC>(xadd + src, xb[i]);
if constexpr (SRC::kCustom) {
SRC::template load<C, KC>(x, pr, pc, H, W, hh, xf[i]);
} else {
if constexpr (ECL == 0) load_frags<KC>(x + src, xa[i]);
if (ADD) load_frags<KC>(xadd + src, xb[i]);
}
#endif
}

Expand Down Expand Up @@ -559,11 +573,12 @@ WG_FN void swin_block(const u8* __restrict__ x, const u8* __restrict__ xadd, u8*
float ss = 0.0f;
#pragma unroll
for (int k = 0; k < KC; ++k) {
acc[i][k] = wg_unpack_fp8(xa[i][k]);
if constexpr (SRC::kCustom) acc[i][k] = xf[i][k];
else acc[i][k] = wg_unpack_fp8(xa[i][k]);
if (ADD) acc[i][k] += wg_unpack_fp8(xb[i][k]);
if (!(WG_NORM_DOT4 && !ADD)) ss += hsum8(acc[i][k] * acc[i][k]);
if (!(WG_NORM_DOT4 && !ADD && !SRC::kCustom)) ss += hsum8(acc[i][k] * acc[i][k]);
}
if (WG_NORM_DOT4 && !ADD) {
if (WG_NORM_DOT4 && !ADD && !SRC::kCustom) {
#pragma unroll
for (int k = 0; k < KC; ++k) {
ss = wg_dot4_fp8(xa[i][k][0], xa[i][k][0], ss);
Expand Down
22 changes: 12 additions & 10 deletions clean/l/FORMATS.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Model L: kernels, maps, blobs and launch order (issue #89)

This is the interface the Model L runtime frame builds against.
- A row marked **exists** has a merged HIP kernel.
- A row marked **exists** has a HIP kernel that passed its GPU check (merged, or in the open Model L kernel PR).
- A row marked **planned** is the interface the kernel is being written to. Its names, arguments, grids and maps are
fixed here, and a change goes through this file first.
- Each D3D12 port declares the same code object, kernel name and arguments as its HIP kernel (`clean/dx12/kernels/l_*.hlsl`).
Expand Down Expand Up @@ -74,14 +74,14 @@ workgroup, in raster order of their map.

| Kernel | Status | Arguments | Grid | Does |
|---|---|---|---|---|
| `lev/input_encoder` | planned (rework of the old values-only variant) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` |
| `l_linear_32_32_relu` | planned | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` |
| `l_linear_32_32_f16` | planned | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` |
| `l_swin_block_c32_enc[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` |
| `l_project_gather` | planned | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` |
| `l_dec_project` | planned | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
| `l_swin_block_c32_dec[_nc]` | planned | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` |
| `l_coef` | planned (new) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` |
| `lev/input_encoder` | exists (GPU-checked) | as Model M's `input_encoder`, with `features` replaced by `q` | ceil(((fh + 4) / 2) ((fw + 4) / 2) / 128) (2 x 2 pixel groups of the halo map) | the 20 values and their dithered FP8 for every halo-map pixel, into `q`. The motion surfaces as Model M, only for pixels inside R. w = `enc0.values` |
| `l_linear_32_32_relu` | exists (GPU-checked) | `x, y, w, npix` | ceil(npix / 128), npix = (fh + 4)(fw + 4) | y = q8(max(0, RN16(b + W x))), FP8 -> FP8. w = `enc0.Win1` |
| `l_linear_32_32_f16` | exists (GPU-checked) | `x, y, w, npix` | as above | y = RN16(b + W x), FP8 -> **FP16**. w = `enc0.Win2` |
| `l_swin_block_c32_enc[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | the encoder block. x = `x2` (FP16 halo map; member at grid (gx, gy) read at map pixel (gy + 2, gx + 2)). out = `x3` (FP8, grid only). H = fh, W = fw. (ox, oy) = `window_offsets["first"]` for the frame parity. w = `enc0_block` |
| `l_project_gather` | exists (GPU-checked) | `x, features, w, tw, th` | ceil(tw th / 128), tw = fw / 2, th = fh / 2 | token (r, c) gathers pixels (2r + p / 2, 2c + p % 2), p = 0..3, as channels 32 p + i. Running FP16 sum per group of 32 from the bias, clamp +-6.28125, FP8 -> `enc0.features`. w = `enc0.project` |
| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
| `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` |
| `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` |

## Launch order of one frame

Expand All @@ -105,6 +105,8 @@ The reconstruction (post) stage is still open.

## Checks (GPU, synthetic data)

- `clean/l/runner/l_c32_check.py`: the C = 32 block against the oracle.
- `clean/l/runner/l_c32_check.py`: the C = 32 blocks (`l_swin_block_c32`, `_enc`, `_dec`) against the oracle.
- `clean/l/runner/l_ops_check.py`: the linears, both projections and `l_coef` against the oracle steps.
- `clean/l/runner/l_enc_check.py`: `lev/input_encoder` (halo map and motion surfaces) against the oracle.
- `clean/l/runner/l_d3d12_check.py` (with its D3D12 port): each D3D12 port against its HIP kernel, byte for byte.
- One check per new kernel follows with that kernel.
Loading
Loading