Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions clean/kernels/encoder.hip
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,11 @@
#ifndef WG_ENC_AB_STORE
#define WG_ENC_AB_STORE 0 // no motion-surface stores
#endif
// Model L (clean/l, #89): instead of the projection, store the quantized values as a token map [fh/2][fw/2][80]
// (sub-pixel p = dx + 2 dy, value i at byte 20 p + i). Built as its own code object; off: this file's kernel unchanged.
#ifndef WG_ENC_VALUES_OUT
#define WG_ENC_VALUES_OUT 0
#endif

// Per-frame constants, passed by value (all 4-byte fields; must match EncParams in clean/runner/enc0.py).
struct WgEncParams {
Expand Down Expand Up @@ -524,7 +529,7 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
else
pixel_values<kHistRaw>(f, P, U, (int)xs[dx], y, c, px);
quantize(px.v, dx ? t2 >> 8 : t2 & 0xffu, q[dx]);
#if !WG_ENC_AB_PROJ
#if !WG_ENC_AB_PROJ && !WG_ENC_VALUES_OUT
// Values 8..15 go to the partner lane (the other lane half of the same token).
x8[dx][0] = wg_xor16_u(q[dx][2]);
x8[dx][1] = wg_xor16_u(q[dx][3]);
Expand All @@ -534,7 +539,18 @@ WG_FN void encoder_main(const Frame& f, u8* __restrict__ features, u32* __restri
#endif
}

#if WG_ENC_AB_PROJ
#if WG_ENC_VALUES_OUT
// Lane half hh holds sub-pixels 2 hh and 2 hh + 1 (row dy = hh): 40 contiguous bytes of the token at 40 hh.
if (tok < ntok) {
const u32 o = (u32)tok * 80u + hh * 40u;
wg_st64(features, o, (v2u){q[0][0], q[0][1]});
wg_st64(features, o + 8u, (v2u){q[0][2], q[0][3]});
wg_st64(features, o + 16u, (v2u){q[0][4], q[1][0]});
wg_st64(features, o + 24u, (v2u){q[1][1], q[1][2]});
wg_st64(features, o + 32u, (v2u){q[1][3], q[1][4]});
}
(void)w; (void)x8;
#elif WG_ENC_AB_PROJ
if (tok < ntok) {
v4u o0 = {q[0][0], q[0][1], q[0][2], q[0][3]}, o1 = {q[1][0], q[1][1], q[1][2], q[1][3] ^ q[0][4] ^ q[1][4]};
wg_st128(features, (u32)tok * kFeat + hh * 32u, o0);
Expand Down
57 changes: 57 additions & 0 deletions clean/l/FORMATS.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Model L: kernels, blobs and launch order (issue #89)

Status: HIP kernels for Model L's encoder chain, decoder front and C = 32 block; no runtime integration yet. Feature
maps are `[tokens][C]` FP8 E4M3 bytes, channel fastest, as in Model M (`clean/FORMATS.md`, "Feature maps"); a token
map at F/2 has `ntok = (fh / 2) * (fw / 2)` tokens in raster order. Blobs: `clean/l/runner/l_pack.py`
(`pack_model_l`); the ten shared Swin stages use Model M's blobs, kernels and launch plan unchanged.

## Code objects

| File | Source | Kernels |
|---|---|---|
| `l_enc_values.co` | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` (values only) |
| `l_ops.co` | `clean/l/kernels/l_ops.hip` (`-I clean/kernels`) | `l_linear_80_32`, `l_linear_32_32`, `l_project_80_32`, `l_copy_channels` |
| `l_swin.co` | `clean/l/kernels/l_swin.hip` (`-I clean/kernels`) | `l_swin_block_c32`, `l_swin_block_c32_nc`, `l_swin_block_c64` (check only) |

The runtime loads `l_enc_values.co` under the tag `lev` (its kernel is launched as `lev/input_encoder`, next to Model
M's own `input_encoder`); `l_ops.co` and `l_swin.co` are untagged. These are also the names the D3D12 ports will
declare.

All are built with `clean/runner/build.py`'s `device_command` (gfx1201). The build's `-cuid` comes from the output file
name, so compare code objects only between outputs of the same name.

## Kernels

| Kernel | Arguments | Grid (workgroups of 256 lanes) | Does |
|---|---|---|---|
| `input_encoder` of `l_enc_values.co` | as Model M's `input_encoder` (color, depth, motion, hist, latent, mhist, **raw**, m_near, m_far, m_center, expo, w, `WgEncParams`) | ceil(ntok / 128) | the per-pixel values and their dithered FP8 quantization as Model M's enc0; writes `raw` [ntok][80] (sub-pixel p = dx + 2 dy, value i at byte 20 p + i) and the three motion surfaces; w = blob `enc0.values` |
| `l_linear_80_32` | x, y, w, ntok | ceil(ntok / 128) | y [ntok][32] = q8(W x + b), x [ntok][80]; w = blob `enc0.Wa` |
| `l_linear_32_32` | x, y, w, ntok | ceil(ntok / 128) | y [ntok][32] = q8(W x + b); w = blob `enc0.Wb` |
| `l_swin_block_c32[_nc]` | x, xadd (null), out, w, H, W, ox, oy | ceil(nwin / 8), nwin = ceil((W + ox) / 4) ceil((H + oy) / 4) | one Swin block at C = 32 (2 heads, MLP 128), as Model M's block kernels; `_nc` when bit 0 of the blob's first word is set; w = blob `enc0.blk` or `dec0.blk` |
| `l_project_80_32` | raw, x, features, w, ntok | ceil(ntok / 128) | features [ntok][64] = q8(clamp(s, +-6.28125)) with s = b, then s = RN16(s + W[:, g] z[g]) for each group g of 32 channels of z = [raw (80) | x (32) | 16 zeros]; w = blob `enc0.project` |
| `l_copy_channels` | src, sstride, soff, dst, dstride, doff, n, ntok | ceil(ntok n / 16 / 256) | dst[t][doff .. doff + n) = src[t][soff .. soff + n) (bytes; strides, offsets and n multiples of 16) |

## Launch order of one frame (default table)

Model M's frame (`clean/FORMATS.md`, "Stage order per frame") with steps 2 and 4 replaced:

2. Encoder (offsets of the block: table entry `window_offsets["first"]` for the frame parity):
1. `input_encoder` (`l_enc_values.co`) -> `raw` [F/2][80], m_near, m_far, m_center
2. `l_linear_80_32` raw -> `x1` [F/2][32]
3. `l_linear_32_32` x1 -> `x2` [F/2][32]
4. `l_swin_block_c32[_nc]` x2 -> `x3` [F/2][32] (H = fh / 2, W = fw / 2)
5. `l_project_80_32` raw, x3 -> `enc0.features` [F/2][64] (what enc1 reads)
3. enc1 ... dec1: Model M's stages with Model L's weights, unchanged.
4. Decoder (offsets: `window_offsets["second"]`):
1. `l_copy_channels` dec1.out (stride 64, offset 0) -> `y` [F/2][32]
2. `l_swin_block_c32[_nc]` y -> `y2` [F/2][32]
3. `l_copy_channels` y2 (32, 0) -> `z` (64, 0), n = 32; `l_copy_channels` dec1.out (64, 32) -> `z` (64, 32), n = 32
4. Model M's `coef_decoder` with features = `z`, w = blob `dec0`

Extra buffers per frame: `raw` 80 ntok bytes, `x1` / `x2` / `x3` 32 ntok each (two can be shared), `y` / `y2` 32 ntok,
`z` 64 ntok. The launch list is derived from the table by `clean/l/runner/l_pack.py` `encoder_launches`.

## Checks (GPU, synthetic data)

`clean/l/runner/l_c32_check.py` and `clean/l/runner/l_chain_check.py`: every kernel in isolation against the oracle
step on the same input, the whole encoder chain, and the decoder front (see their docstrings for the rules).
4 changes: 3 additions & 1 deletion clean/l/README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Model L (issue #89)

Work in progress for a new model choice, milestone 0.7.0. Nothing here is used by the runtime, the front end or the
launcher yet.
launcher yet. Kernels, blobs and the launch order of a frame: [FORMATS.md](FORMATS.md).

| Path | Content |
|---|---|
Expand All @@ -10,6 +10,8 @@ launcher yet.
| `runner/l_pack.py` | weight packing of Model L's own launches: the encoder's launch list derived from the table (`encoder_launches`) and one blob per launch (`pack_model_l`), each in a format Model M already uses (merge / expand section, Swin block section, enc0 and dec0 blobs); decoders for tests. The ten shared stages use `clean/runner/packing.py` unchanged |
| `kernels/l_swin.hip` | the C = 32 Swin block kernels (`l_swin_block_c32[_nc]`) from Model M's block template (`clean/kernels/swin.hip`, included for its templates only), same blob format and launch geometry as Model M's block kernels |
| `runner/l_c32_check.py` | GPU check (synthetic): the C = 32 kernel against the oracle, `_nc` against the clamping kernel, and this file's C = 64 build of the template against Model M's `swin_block_c64` |
| `kernels/l_ops.hip` | the encoder chain's pointwise linears and output projection, and the channel copy of the decoder front |
| `runner/l_chain_check.py` | GPU check (synthetic): each kernel of the encoder chain and the decoder front against the oracle step on the same input, the whole encoder chain, the decoder front; Model M's encoder.co against a build of main |
| `tests/test_l_pack.py` | packing round trips, the block packer against Model M's stage blobs, and the unpacked weights applied the kernels' way against the oracle's maps, byte for byte |
| `tests/test_l_reference.py` | self-checks on synthetic data; the main ones reduce Model L to Model M through the table and require Model M's oracle results byte for byte |

Expand Down
107 changes: 107 additions & 0 deletions clean/l/kernels/l_ops.hip
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// SPDX-License-Identifier: MIT
// Model L (issue #89): the small FP8-map kernels of Model L's encoder and decoder chains (clean/l/runner/l_pack.py
// encoder_launches). Maps are [tokens][C] FP8 E4M3 bytes, channel fastest (clean/FORMATS.md "Feature maps"). Blobs:
// clean/l/runner/l_pack.py. Launch: 1-D grid of ceil(tokens / 128) workgroups of 256 lanes (8 waves); wave i handles
// tokens 16 i .. 16 i + 15; lane (t, h) works on token 16 i + t, lane half h on channels [h C/2, (h + 1) C/2) (layout L)
// or, for the projection, on the 16 channels of each group that its K slots name.
#include "wg_platform.h"
#include "wg_mem.h"

namespace {

constexpr int kThreads = 256;
constexpr int kTokensPerWave = 16;

WG_FN int token_of(u32 tid) {
return (int)(wg_block_id() * (kThreads / WG_WAVE) + (tid >> 5)) * kTokensPerWave + (int)(tid & 15u);
}

// y = q8(W x + b), FP32 accumulation over the K groups in order. Blob: bias tile vector (rows R_L(CO)), then fragments
// [mt < CO/16][kc < CI/16] with rows R_L(CO) and K order K_L(CI) (merge / expand section format).
template <int CI, int CO>
WG_FN void linear_map(const u8* __restrict__ x, u8* __restrict__ y, const u8* __restrict__ w, int ntok) {
constexpr int KC = CI / 16, MT = CO / 16;
static_assert(CI % 16 == 0 && CO % 16 == 0, "widths in multiples of 16");
const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u;
const int tok = token_of(tid);
const int tk = tok < ntok ? tok : ntok - 1;
v2i xf[KC];
#pragma unroll
for (int kc = 0; kc < KC; ++kc) xf[kc] = *(const v2i*)(x + (size_t)tk * CI + hh * (CI / 2) + 8 * kc);
const float* bias = (const float*)w + hh * 8;
const u8* frag = w + 4 * CO;
#pragma unroll
for (int mt = 0; mt < MT; ++mt) {
v8f acc = wg_load8f(bias + mt * 16);
#pragma unroll
for (int kc = 0; kc < KC; ++kc) acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KC + kc) * 256))[lane], xf[kc], acc);
const v2i q = wg_pack_q8(acc);
if (tok < ntok) *(v2i*)(y + (size_t)tok * CO + hh * (CO / 2) + 8 * mt) = q;
}
}

// 8 bytes of the concatenated input z = [a (CA channels) | b (CB channels) | zeros] at channel c (c a multiple of 8;
// CA and CB are multiples of 8, so the 8 bytes never straddle two parts).
template <int CA, int CB>
WG_FN v2i concat8(const u8* a, const u8* b, int tk, int c) {
if (c < CA) return *(const v2i*)(a + (size_t)tk * CA + c);
if (c < CA + CB) return *(const v2i*)(b + (size_t)tk * CB + (c - CA));
v2i z = {0, 0};
return z;
}

// features = q8(clamp(s, +-6.28125)), s = b; for each group g of 32 channels of z: s = RN16(s + W[:, g] z[g]).
// Blob: bias tile vector (rows R_L(64)), then fragments [mt < 4][kb < NZ/16], rows R_L(64), K order: block kb slot
// (h, j) = channel 32 (kb / 2) + 16 h + 8 (kb % 2) + j (clean/l/runner/l_pack.py kord_groups32).
template <int CA, int CB, int NZ>
WG_FN void project_map(const u8* __restrict__ a, const u8* __restrict__ b, u8* __restrict__ feat,
const u8* __restrict__ w, int ntok) {
constexpr int CO = 64, MT = CO / 16, KB = NZ / 16;
static_assert(NZ % 32 == 0 && CA + CB <= NZ && CA % 8 == 0 && CB % 8 == 0, "groups of 32, parts of 8 bytes");
const u32 tid = wg_lane_id(), lane = tid & 31u, hh = (tid >> 4) & 1u;
const int tok = token_of(tid);
const int tk = tok < ntok ? tok : ntok - 1;
v2i zf[KB];
#pragma unroll
for (int kb = 0; kb < KB; ++kb) zf[kb] = concat8<CA, CB>(a, b, tk, 32 * (kb >> 1) + 16 * (int)hh + 8 * (kb & 1));
const float* bias = (const float*)w + hh * 8;
const u8* frag = w + 4 * CO;
#pragma unroll
for (int mt = 0; mt < MT; ++mt) {
v8f acc = wg_load8f(bias + mt * 16);
#pragma unroll
for (int g = 0; g < KB / 2; ++g) {
acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g) * 256))[lane], zf[2 * g], acc);
acc = wg_wmma_fp8(((const v2i*)(frag + (mt * KB + 2 * g + 1) * 256))[lane], zf[2 * g + 1], acc);
#pragma unroll
for (int i = 0; i < 8; ++i) acc[i] = wg_rn16(acc[i]);
}
#pragma unroll
for (int i = 0; i < 8; ++i) acc[i] = wg_fmin(wg_fmax(acc[i], -6.28125f), 6.28125f);
const v2i q = wg_pack_q8(acc);
if (tok < ntok) *(v2i*)(feat + (size_t)tok * CO + hh * (CO / 2) + 8 * mt) = q;
}
}

} // namespace

// Pointwise linears of Model L's encoder: 80 -> 32 (the raw value map; the select is zero weights) and 32 -> 32.
WG_KERNEL_N(l_linear_80_32, kThreads)(const u8* x, u8* y, const u8* w, int ntok) { linear_map<80, 32>(x, y, w, ntok); }
WG_KERNEL_N(l_linear_32_32, kThreads)(const u8* x, u8* y, const u8* w, int ntok) { linear_map<32, 32>(x, y, w, ntok); }

// The encoder's output projection over [raw (80) | x (32) | 16 zeros] -> the 64 FP8 features enc1 reads.
WG_KERNEL_N(l_project_80_32, kThreads)(const u8* raw, const u8* x, u8* feat, const u8* w, int ntok) {
project_map<80, 32, 128>(raw, x, feat, w, ntok);
}

// dst[t][doff .. doff + n) = src[t][soff .. soff + n) for every token t < ntok, n a multiple of 16 bytes; one lane per
// 16 bytes. Splits dec1's map for the decoder block and joins the block output back (clean/l/runner).
WG_KERNEL_N(l_copy_channels, kThreads)(const u8* src, int sstride, int soff, u8* dst, int dstride, int doff, int n,
int ntok) {
const int chunks = n >> 4;
const long long i = (long long)wg_block_id() * kThreads + wg_lane_id();
const long long t = i / chunks;
if (t >= ntok) return;
const int k = (int)(i - t * chunks);
*(v4u*)(dst + (size_t)t * dstride + doff + 16 * k) = *(const v4u*)(src + (size_t)t * sstride + soff + 16 * k);
}
Loading
Loading