Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion clean/l/FORMATS.md
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ No kernel works in place.
| File | Source | Kernels |
|---|---|---|
| `l_enc_values.co` (loaded under the tag `lev`) | `clean/kernels/encoder.hip` with `-DWG_ENC_VALUES_OUT=1` | `input_encoder` |
| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef` |
| `l_ops.co` | `clean/l/kernels/l_ops.hip` | `l_linear_32_32_relu`, `l_linear_32_32_f16`, `l_project_gather`, `l_dec_project`, `l_coef`, `l_coef_pack12` |
| `l_post.co` | `clean/l/kernels/l_post.hip` (includes Model M's `clean/kernels/post.hip` for its functions; Model M's kernels come along unused) | `l_post_recon` (HDR) |
| `l_swin.co` | `clean/l/kernels/l_swin.hip` | `l_swin_block_c32[_nc]` (exists, #102: FP8 in / FP8 out, mirror border; block checks only), `l_swin_block_c32_enc[_nc]`, `l_swin_block_c32_dec[_nc]` |

Expand All @@ -83,6 +83,7 @@ workgroup, in raster order of their map.
| `l_dec_project` | exists (GPU-checked) | `z, y, w, tw, th` | ceil(tw th / 128) | per token of dec1's output z [th][tw][64]: 128 outputs (two RN16 groups of 32 inputs, then FP8). Output 32 s + k goes to pixel (2r + s / 2, 2c + s % 2), channel k, of `y`. w = `dec0.project` |
| `l_swin_block_c32_dec[_nc]` | exists (GPU-checked) | `x, xadd (null), out, w, H, W, ox, oy` | as the encoder block | the decoder block. x = `y` (FP8). A member outside the grid takes token-grid mirrored pixels with the sub-pixel kept. out = `y2` (**FP16**). (ox, oy) = `window_offsets["second"]`. w = `dec0_block` |
| `l_coef` | exists (GPU-checked) | `y2, A, B, C, latent, w, params` | ceil((fh / 2)(fw / 2) / 128), lanes as Model M's `coef_decoder` | Model L's per-pixel functions on the 13 channels `table.L_DEC_VALUES` of `y2`: A, B, C in Model M's forms, latent = tanh(candidate). Writes the same buffers and formats as `coef_decoder`, pixels inside R only. **Reads no motion, previous latent, jitter or reset.** params = Model M's `WgDecParams` (only rw, rh, fw, fh are read). w = `dec0.coef` |
| `l_coef_pack12` | exists (GPU-checked) | as `l_coef` (`B` unused, may be null) | as `l_coef` | `l_coef`'s values in Model M's dec0 "pack12" layout (`clean/kernels/decoder.hip` `WG_DEC_PACK12`, read by the output stage's `_p12` kernels): image A, 8 bytes per pixel = A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); latent as `l_coef`. Only padding bytes are left out: unpacked, byte-identical to `l_coef` (`l_ops_check.py`) |

## Launch order of one frame

Expand Down
33 changes: 28 additions & 5 deletions clean/l/kernels/l_ops.hip
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,13 @@ WG_KERNEL_N(l_dec_project, kThreads)(const u8* z, u8* y, const u8* w, int tw, in
// Grid as coef_decoder: ceil((fh / 2)(fw / 2) / 128); lane (t, h) of wave i handles token t' = 16 i + t = (r, c), pixels
// (2 c, 2 r + h) and (2 c + 1, 2 r + h). w = blob dec0.coef (Model M's dec0 blob: S, T at byte 320). No motion,
// previous latent, jitter or reset is read.
WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) {
// PACK12 (kernel l_coef_pack12): A, B and C without padding bytes, in Model M's dec0 "pack12" layout
// (clean/kernels/decoder.hip WG_DEC_PACK12, read by the output stage's _p12 kernels): image A, 8 bytes per pixel =
// A0 A1 A2 B0 | B1 B2 C2 (FP16); image C, 4 bytes per pixel = C0 C1 (FP16); B unused. The latent as l_coef. Only
// padding bytes (zero in the base form) are left out: the values are the same.
namespace {
template <bool PACK12>
WG_FN void l_coef_body(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, const LDecParams& P) {
const u32 tid = wg_lane_id(), hh = (tid >> 4) & 1u;
const int tw = P.fw >> 1, ntok = (P.fh >> 1) * tw;
const int tok = item_of(tid);
Expand All @@ -235,10 +241,27 @@ WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent,
l_values((const _Float16*)y2, P.fw, x, y, a);
const LCodes o = l_pixel(a, S, T);
const u32 i = (u32)(y * P.rw + x);
A[i] = o.A;
B[i] = o.B;
C[2 * i] = o.C0;
C[2 * i + 1] = o.C1;
if constexpr (PACK12) {
A[2 * i] = o.A | (o.B << 24);
A[2 * i + 1] = (o.B >> 8) | (o.C1 << 16);
C[i] = o.C0;
(void)B;
} else {
A[i] = o.A;
B[i] = o.B;
C[2 * i] = o.C0;
C[2 * i + 1] = o.C1;
}
latent[i] = o.L;
}
}
} // namespace

WG_KERNEL_N(l_coef, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) {
l_coef_body<false>(y2, A, B, C, latent, w, P);
}

// Same arguments as l_coef; A is the 8-byte image A, C the 4-byte image C, B may be null.
WG_KERNEL_N(l_coef_pack12, kThreads)(const u8* y2, u32* A, u32* B, u32* C, u32* latent, const u8* w, LDecParams P) {
l_coef_body<true>(y2, A, B, C, latent, w, P);
}
84 changes: 82 additions & 2 deletions clean/l/runner/l_ops_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
- l_linear_32_32_f16: RN16(b + Win2 x), FP16;
- l_project_gather: the 2 x 2 gather and the 128 -> 64 projection (running FP16 sum per group of 32, clamp, FP8);
- l_dec_project: the 64 -> 4 x 32 projection scattered to the pixel map;
- l_coef_pack12 (pack12 store form): its images, unpacked, byte-identical to l_coef's A, B, C and latent, with
sentinel-filled buffers (no byte past a buffer written, every pixel inside R written) and a negative control;
- l_coef: Model L's per-pixel functions on an FP16 map against oracle.coefficients_l: stored codes of A, B and the
latent within one step, C within 4 FP16 ulp (the oracle's exponential and tanh are exact; the kernel's use the
hardware operations), with the counts that differ at all.
Expand Down Expand Up @@ -162,6 +164,79 @@ def boundary_cases(g, rng, enc):
return out


SENTINEL = 0xA5
SLACK = 64


def _launch_coef(g, kernel, yb, blob, sizes, P, grid, null_b=False):
"""Run an l_coef form with output buffers of the given byte sizes (A, B, C, latent), each followed by SLACK bytes,
all filled with SENTINEL first. Returns the downloaded buffers including their slack."""
h = g.hip
py, pw = h.malloc(len(yb)), h.malloc(len(blob))
h.upload(py, yb)
h.upload(pw, blob)
outs = []
for n in sizes:
if n is None:
outs.append(None)
continue
p = h.malloc(n + SLACK)
h.memset(p, SENTINEL, n + SLACK)
outs.append(p)
args = [ctypes.c_void_p(py)] + [ctypes.c_void_p(p or 0) for p in outs] + [ctypes.c_void_p(pw), P]
h.launch(h.get_function(g.mod, kernel), grid, 256, args)
h.sync()
res = [None if p is None else np.frombuffer(h.download(p, n + SLACK), np.uint8) for p, n in zip(outs, sizes)]
for p in [py, pw] + [p for p in outs if p]:
h.free(p)
return res


def unpack12(a8, c4):
"""pack12 images (image A: 2 words per pixel, image C: 1 word) -> (A codes word, B codes word, C words [C0C1,
C2]) in the base form's layout (padding bytes 0)."""
w0, w1 = a8[0::2], a8[1::2]
A = w0 & 0xFFFFFF
B = (w0 >> 24) | ((w1 & 0xFFFF) << 8)
C = np.stack([c4, w1 >> 16], axis=-1).reshape(-1)
return A, B, C


def coef_pack12_case(g, rng, dec, fh, fw, rh, rw):
"""l_coef and l_coef_pack12 on the same FP16 map: the pack12 images, unpacked, must equal the base form's A, B,
C and latent byte for byte; no byte past any buffer may be written; every pixel inside R must be written (the base
form's padding bytes are 0, not the sentinel); and a one-bit change in the pack12 image must be caught (negative
control)."""
from clean.runner.dec0 import DecParams
y2 = L.f16(rng.normal(0, 2.0, (fh, fw, 32)))
yb = y2.astype(np.float16).tobytes()
blob = LP.pack_coef(dec)
npx = rh * rw
P = DecParams(rw=rw, rh=rh, fw=fw, fh=fh)
grid = -(-((fh // 2) * (fw // 2)) // 128)
base = _launch_coef(g, "l_coef", yb, blob, (4 * npx, 4 * npx, 8 * npx, 4 * npx), P, grid)
p12 = _launch_coef(g, "l_coef_pack12", yb, blob, (8 * npx, None, 4 * npx, 4 * npx), P, grid)
m = {"case": [fh, fw, rh, rw], "pixels": npx}
m["slack_untouched"] = all(bool((b[-SLACK:] == SENTINEL).all()) for b in base + [p for p in p12 if p is not None])
w = lambda b, n: b[:n].view(np.uint32) # noqa: E731
bA, bB, bC, bL = w(base[0], 4 * npx), w(base[1], 4 * npx), w(base[2], 8 * npx), base[3][:4 * npx]
m["base_all_written"] = bool(((bA >> 24) == 0).all() and ((bB >> 24) == 0).all() and ((bC[1::2] >> 16) == 0).all())
a8, c4, pL = w(p12[0], 8 * npx), w(p12[2], 4 * npx), p12[3][:4 * npx]
uA, uB, uC = unpack12(a8, c4)
m["A_identical"] = bool(np.array_equal(uA, bA))
m["B_identical"] = bool(np.array_equal(uB, bB))
m["C_identical"] = bool(np.array_equal(uC, bC))
m["latent_identical"] = bool(np.array_equal(pL, bL))
bad = a8.copy()
k = int(rng.integers(0, npx))
bad[2 * k + 1] ^= np.uint32(1 << 20) # one bit of C2 of pixel k
nA, nB, nC = unpack12(bad, c4)
m["negative_control_caught"] = not np.array_equal(nC, bC)
m["pass"] = all(m[k_] for k_ in ("slack_untouched", "base_all_written", "A_identical", "B_identical", "C_identical",
"latent_identical", "negative_control_caught"))
return m


def _fp8_metrics(got, ref, case):
m = C32.metrics(got, ref)
m.update({"case": case, "pass": C32.passes(m)})
Expand Down Expand Up @@ -212,6 +287,7 @@ def run(build, rocm, seeds=(89,)):
res.setdefault("boundary", []).extend(boundary_cases(g, rng, enc))
for case in COEF_CASES:
res.setdefault("coef", []).append(coef_case(g, rng, dec, *case))
res.setdefault("coef_pack12", []).append(coef_pack12_case(g, rng, dec, *case))
res["coef_total"] = {k: {"values": sum(m[k]["values"] for m in res["coef"]),
"differ": sum(m[k]["differ"] for m in res["coef"])}
for k in ("codes_A", "codes_B", "C", "codes_L")}
Expand All @@ -220,7 +296,11 @@ def run(build, rocm, seeds=(89,)):
"differ": sum(m["differ"] for m in res[k]),
"max_steps": max(m["max_steps"] for m in res[k])}
res["f16_total"]["f16_differ"] = sum(m["f16_differ"] for m in res["f16"])
res["pass"] = all(m["pass"] for k in ("relu", "f16", "project", "dec_project", "coef", "boundary") for m in res[k])
kinds = ("relu", "f16", "project", "dec_project", "coef", "boundary", "coef_pack12")
p12 = res["coef_pack12"]
res["coef_pack12_total"] = {"cases": len(p12), "pixels": sum(m["pixels"] for m in p12),
"all_pass": all(m["pass"] for m in p12)}
res["pass"] = all(m["pass"] for k in kinds for m in res[k])
return res


Expand All @@ -235,7 +315,7 @@ def main(argv=None):
with open(a.json, "w") as f:
json.dump(res, f, indent=1)
print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"})
for k in ("relu", "f16", "project", "dec_project", "coef", "boundary"):
for k in ("relu", "f16", "project", "dec_project", "coef", "boundary", "coef_pack12"):
for m in res[k]:
if not m["pass"]:
print("FAIL", k, json.dumps(m))
Expand Down
Loading