Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 210 additions & 13 deletions clean/k/dx12/hlsl/k_block.hlsli

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions clean/k/dx12/hlsl/k_head_rows.inc
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
// SPDX-License-Identifier: MIT
// Physical row for each logical d0 head channel. Unused rows are -1.
0, 1, -1, 6, 7, 16, 17, 18, -1, 2,
8, 9, -1, 19, 20, 21, 22, 23, -1, 10,
3, 4, -1, 14, 15, 24, 25, 26, -1, 5,
11, 12, -1, 27, 28, 29, 30, 31, -1, 13
73 changes: 73 additions & 0 deletions clean/k/dx12/k_plan_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,11 @@
import ctypes
import json
import os
from pathlib import Path
import struct

from clean.k.runner.k_pack import block_sections, linear_sections, stage_sections

HIP_SENTINEL, D3D12_SENTINEL, SLACK = 0xA5, 0x5A, 64


Expand Down Expand Up @@ -43,6 +46,76 @@ def __getattr__(self, name):
return getattr(self.hip, name)


def head_rows():
"""Physical rows for the 40 logical channels, shared with the C++ packer and shader."""
path = Path(__file__).with_name("hlsl") / "k_head_rows.inc"
values = " ".join(line.split("//", 1)[0] for line in path.read_text(encoding="utf-8").splitlines())
return tuple(int(value) for value in values.split(","))


def _bias32(half):
"""Widen FP16 bits as k_bias32_bits does, including NaN sign and payload."""
payload = half & 0x3FF
if half & 0x7C00 == 0x7C00 and payload:
return (half & 0x8000) << 16 | 0x7FC00000 | payload << 13
return struct.unpack("<I", struct.pack("<f", struct.unpack("<e", struct.pack("<H", half))[0]))[0]


def narrow_d3d12_inputs(st, blob):
"""Return the FP32 bias plane and optional compact d0 weights for the two-window ports."""
C, heads, mix = st["C"], st["heads"], st["mix"]
stage = stage_sections(st)
block = block_sections(C, heads, mix)
bias = bytearray()

def append(data, offset, count):
for (half,) in struct.iter_unpack("<H", data[offset:offset + count * 2]):
bias.extend(struct.pack("<I", _bias32(half)))

if "embed" in stage:
append(blob, stage["embed"] + linear_sections(C, 16)["b"], C)
if "expand" in stage:
append(blob, stage["expand"] + linear_sections(4 * C, st["C_prev"])["b"], 4 * C)
for name, count in (("bo", C), ("b1", 4 * C), ("b2", C)):
append(blob, stage["block"] + block[name], count)
if "merge" in stage:
append(blob, stage["merge"] + linear_sections(st["C_next"], 4 * C)["b"], st["C_next"])
compact = None
if "head" in stage:
original = blob[stage["head"]:]
old_bias = linear_sections(40, C)["b"]
compact = bytearray(linear_sections(32, C)["b"])
head_bias = bytearray(32 * 2)
tiles = C // 16
for logical, physical in enumerate(head_rows()):
if physical < 0:
continue
for tile in range(tiles):
for half_tile in (0, 16):
src = 512 * (logical // 16 * tiles + tile) + 16 * (half_tile + logical % 16)
dst = 512 * (physical // 16 * tiles + tile) + 16 * (half_tile + physical % 16)
compact[dst:dst + 16] = original[src:src + 16]
head_bias[physical * 2:physical * 2 + 2] = original[old_bias + logical * 2:old_bias + logical * 2 + 2]
append(head_bias, 0, 32)
if mix == "softmax":
append(blob, stage["block"] + block["mix"], heads * 64 * 64)
return bytes(bias), None if compact is None else bytes(compact)


def narrow_d3d12_args(plan, pdev, st, blob):
"""Allocate the D3D12-only inputs and return their arguments in manifest order."""
bias, compact = narrow_d3d12_inputs(st, blob)
args = []
for name, data in (("bias32", bias), ("compact_head", compact)):
if data is None:
continue
pointer = pdev.malloc(len(data))
plan["bufs"][name] = pointer
pdev.upload(pointer, data)
args.append(ctypes.c_void_p(pointer))
return args


def manifest_args(build):
"""kernel name -> its argument tokens, from <build>/dx12/manifest.json."""
with open(os.path.join(build, "dx12", "manifest.json")) as f:
Expand Down
16 changes: 12 additions & 4 deletions clean/k/dx12/k_port_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
CONTROL = "scale_rgb" # Model M, frame.co
# family -> (HIP code objects, D3D12 kernels)
NARROW_KERNELS = ["k_stage_e2", "k_stage_e0", "k_stage_e1", "k_stage_d2", "k_stage_d1", "k_stage2_e0", "k_stage2_e1",
"k_stage2_d1", "k_stage2_d0_hp", "k_stage2_d0_hidden"]
"k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp", "k_stage2_d0_hidden"]
FAMILY_KERNELS = {"expo": (["k_io.co"], ["k_expo_scalars"]), "warm": ([], ["k_warm"]),
"mixprobe": ([], ["k_mix_probe", "k_mix_probe__rodd"]),
"narrow": (["k_block.co"], NARROW_KERNELS),
Expand Down Expand Up @@ -359,7 +359,9 @@ def check_mixprobe(items, read):
NARROW = {"k_stage_e2": ("e2", "", None), "k_stage_e0": ("e0", "", None), "k_stage_e1": ("e1", "", None),
"k_stage_d2": ("d2", "", None), "k_stage_d1": ("d1", "", None), "k_stage2_e0": ("e0", "@w2", None),
"k_stage2_e1": ("e1", "@w2", None), "k_stage2_d1": ("d1", "@w2", None),
"k_stage2_d1__rodd": ("d1", "@w2", None),
"k_stage2_d0_hp": ("d0", "@w2", "hp"), "k_stage2_d0_hidden": ("d0", "@w2", "hidden")}
NARROW_BIAS32 = {"k_stage2_e0", "k_stage2_e1", "k_stage2_d1", "k_stage2_d1__rodd", "k_stage2_d0_hp"}
# block-level sizes (W, H) and window offsets (spec 44: even): grids with fewer than a workgroup's windows among them
NARROW_SIZES = [(8, 8), (24, 16), (17, 13), (40, 24), (9, 33), (12, 20)]
NARROW_OFFSETS = [(0, 0), (2, 2), (4, 0), (0, 6), (6, 4)]
Expand Down Expand Up @@ -433,6 +435,12 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non
plan["launches"] = [(kernel, grid, block, args + [ctypes.c_void_p(hid)])]
plan["outputs"] = [("hidden", (H, W, 4))] + (
[("head", (2 * H, 2 * W, 4))] if d0 == "hp" else [("head", (H, W, 40))])
k, grid, block, hip_args = plan["launches"][0]
assert k == kernel.removesuffix("__rodd"), (k, kernel)
dx_args = hip_args
if kernel in NARROW_BIAS32:
dx_args = hip_args + PB.narrow_d3d12_args(plan, pdev, st, blob)
dx_plan = dict(plan, launches=[(kernel, grid, block, dx_args)])
outputs = [n for n, _ in plan["outputs"]]
for n, p in plan["bufs"].items():
if p in pdev.inputs:
Expand All @@ -444,11 +452,11 @@ def run_stages(table, co_name, tag, build, rocm, seeds, cdir, hip, grid_list=Non
sizes = {n: pdev.sizes[plan["bufs"][n]] for n in outputs}
got = {n: bytes(hip.h.download(plan["bufs"][n], sizes[n] + SLACK)) for n in outputs}
defined = {n: min(int(np.prod(sh)) * itemsize(n), sizes[n]) for n, sh in plan["outputs"]}
lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir, [co_name],
lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir, [co_name],
outputs)
if neg is None and plan["launches"][0][1] >= 4: # the dropped windows hold output pixels
if neg is None and grid >= 4: # the dropped windows hold output pixels
neg = name
lines += PB.chain_lines(name, plan, pdev, lambda k: tokens[k], cdir,
lines += PB.chain_lines(name, dx_plan, pdev, lambda k: tokens[k], cdir,
[co_name], outputs, prefix="negative_", grid_div=2)
items.append({"case": name, "kernel": kernel, "stage": stage, "size": list(size),
"offset": list(offset), "specials": special, "outputs": outputs, "hip": got,
Expand Down
38 changes: 32 additions & 6 deletions clean/k/dx12/kernels/k_block.hlsl
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,25 @@
// The FP16-accumulating forms (k_stageh_*) and the FP8 forms are not ported: the exact path only.
// Basis: our HIP kernels (spec 41 block, spec 44 windows, spec 43 6.8 hidden history, spec 40 table); exactness and the
// forms that differ (bit-identical) are listed in hlsl/k_block.hlsli and hlsl/k_common.hlsli. Arguments as HIP:
// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also the hidden history pointer (",p").
// KBlkParams by value (token s184:p19.b32, 27 constants), the d0 stages also take the hidden history pointer.
// The narrow two-window ports take a FP32 bias plane, and d0_hp takes a separate compact head weight pointer.
// Launch as HIP: 128 lanes (4 waves of 32) per workgroup, one workgroup per window (G = 1) or per two windows (G = 2).
//
// @wg-kernel k_stage_e2 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1
// @wg-kernel k_stage2_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1
// @wg-kernel k_stage_d2 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_BIAS32=1,KB_BASE_FORM=1
// @wg-kernel k_stage_e0 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=6144
// @wg-kernel k_stage_e1 co=k_block.co block=128 wave=32 args=s184:p19.b32 defines=KB_LDS_HALFS=4096
// @wg-kernel k_stage_d1 co=k_block.co block=128 wave=32 args=s184:p19.b32
// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1
// @wg-kernel k_stage2_d0_hp co=k_block.co block=128 wave=32 args=s184:p19.b32,p,p,p defines=KB_HIDDEN_ARG=1,KB_COMPACT_HEAD_ARG=1,KB_BIAS32=1,KB_BASE_FORM=1
// @wg-kernel k_stage2_d0_hidden co=k_block.co block=128 wave=32 args=s184:p19.b32,p defines=KB_HIDDEN_ARG=1
// The __rodd forms: the same stages with the round-to-odd softmax weights (hlsl/k_smx.hlsli K_SMX 1), launched by the
// native D3D12 runtime when the mix-form probe (kernels/k_mix_probe.hlsl) fails on the adapter's driver.
// @wg-kernel k_stage_e2__rodd co=k_block.co entry=k_stage_e2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1
// @wg-kernel k_stage_d2__rodd co=k_block.co entry=k_stage_d2 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1
// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32 defines=K_SMX=1
// @wg-kernel k_stage2_d1__rodd co=k_block.co entry=k_stage2_d1 block=128 wave=32 args=s184:p19.b32,p defines=K_SMX=1,KB_BIAS32=1,KB_BASE_FORM=1
#include "wg_platform.hlsli"
#include "k_blkparams.hlsli"

Expand All @@ -43,20 +44,33 @@ WG_ARGS_BEGIN
#if KB_HIDDEN_ARG
uint hidden_p; // the hidden history Hh written by the d0 epilogue
#endif
#if KB_BIAS32
uint P_b32;
#endif
#if KB_COMPACT_HEAD_ARG
uint P_compact_head;
#endif
WG_ARGS_END
#include "k_block.hlsli"

#if KB_HIDDEN_ARG
WG_KERNEL(k_stage2_d0_hp)
{
WG_SETUP();
#if KB_COMPACT_HEAD_ARG
k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id(), P_b32,
P_compact_head);
#else
k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, true, false>(hidden_p, wg_block_id());
#endif
}
#if !KB_COMPACT_HEAD_ARG
WG_KERNEL(k_stage2_d0_hidden)
{
WG_SETUP();
k_block<32, 2, KMIX_FIXED, KIN_EXPAND, 64, 0, true, 2, true, false, false>(hidden_p, wg_block_id());
}
#endif
#else
WG_KERNEL(k_stage_e2)
{
Expand All @@ -66,12 +80,20 @@ WG_KERNEL(k_stage_e2)
WG_KERNEL(k_stage2_e0)
{
WG_SETUP();
#if KB_BIAS32
k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32);
#else
k_block<32, 2, KMIX_FIXED, KIN_EMBED, 16, 64, false, 2, false, false, false>(0u, wg_block_id());
#endif
}
WG_KERNEL(k_stage2_e1)
{
WG_SETUP();
#if KB_BIAS32
k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id(), P_b32);
#else
k_block<64, 2, KMIX_FIXED, KIN_MAP, 64, 64, false, 2, false, false, false>(0u, wg_block_id());
#endif
}
WG_KERNEL(k_stage_d2)
{
Expand All @@ -81,7 +103,11 @@ WG_KERNEL(k_stage_d2)
WG_KERNEL(k_stage2_d1)
{
WG_SETUP();
#if KB_BIAS32
k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id(), P_b32);
#else
k_block<64, 2, KMIX_SOFTMAX, KIN_EXPAND, 64, 0, false, 2, false, false, true>(0u, wg_block_id());
#endif
}
WG_KERNEL(k_stage_e0)
{
Expand Down
22 changes: 22 additions & 0 deletions clean/k/runtime/k_pack.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -186,4 +186,26 @@ std::vector<uint8_t> k_pack_stage(const std::string& stage, const StageTensors&
return blob;
}

std::vector<uint8_t> k_pack_compact_head(const uint8_t* original, size_t channels) {
constexpr int row[40] = {
#include "../dx12/hlsl/k_head_rows.inc"
};
const size_t kt_count = channels / 16;
const size_t old_bias = (48 * channels * 2 + kAlign - 1) / kAlign * kAlign;
const size_t new_bias = (32 * channels * 2 + kAlign - 1) / kAlign * kAlign;
std::vector<uint8_t> out(new_bias + kAlign, 0);
for (size_t logical = 0; logical < 40; ++logical) {
if (row[logical] < 0) continue;
const size_t physical = (size_t)row[logical];
for (size_t kt = 0; kt < kt_count; ++kt)
for (size_t half = 0; half < 2; ++half) {
const size_t from = (logical / 16 * kt_count + kt) * 512 + (16 * half + logical % 16) * 16;
const size_t to = (physical / 16 * kt_count + kt) * 512 + (16 * half + physical % 16) * 16;
std::memcpy(out.data() + to, original + from, 16);
}
std::memcpy(out.data() + new_bias + physical * 2, original + old_bias + logical * 2, 2);
}
return out;
}

} // namespace wgrt
4 changes: 4 additions & 0 deletions clean/k/runtime/k_pack.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
// clean/k/reference/params.py normalize; byte-identical blobs (clean/k/tests/test_k_runtime_pack.py).
#pragma once

#include <cstddef>
#include <cstdint>
#include <string>
#include <vector>
Expand All @@ -22,4 +23,7 @@ namespace wgrt {
// k_stage_blob_bytes(*k_stage(stage)) bytes. On failure returns an empty vector with a message in *err.
std::vector<uint8_t> k_pack_stage(const std::string& stage, const StageTensors& t, std::string* err);

// Repack the 32 live rows of d0's head from the original gfx12 linear blob.
std::vector<uint8_t> k_pack_compact_head(const uint8_t* original, size_t channels);

} // namespace wgrt
2 changes: 1 addition & 1 deletion clean/k/runtime/k_smx_selfcheck.h
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

namespace k_smx_selfcheck {

constexpr char kSourceDigest[] = "ef7da006529622c2fa87d1d3371038cdf85467b1fb0010b361f334784b6fa92b";
constexpr char kSourceDigest[] = "a47546460b78b46b3ae0775661c6c4160ad20af09b6106c16322f8d164136264";
constexpr int kCount = 133;
constexpr const char* kNames[kCount] = {
"k_stage_e2@512x384/o4,4",
Expand Down
Loading
Loading