Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions clean/runtime/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -632,6 +632,54 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
return true;
}

// ---- the building blocks for other models' packers (model.h, namespace pack_parts): the packers' own helpers
namespace pack_parts {

std::vector<std::vector<uint8_t>> frags(const float* m, int rows, int cols, const Order& R, const Order& K) {
return wgrt::frags(Mat{m, rows, cols}, R, K);
}
void put_frags(std::vector<uint8_t>* b, const std::vector<std::vector<uint8_t>>& f) { wgrt::put_frags(b, f); }
void put_tile_vector(std::vector<uint8_t>* b, const float* vec, const Order& R, const DRow& drow) {
wgrt::put_tile_vector(b, vec, R, drow);
}
Order rows_identity(int n) { return wgrt::rows_identity(n); }
Order rows_layout_l(int C, const DRow& drow) { return wgrt::rows_layout_l(C, drow); }
Order kord_layout_l(int C) { return wgrt::kord_layout_l(C); }
Order kord_drow(int n, const DRow& drow) { return wgrt::kord_drow(n, drow); }
size_t align_up(size_t n) { return wgrt::align_up(n); }

bool pack_swin_block(const StageTensors& t, const std::string& prefix, int C, int heads, int mlp, const DRow& drow,
std::vector<uint8_t>* out, std::string* err) {
out->clear();
if (C <= 0 || C % 16 || heads <= 0 || mlp <= 0 || mlp % 16) {
*err = "block shape C " + std::to_string(C) + ", " + std::to_string(heads) + " heads, MLP " + std::to_string(mlp) +
": C and MLP multiples of 16 needed";
return false;
}
const size_t D = (size_t)heads * kHeadDim;
const std::pair<const char*, size_t> want[] = {
{"gamma1", (size_t)C}, {"gamma2", (size_t)C}, {"Wqk", D * C}, {"Wv", D * C}, {"B", (size_t)heads * 256},
{"Wp", (size_t)C * D}, {"bp", (size_t)C}, {"W1", (size_t)mlp * C}, {"b1", (size_t)mlp},
{"W2", (size_t)C * mlp}, {"b2", (size_t)C}};
StageTensors block; // the block's tensors under their plain names
for (const auto& w : want) {
const auto it = t.find(prefix + w.first);
if (it == t.end()) return *err = "missing tensor " + prefix + w.first, false;
if (it->second.f.size() != w.second)
return *err = "tensor " + prefix + w.first + " has " + std::to_string(it->second.f.size()) + " values, expected " +
std::to_string(w.second),
false;
block[w.first] = it->second;
}
const SwinStageInfo s = {'b', C, heads, mlp, 0, 0};
wgrt::pack_block(block, s, drow, out);
if (out->size() != block_layout(C, heads, mlp).size) return *err = "block size mismatch", false;
pad_to(out, wgrt::align_up(out->size()));
return true;
}

} // namespace pack_parts

bool pack_stage_blob(const std::string& stage, const StageTensors& t, const DRow& drow, std::vector<uint8_t>* out,
std::string* err) {
out->clear();
Expand Down
26 changes: 26 additions & 0 deletions clean/runtime/model.h
Original file line number Diff line number Diff line change
Expand Up @@ -68,4 +68,30 @@ const SwinStageInfo* swin_stage_info(const std::string& stage);

DRow drow_gfx12();

// Building blocks of the packers above, for the packers of other models of the same family (Model L,
// clean/l/runtime; #89): the same code that packs Model M's blobs (ports of clean/runner/packing.py).
namespace pack_parts {

using Order = std::vector<int>; // R or K order of a matrix; -1 = padding (a zero)

// Fragments of the row-major matrix m [rows][cols]: [mt][kc], each 256 bytes; lane 16 h + r, byte j holds
// m[R[16 mt + r]][K[16 kc + 8 h + j]] as FP8 E4M3 (packing.frag_array).
std::vector<std::vector<uint8_t>> frags(const float* m, int rows, int cols, const Order& R, const Order& K);
void put_frags(std::vector<uint8_t>* b, const std::vector<std::vector<uint8_t>>& f); // appended in order
// Tile-ordered float32 vector: packed[16 mt + 8 h + v] = vec[R[16 mt + drow[h][v]]] (packing.pack_tile_vector)
void put_tile_vector(std::vector<uint8_t>* b, const float* vec, const Order& R, const DRow& drow);
Order rows_identity(int n);
Order rows_layout_l(int C, const DRow& drow); // R_L(C)
Order kord_layout_l(int C); // K_L(C)
Order kord_drow(int n, const DRow& drow);
size_t align_up(size_t n); // to 256 bytes (packing.ALIGN)

// One Swin block section at any width (packing.pack_block + the padding to 256 bytes of pack_stage), from the tensors
// of `t` whose names start with `prefix` ("" for a stage of its own; e.g. "blk." for a block inside a stage).
// False (and *err) when a tensor is missing or has the wrong number of values.
bool pack_swin_block(const StageTensors& t, const std::string& prefix, int C, int heads, int mlp, const DRow& drow,
std::vector<uint8_t>* out, std::string* err);

} // namespace pack_parts

} // namespace wgrt
75 changes: 75 additions & 0 deletions clean/tests/model_pack_dump.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// SPDX-License-Identifier: MIT
// The runtime's C++ packers (clean/runtime/model.cpp) on the CPU, for clean/tests/test_model_pack_cpp.py:
//
// model_pack_dump stages <model dir> <out dir>
// loads the converter's raw tensors (load_raw_model) and writes <out dir>/<stage>.bin = pack_stage_blob for
// every stage with a blob (enc0, enc1 ... dec1, dec0); prints "OK <n>" or "ERR <message>"
// model_pack_dump block <tensor dir> <prefix> <C> <heads> <mlp> <out file>
// reads <tensor dir>/<name>.bin (float32, the file name is the tensor name, e.g. "blk.W1.bin"), packs one Swin
// block section with pack_parts::pack_swin_block and writes it; prints "OK <bytes>" or "ERR <message>"
#include <cstdio>
#include <cstdlib>
#include <fstream>
#include <map>
#include <string>
#include <vector>

#include "../runtime/model.h"

namespace {

bool write_file(const std::string& path, const std::vector<uint8_t>& b) {
std::ofstream f(path, std::ios::binary);
f.write(reinterpret_cast<const char*>(b.data()), (std::streamsize)b.size());
return (bool)f;
}

bool read_f32(const std::string& path, std::vector<float>* out) {
std::ifstream f(path, std::ios::binary | std::ios::ate);
if (!f) return false;
const std::streamsize n = f.tellg();
if (n % 4) return false;
out->resize((size_t)n / 4);
f.seekg(0);
return (bool)f.read(reinterpret_cast<char*>(out->data()), n);
}

} // namespace

int main(int argc, char** argv) {
const std::string mode = argc > 1 ? argv[1] : "";
const wgrt::DRow d = wgrt::drow_gfx12();
std::string err;
if (mode == "stages" && argc == 4) {
std::map<std::string, wgrt::StageTensors> model;
if (!wgrt::load_raw_model(argv[2], &model, &err)) return std::printf("ERR %s\n", err.c_str()), 0;
int n = 0;
for (const std::string& s : wgrt::model_stages()) {
if (s == "post") continue; // no blob of its own
std::vector<uint8_t> blob;
if (!wgrt::pack_stage_blob(s, model.at(s), d, &blob, &err)) return std::printf("ERR %s\n", err.c_str()), 0;
if (!write_file(std::string(argv[3]) + "/" + s + ".bin", blob)) return std::printf("ERR write %s\n", s.c_str()), 0;
++n;
}
std::printf("OK %d\n", n);
return 0;
}
if (mode == "block" && argc == 8) {
const std::string dir = argv[2], prefix = argv[3];
const char* names[] = {"gamma1", "gamma2", "Wqk", "Wv", "B", "Wp", "bp", "W1", "b1", "W2", "b2"};
wgrt::StageTensors t;
for (const char* nm : names) {
wgrt::Tensor x;
if (read_f32(dir + "/" + prefix + nm + ".bin", &x.f)) t[prefix + nm] = x; // a missing file: the packer says so
}
std::vector<uint8_t> blob;
if (!wgrt::pack_parts::pack_swin_block(t, prefix, std::atoi(argv[4]), std::atoi(argv[5]), std::atoi(argv[6]), d,
&blob, &err))
return std::printf("ERR %s\n", err.c_str()), 0;
if (!write_file(argv[7], blob)) return std::printf("ERR write\n"), 0;
std::printf("OK %zu\n", blob.size());
return 0;
}
std::printf("ERR usage\n");
return 2;
}
178 changes: 178 additions & 0 deletions clean/tests/test_model_pack_cpp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: MIT
"""The runtime's C++ packers (clean/runtime/model.cpp) against the Python packers, byte for byte, on the CPU (#89):

- every stage blob of a synthetic Model M (enc0, the ten Swin stages, dec0) loaded from the converter's raw layout:
pack_stage_blob against clean/runner/enc0.py pack, clean/runner/packing.py pack_stage and clean/runner/dec0.py pack;
- the building block for other models' packers, pack_parts::pack_swin_block: at Model M's widths (equal to the block
section of the stage blob) and at width 32 with a name prefix (Model L's blocks), against packing.pack_block, plus its
refusals (missing tensor, wrong size).

No GPU and no model data: clean/tests/model_pack_dump.cpp is built with the host C++ compiler (CXX, clang++ or g++; on
Windows MSVC cl after them, from PATH or through vcvars64.bat; skipped without one).

Run: python -m unittest clean.tests.test_model_pack_cpp
"""
import os
import shutil
import subprocess
import tempfile
import unittest

import numpy as np

from clean.reference import swin as Sw
from clean.reference.fp8 import f16
from clean.runner import dec0 as D0
from clean.runner import enc0 as E0
from clean.runner import packing as P
from clean.runner import synthetic_model as SM
from clean.runner.pipeline import logical_tensors

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
SOURCES = [os.path.join("clean", "tests", "model_pack_dump.cpp"), os.path.join("clean", "runtime", "model.cpp"),
os.path.join("clean", "runtime", "embedded.cpp"), os.path.join("clean", "converter", "src", "sha256.cpp"),
os.path.join("clean", "converter", "src", "json.cpp")]
BLOCK_NAMES = ("gamma1", "gamma2", "Wqk", "Wv", "B", "Wp", "bp", "W1", "b1", "W2", "b2")
_EXE = None


def _vcvars64():
vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
"Microsoft Visual Studio", "Installer", "vswhere.exe")
if not os.path.exists(vswhere):
return None
r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True,
text=True)
vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat")
return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None


def dump_exe():
global _EXE
if _EXE is None:
tmp = tempfile.mkdtemp(prefix="wg_mp_")
exe = os.path.join(tmp, "model_pack_dump" + (".exe" if os.name == "nt" else ""))
cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++")
if cxx:
cmd = [cxx, "-std=c++17", "-O1"] + (["-D_CRT_SECURE_NO_WARNINGS"] if os.name == "nt" else [])
r = subprocess.run(cmd + [os.path.join(ROOT, s) for s in SOURCES] + ["-o", exe], capture_output=True,
text=True)
elif os.name == "nt" and (shutil.which("cl") or _vcvars64()):
vcvars = None if shutil.which("cl") else _vcvars64()
objdir = os.path.join(tmp, "obj")
os.makedirs(objdir)
cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s /Fo"%s\\\\" /Fe"%s"'
% (" ".join('"%s"' % os.path.join(ROOT, s) for s in SOURCES), objdir, exe))
bat = os.path.join(tmp, "build.bat")
with open(bat, "w") as f:
f.write("@echo off\r\n" + ('call "%s" >nul || exit /b 1\r\n' % vcvars if vcvars else "") +
cl + " || exit /b 1\r\n")
r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True)
else:
raise unittest.SkipTest("no host C++ compiler (CXX, clang++, g++ or MSVC cl)")
if r.returncode != 0:
raise RuntimeError("build of model_pack_dump failed:\n" + r.stdout + r.stderr)
_EXE = exe
return _EXE


def dump(*args):
r = subprocess.run([dump_exe(), *args], capture_output=True, text=True)
return r.stdout.strip()


def first_diff(got, want):
if len(got) != len(want):
return "length %d, expected %d" % (len(got), len(want))
d = np.flatnonzero(np.frombuffer(got, np.uint8) != np.frombuffer(want, np.uint8))
return "%d bytes differ, first at %d" % (len(d), d[0])


def block_tensors(C, heads, mlp, rng):
"""Synthetic block tensors at any width (FP8-exact matrices, FP16-exact vectors as the converter writes)."""
D = heads * 32
q8 = lambda a: np.asarray(Sw.q8(a), np.float32) # noqa: E731
t = {"gamma1": 0.5 + rng.random(C), "gamma2": 0.5 + rng.random(C),
"Wqk": q8(rng.standard_normal((heads, 32, C)) / np.sqrt(C)), "Wv": q8(rng.standard_normal((heads, 32, C)) / np.sqrt(C)),
"B": rng.standard_normal((heads, 16, 16)), "Wp": q8(rng.standard_normal((C, D)) / np.sqrt(D)),
"bp": 0.1 * rng.standard_normal(C), "W1": q8(rng.standard_normal((mlp, C)) / np.sqrt(C)),
"b1": 0.1 * rng.standard_normal(mlp), "W2": q8(rng.standard_normal((C, mlp)) / np.sqrt(mlp)),
"b2": 0.1 * rng.standard_normal(C)}
return {k: (np.asarray(v, np.float32) if k.startswith("W") else f16(v).astype(np.float32)) for k, v in t.items()}


def pack_block_py(t, C, heads, mlp):
blob = P.pack_block(t, C, heads, mlp, P.resolve_d_row("gfx12"))
return blob + bytes((-len(blob)) % P.ALIGN)


class StageBlobs(unittest.TestCase):
def test_model_m_stages(self):
for seed in (1, 2):
t = SM.synthetic_tensors(seed)
d = tempfile.mkdtemp(prefix="wg_mp_")
SM.write_raw(d, t)
out = os.path.join(d, "out")
os.makedirs(out)
self.assertEqual(dump("stages", d, out), "OK 12")
lt = logical_tensors(t) # the converter's raw shapes -> the Python packers' shapes (as the pipeline)
want = {"enc0": E0.pack(lt["enc0"]), "dec0": D0.pack(lt["dec0"])}
want.update({s: P.pack_stage(s, lt[s]) for s in Sw.STAGES})
for s, w in want.items():
with open(os.path.join(out, s + ".bin"), "rb") as f:
got = f.read()
if got != w:
self.fail("seed %d, %s: %s" % (seed, s, first_diff(got, w)))


class SwinBlockBuildingBlock(unittest.TestCase):
def run_block(self, t, prefix, C, heads, mlp):
d = tempfile.mkdtemp(prefix="wg_mp_")
for name, a in t.items():
np.ascontiguousarray(a, "<f4").tofile(os.path.join(d, prefix + name + ".bin"))
out = os.path.join(d, "block.bin")
res = dump("block", d, prefix, str(C), str(heads), str(mlp), out)
got = None
if res.startswith("OK"):
with open(out, "rb") as f:
got = f.read()
return res, got

def test_model_m_widths_equal_the_stage_blocks(self):
rng = np.random.default_rng(5)
for s in ("enc1", "enc2", "b0", "dec2"):
cfg = Sw.STAGES[s]
p = {k: np.asarray(v, np.float32) for k, v in Sw.random_params(s, rng).items() if k in BLOCK_NAMES}
res, got = self.run_block(p, "", cfg["C"], cfg["heads"], cfg["mlp"])
want = pack_block_py(p, cfg["C"], cfg["heads"], cfg["mlp"])
self.assertTrue(res.startswith("OK"), res)
if got != want:
self.fail("%s: %s" % (s, first_diff(got, want)))
full = P.pack_stage(s, Sw.random_params(s, np.random.default_rng(9))) # (another blob, same section size)
self.assertEqual(len(want), P.stage_sections(s)[1])
self.assertLessEqual(len(want), len(full))

def test_width_32_with_prefix(self):
"""Model L's width-32 blocks (2 heads, MLP 128), tensors named "blk.<name>"."""
for seed in (3, 4):
t = block_tensors(32, 2, 128, np.random.default_rng(seed))
res, got = self.run_block(t, "blk.", 32, 2, 128)
want = pack_block_py(t, 32, 2, 128)
self.assertEqual(res, "OK %d" % len(want))
if got != want:
self.fail(first_diff(got, want))

def test_refusals(self):
t = block_tensors(32, 2, 128, np.random.default_rng(6))
t2 = dict(t)
del t2["W2"]
self.assertEqual(self.run_block(t2, "blk.", 32, 2, 128)[0], "ERR missing tensor blk.W2")
t3 = dict(t)
t3["b1"] = t3["b1"][:64]
self.assertEqual(self.run_block(t3, "blk.", 32, 2, 128)[0],
"ERR tensor blk.b1 has 64 values, expected 128")
self.assertTrue(self.run_block(t, "blk.", 40, 2, 128)[0].startswith("ERR block shape C 40"))


if __name__ == "__main__":
unittest.main()
Loading