Skip to content

Commit 4e0601b

Browse files
Runtime packers: building blocks for other models' packers; C++ blobs checked against Python (#101)
2 parents cf8e8ac + 61b2e61 commit 4e0601b

4 files changed

Lines changed: 327 additions & 0 deletions

File tree

‎clean/runtime/model.cpp‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,54 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
638638
return true;
639639
}
640640

641+
// ---- the building blocks for other models' packers (model.h, namespace pack_parts): the packers' own helpers
642+
namespace pack_parts {
643+
644+
std::vector<std::vector<uint8_t>> frags(const float* m, int rows, int cols, const Order& R, const Order& K) {
645+
return wgrt::frags(Mat{m, rows, cols}, R, K);
646+
}
647+
void put_frags(std::vector<uint8_t>* b, const std::vector<std::vector<uint8_t>>& f) { wgrt::put_frags(b, f); }
648+
void put_tile_vector(std::vector<uint8_t>* b, const float* vec, const Order& R, const DRow& drow) {
649+
wgrt::put_tile_vector(b, vec, R, drow);
650+
}
651+
Order rows_identity(int n) { return wgrt::rows_identity(n); }
652+
Order rows_layout_l(int C, const DRow& drow) { return wgrt::rows_layout_l(C, drow); }
653+
Order kord_layout_l(int C) { return wgrt::kord_layout_l(C); }
654+
Order kord_drow(int n, const DRow& drow) { return wgrt::kord_drow(n, drow); }
655+
size_t align_up(size_t n) { return wgrt::align_up(n); }
656+
657+
bool pack_swin_block(const StageTensors& t, const std::string& prefix, int C, int heads, int mlp, const DRow& drow,
658+
std::vector<uint8_t>* out, std::string* err) {
659+
out->clear();
660+
if (C <= 0 || C % 16 || heads <= 0 || mlp <= 0 || mlp % 16) {
661+
*err = "block shape C " + std::to_string(C) + ", " + std::to_string(heads) + " heads, MLP " + std::to_string(mlp) +
662+
": C and MLP multiples of 16 needed";
663+
return false;
664+
}
665+
const size_t D = (size_t)heads * kHeadDim;
666+
const std::pair<const char*, size_t> want[] = {
667+
{"gamma1", (size_t)C}, {"gamma2", (size_t)C}, {"Wqk", D * C}, {"Wv", D * C}, {"B", (size_t)heads * 256},
668+
{"Wp", (size_t)C * D}, {"bp", (size_t)C}, {"W1", (size_t)mlp * C}, {"b1", (size_t)mlp},
669+
{"W2", (size_t)C * mlp}, {"b2", (size_t)C}};
670+
StageTensors block; // the block's tensors under their plain names
671+
for (const auto& w : want) {
672+
const auto it = t.find(prefix + w.first);
673+
if (it == t.end()) return *err = "missing tensor " + prefix + w.first, false;
674+
if (it->second.f.size() != w.second)
675+
return *err = "tensor " + prefix + w.first + " has " + std::to_string(it->second.f.size()) + " values, expected " +
676+
std::to_string(w.second),
677+
false;
678+
block[w.first] = it->second;
679+
}
680+
const SwinStageInfo s = {'b', C, heads, mlp, 0, 0};
681+
wgrt::pack_block(block, s, drow, out);
682+
if (out->size() != block_layout(C, heads, mlp).size) return *err = "block size mismatch", false;
683+
pad_to(out, wgrt::align_up(out->size()));
684+
return true;
685+
}
686+
687+
} // namespace pack_parts
688+
641689
bool pack_stage_blob(const std::string& stage, const StageTensors& t, const DRow& drow, std::vector<uint8_t>* out,
642690
std::string* err) {
643691
out->clear();

‎clean/runtime/model.h‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,4 +86,30 @@ const SwinStageInfo* swin_stage_info(const std::string& stage);
8686

8787
DRow drow_gfx12();
8888

89+
// Building blocks of the packers above, for the packers of other models of the same family (Model L,
90+
// clean/l/runtime; #89): the same code that packs Model M's blobs (ports of clean/runner/packing.py).
91+
namespace pack_parts {
92+
93+
using Order = std::vector<int>; // R or K order of a matrix; -1 = padding (a zero)
94+
95+
// Fragments of the row-major matrix m [rows][cols]: [mt][kc], each 256 bytes; lane 16 h + r, byte j holds
96+
// m[R[16 mt + r]][K[16 kc + 8 h + j]] as FP8 E4M3 (packing.frag_array).
97+
std::vector<std::vector<uint8_t>> frags(const float* m, int rows, int cols, const Order& R, const Order& K);
98+
void put_frags(std::vector<uint8_t>* b, const std::vector<std::vector<uint8_t>>& f); // appended in order
99+
// Tile-ordered float32 vector: packed[16 mt + 8 h + v] = vec[R[16 mt + drow[h][v]]] (packing.pack_tile_vector)
100+
void put_tile_vector(std::vector<uint8_t>* b, const float* vec, const Order& R, const DRow& drow);
101+
Order rows_identity(int n);
102+
Order rows_layout_l(int C, const DRow& drow); // R_L(C)
103+
Order kord_layout_l(int C); // K_L(C)
104+
Order kord_drow(int n, const DRow& drow);
105+
size_t align_up(size_t n); // to 256 bytes (packing.ALIGN)
106+
107+
// One Swin block section at any width (packing.pack_block + the padding to 256 bytes of pack_stage), from the tensors
108+
// of `t` whose names start with `prefix` ("" for a stage of its own; e.g. "blk." for a block inside a stage).
109+
// False (and *err) when a tensor is missing or has the wrong number of values.
110+
bool pack_swin_block(const StageTensors& t, const std::string& prefix, int C, int heads, int mlp, const DRow& drow,
111+
std::vector<uint8_t>* out, std::string* err);
112+
113+
} // namespace pack_parts
114+
89115
} // namespace wgrt

‎clean/tests/model_pack_dump.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// SPDX-License-Identifier: MIT
2+
// The runtime's C++ packers (clean/runtime/model.cpp) on the CPU, for clean/tests/test_model_pack_cpp.py:
3+
//
4+
// model_pack_dump stages <model dir> <out dir>
5+
// loads the converter's raw tensors (load_raw_model) and writes <out dir>/<stage>.bin = pack_stage_blob for
6+
// every stage with a blob (enc0, enc1 ... dec1, dec0); prints "OK <n>" or "ERR <message>"
7+
// model_pack_dump block <tensor dir> <prefix> <C> <heads> <mlp> <out file>
8+
// reads <tensor dir>/<name>.bin (float32, the file name is the tensor name, e.g. "blk.W1.bin"), packs one Swin
9+
// block section with pack_parts::pack_swin_block and writes it; prints "OK <bytes>" or "ERR <message>"
10+
#include <cstdio>
11+
#include <cstdlib>
12+
#include <fstream>
13+
#include <map>
14+
#include <string>
15+
#include <vector>
16+
17+
#include "../runtime/model.h"
18+
19+
namespace {
20+
21+
bool write_file(const std::string& path, const std::vector<uint8_t>& b) {
22+
std::ofstream f(path, std::ios::binary);
23+
f.write(reinterpret_cast<const char*>(b.data()), (std::streamsize)b.size());
24+
return (bool)f;
25+
}
26+
27+
bool read_f32(const std::string& path, std::vector<float>* out) {
28+
std::ifstream f(path, std::ios::binary | std::ios::ate);
29+
if (!f) return false;
30+
const std::streamsize n = f.tellg();
31+
if (n % 4) return false;
32+
out->resize((size_t)n / 4);
33+
f.seekg(0);
34+
return (bool)f.read(reinterpret_cast<char*>(out->data()), n);
35+
}
36+
37+
} // namespace
38+
39+
int main(int argc, char** argv) {
40+
const std::string mode = argc > 1 ? argv[1] : "";
41+
const wgrt::DRow d = wgrt::drow_gfx12();
42+
std::string err;
43+
if (mode == "stages" && argc == 4) {
44+
std::map<std::string, wgrt::StageTensors> model;
45+
if (!wgrt::load_raw_model(argv[2], &model, &err)) return std::printf("ERR %s\n", err.c_str()), 0;
46+
int n = 0;
47+
for (const std::string& s : wgrt::model_stages()) {
48+
if (s == "post") continue; // no blob of its own
49+
std::vector<uint8_t> blob;
50+
if (!wgrt::pack_stage_blob(s, model.at(s), d, &blob, &err)) return std::printf("ERR %s\n", err.c_str()), 0;
51+
if (!write_file(std::string(argv[3]) + "/" + s + ".bin", blob)) return std::printf("ERR write %s\n", s.c_str()), 0;
52+
++n;
53+
}
54+
std::printf("OK %d\n", n);
55+
return 0;
56+
}
57+
if (mode == "block" && argc == 8) {
58+
const std::string dir = argv[2], prefix = argv[3];
59+
const char* names[] = {"gamma1", "gamma2", "Wqk", "Wv", "B", "Wp", "bp", "W1", "b1", "W2", "b2"};
60+
wgrt::StageTensors t;
61+
for (const char* nm : names) {
62+
wgrt::Tensor x;
63+
if (read_f32(dir + "/" + prefix + nm + ".bin", &x.f)) t[prefix + nm] = x; // a missing file: the packer says so
64+
}
65+
std::vector<uint8_t> blob;
66+
if (!wgrt::pack_parts::pack_swin_block(t, prefix, std::atoi(argv[4]), std::atoi(argv[5]), std::atoi(argv[6]), d,
67+
&blob, &err))
68+
return std::printf("ERR %s\n", err.c_str()), 0;
69+
if (!write_file(argv[7], blob)) return std::printf("ERR write\n"), 0;
70+
std::printf("OK %zu\n", blob.size());
71+
return 0;
72+
}
73+
std::printf("ERR usage\n");
74+
return 2;
75+
}

‎clean/tests/test_model_pack_cpp.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
# SPDX-License-Identifier: MIT
2+
"""The runtime's C++ packers (clean/runtime/model.cpp) against the Python packers, byte for byte, on the CPU (#89):
3+
4+
- every stage blob of a synthetic Model M (enc0, the ten Swin stages, dec0) loaded from the converter's raw layout:
5+
pack_stage_blob against clean/runner/enc0.py pack, clean/runner/packing.py pack_stage and clean/runner/dec0.py pack;
6+
- the building block for other models' packers, pack_parts::pack_swin_block: at Model M's widths (equal to the block
7+
section of the stage blob) and at width 32 with a name prefix (Model L's blocks), against packing.pack_block, plus its
8+
refusals (missing tensor, wrong size).
9+
10+
No GPU and no model data: clean/tests/model_pack_dump.cpp is built with the host C++ compiler (CXX, clang++ or g++; on
11+
Windows MSVC cl after them, from PATH or through vcvars64.bat; skipped without one).
12+
13+
Run: python -m unittest clean.tests.test_model_pack_cpp
14+
"""
15+
import os
16+
import shutil
17+
import subprocess
18+
import tempfile
19+
import unittest
20+
21+
import numpy as np
22+
23+
from clean.reference import swin as Sw
24+
from clean.reference.fp8 import f16
25+
from clean.runner import dec0 as D0
26+
from clean.runner import enc0 as E0
27+
from clean.runner import packing as P
28+
from clean.runner import synthetic_model as SM
29+
from clean.runner.pipeline import logical_tensors
30+
31+
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
32+
SOURCES = [os.path.join("clean", "tests", "model_pack_dump.cpp"), os.path.join("clean", "runtime", "model.cpp"),
33+
os.path.join("clean", "runtime", "embedded.cpp"), os.path.join("clean", "converter", "src", "sha256.cpp"),
34+
os.path.join("clean", "converter", "src", "json.cpp")]
35+
BLOCK_NAMES = ("gamma1", "gamma2", "Wqk", "Wv", "B", "Wp", "bp", "W1", "b1", "W2", "b2")
36+
_EXE = None
37+
38+
39+
def _vcvars64():
40+
vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
41+
"Microsoft Visual Studio", "Installer", "vswhere.exe")
42+
if not os.path.exists(vswhere):
43+
return None
44+
r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True,
45+
text=True)
46+
vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat")
47+
return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None
48+
49+
50+
def dump_exe():
51+
global _EXE
52+
if _EXE is None:
53+
tmp = tempfile.mkdtemp(prefix="wg_mp_")
54+
exe = os.path.join(tmp, "model_pack_dump" + (".exe" if os.name == "nt" else ""))
55+
cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++")
56+
if cxx:
57+
cmd = [cxx, "-std=c++17", "-O1"] + (["-D_CRT_SECURE_NO_WARNINGS"] if os.name == "nt" else [])
58+
r = subprocess.run(cmd + [os.path.join(ROOT, s) for s in SOURCES] + ["-o", exe], capture_output=True,
59+
text=True)
60+
elif os.name == "nt" and (shutil.which("cl") or _vcvars64()):
61+
vcvars = None if shutil.which("cl") else _vcvars64()
62+
objdir = os.path.join(tmp, "obj")
63+
os.makedirs(objdir)
64+
cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s /Fo"%s\\\\" /Fe"%s"'
65+
% (" ".join('"%s"' % os.path.join(ROOT, s) for s in SOURCES), objdir, exe))
66+
bat = os.path.join(tmp, "build.bat")
67+
with open(bat, "w") as f:
68+
f.write("@echo off\r\n" + ('call "%s" >nul || exit /b 1\r\n' % vcvars if vcvars else "") +
69+
cl + " || exit /b 1\r\n")
70+
r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True)
71+
else:
72+
raise unittest.SkipTest("no host C++ compiler (CXX, clang++, g++ or MSVC cl)")
73+
if r.returncode != 0:
74+
raise RuntimeError("build of model_pack_dump failed:\n" + r.stdout + r.stderr)
75+
_EXE = exe
76+
return _EXE
77+
78+
79+
def dump(*args):
80+
r = subprocess.run([dump_exe(), *args], capture_output=True, text=True)
81+
return r.stdout.strip()
82+
83+
84+
def first_diff(got, want):
85+
if len(got) != len(want):
86+
return "length %d, expected %d" % (len(got), len(want))
87+
d = np.flatnonzero(np.frombuffer(got, np.uint8) != np.frombuffer(want, np.uint8))
88+
return "%d bytes differ, first at %d" % (len(d), d[0])
89+
90+
91+
def block_tensors(C, heads, mlp, rng):
92+
"""Synthetic block tensors at any width (FP8-exact matrices, FP16-exact vectors as the converter writes)."""
93+
D = heads * 32
94+
q8 = lambda a: np.asarray(Sw.q8(a), np.float32) # noqa: E731
95+
t = {"gamma1": 0.5 + rng.random(C), "gamma2": 0.5 + rng.random(C),
96+
"Wqk": q8(rng.standard_normal((heads, 32, C)) / np.sqrt(C)), "Wv": q8(rng.standard_normal((heads, 32, C)) / np.sqrt(C)),
97+
"B": rng.standard_normal((heads, 16, 16)), "Wp": q8(rng.standard_normal((C, D)) / np.sqrt(D)),
98+
"bp": 0.1 * rng.standard_normal(C), "W1": q8(rng.standard_normal((mlp, C)) / np.sqrt(C)),
99+
"b1": 0.1 * rng.standard_normal(mlp), "W2": q8(rng.standard_normal((C, mlp)) / np.sqrt(mlp)),
100+
"b2": 0.1 * rng.standard_normal(C)}
101+
return {k: (np.asarray(v, np.float32) if k.startswith("W") else f16(v).astype(np.float32)) for k, v in t.items()}
102+
103+
104+
def pack_block_py(t, C, heads, mlp):
105+
blob = P.pack_block(t, C, heads, mlp, P.resolve_d_row("gfx12"))
106+
return blob + bytes((-len(blob)) % P.ALIGN)
107+
108+
109+
class StageBlobs(unittest.TestCase):
110+
def test_model_m_stages(self):
111+
for seed in (1, 2):
112+
t = SM.synthetic_tensors(seed)
113+
d = tempfile.mkdtemp(prefix="wg_mp_")
114+
SM.write_raw(d, t)
115+
out = os.path.join(d, "out")
116+
os.makedirs(out)
117+
self.assertEqual(dump("stages", d, out), "OK 12")
118+
lt = logical_tensors(t) # the converter's raw shapes -> the Python packers' shapes (as the pipeline)
119+
want = {"enc0": E0.pack(lt["enc0"]), "dec0": D0.pack(lt["dec0"])}
120+
want.update({s: P.pack_stage(s, lt[s]) for s in Sw.STAGES})
121+
for s, w in want.items():
122+
with open(os.path.join(out, s + ".bin"), "rb") as f:
123+
got = f.read()
124+
if got != w:
125+
self.fail("seed %d, %s: %s" % (seed, s, first_diff(got, w)))
126+
127+
128+
class SwinBlockBuildingBlock(unittest.TestCase):
129+
def run_block(self, t, prefix, C, heads, mlp):
130+
d = tempfile.mkdtemp(prefix="wg_mp_")
131+
for name, a in t.items():
132+
np.ascontiguousarray(a, "<f4").tofile(os.path.join(d, prefix + name + ".bin"))
133+
out = os.path.join(d, "block.bin")
134+
res = dump("block", d, prefix, str(C), str(heads), str(mlp), out)
135+
got = None
136+
if res.startswith("OK"):
137+
with open(out, "rb") as f:
138+
got = f.read()
139+
return res, got
140+
141+
def test_model_m_widths_equal_the_stage_blocks(self):
142+
rng = np.random.default_rng(5)
143+
for s in ("enc1", "enc2", "b0", "dec2"):
144+
cfg = Sw.STAGES[s]
145+
p = {k: np.asarray(v, np.float32) for k, v in Sw.random_params(s, rng).items() if k in BLOCK_NAMES}
146+
res, got = self.run_block(p, "", cfg["C"], cfg["heads"], cfg["mlp"])
147+
want = pack_block_py(p, cfg["C"], cfg["heads"], cfg["mlp"])
148+
self.assertTrue(res.startswith("OK"), res)
149+
if got != want:
150+
self.fail("%s: %s" % (s, first_diff(got, want)))
151+
full = P.pack_stage(s, Sw.random_params(s, np.random.default_rng(9))) # (another blob, same section size)
152+
self.assertEqual(len(want), P.stage_sections(s)[1])
153+
self.assertLessEqual(len(want), len(full))
154+
155+
def test_width_32_with_prefix(self):
156+
"""Model L's width-32 blocks (2 heads, MLP 128), tensors named "blk.<name>"."""
157+
for seed in (3, 4):
158+
t = block_tensors(32, 2, 128, np.random.default_rng(seed))
159+
res, got = self.run_block(t, "blk.", 32, 2, 128)
160+
want = pack_block_py(t, 32, 2, 128)
161+
self.assertEqual(res, "OK %d" % len(want))
162+
if got != want:
163+
self.fail(first_diff(got, want))
164+
165+
def test_refusals(self):
166+
t = block_tensors(32, 2, 128, np.random.default_rng(6))
167+
t2 = dict(t)
168+
del t2["W2"]
169+
self.assertEqual(self.run_block(t2, "blk.", 32, 2, 128)[0], "ERR missing tensor blk.W2")
170+
t3 = dict(t)
171+
t3["b1"] = t3["b1"][:64]
172+
self.assertEqual(self.run_block(t3, "blk.", 32, 2, 128)[0],
173+
"ERR tensor blk.b1 has 64 values, expected 128")
174+
self.assertTrue(self.run_block(t, "blk.", 40, 2, 128)[0].startswith("ERR block shape C 40"))
175+
176+
177+
if __name__ == "__main__":
178+
unittest.main()

0 commit comments

Comments
 (0)