Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 31 additions & 26 deletions clean/l/runtime/l_model.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: MIT
// Model L's model data in the runtime (see l_model.h). Shapes mirror clean/l/reference/table.py for the default table
// (checked by clean/l/runtime/tests/test_l_runtime_load.py).
// Model L's model data in the runtime (see l_model.h). Checked against the preset-L map and the structure table by
// clean/l/runtime/tests/test_l_runtime_load.py.
#include "l_model.h"

namespace wgrt {
Expand All @@ -10,38 +10,43 @@ namespace {

const int kHeadDim = 32, kWindowTokens = 16;

// One Swin block's tensors with a name prefix (table.block_shapes).
void add_block(std::vector<TensorSpec>* v, const std::string& prefix, int C, int heads, int mlp) {
const std::vector<TensorSpec> b = {{prefix + "gamma1", {C}, false},
{prefix + "gamma2", {C}, false},
{prefix + "Wqk", {heads, kHeadDim, C}, false},
{prefix + "Wv", {heads, kHeadDim, C}, false},
{prefix + "B", {heads, kWindowTokens, kWindowTokens}, false},
{prefix + "Wp", {C, kHeadDim * heads}, false},
{prefix + "bp", {C}, false},
{prefix + "W1", {mlp, C}, false},
{prefix + "b1", {mlp}, false},
{prefix + "W2", {C, mlp}, false},
{prefix + "b2", {C}, false}};
v->insert(v->end(), b.begin(), b.end());
// One Swin block's tensors (table.block_shapes; the map's block record set).
std::vector<TensorSpec> block_specs(int C, int heads, int mlp) {
return {{"gamma1", {C}, false},
{"gamma2", {C}, false},
{"Wqk", {heads, kHeadDim, C}, false},
{"Wv", {heads, kHeadDim, C}, false},
{"B", {heads, kWindowTokens, kWindowTokens}, false},
{"Wp", {C, kHeadDim * heads}, false},
{"bp", {C}, false},
{"W1", {mlp, C}, false},
{"b1", {mlp}, false},
{"W2", {C, mlp}, false},
{"b2", {C}, false}};
}

} // namespace

StageSpecs stage_specs() {
StageSpecs s;
// enc0: the projection reads [raw (80) | x (32) | 16 zeros] = 128 inputs; the first linear reads the 20 values of
// one sub-pixel (table: select 0..20, linear 20 -> 32), the second 32 -> 32; then the width-32 block.
std::vector<TensorSpec> enc0 = {{"W", {64, 128}, false}, {"b", {64}, false}, {"T", {8, 16, 16}, true},
{"sigma1", {1}, false}, {"sigma2", {1}, false}, {"Wa", {32, 20}, false},
{"ba", {32}, false}, {"Wb", {32, 32}, false}, {"bb", {32}, false}};
add_block(&enc0, "blk.", 32, 2, 128);
s.emplace_back("enc0", enc0);
// enc0: the projection reads the 2 x 2 pixels of 32 channels of each token (128 inputs); the first input linear
// reads the 20 values of a pixel, the second 32.
s.emplace_back("enc0", std::vector<TensorSpec>{{"W", {64, 128}, false},
{"b", {64}, false},
{"T", {8, 16, 16}, true},
{"sigma1", {1}, false},
{"sigma2", {1}, false},
{"Win1", {32, 20}, false},
{"bin1", {32}, false},
{"Win2", {32, 32}, false},
{"bin2", {32}, false}});
s.emplace_back("enc0_block", block_specs(32, 2, 128));
for (const std::string& st : model_stages())
if (st != "enc0" && st != "dec0" && st != "post") s.emplace_back(st, wgrt::stage_specs(st));
std::vector<TensorSpec> dec0 = wgrt::stage_specs("dec0");
add_block(&dec0, "blk.", 32, 2, 128);
s.emplace_back("dec0", dec0);
s.emplace_back("dec0_block", block_specs(32, 2, 128));
// dec0: 32 outputs per sub-pixel (Model M: 17), then Model M's scalars
s.emplace_back("dec0", std::vector<TensorSpec>{
{"W", {4, 32, 64}, false}, {"b", {4, 32}, false}, {"S", {1}, false}, {"T", {1}, false}});
s.emplace_back("post", wgrt::stage_specs("post"));
return s;
}
Expand Down
14 changes: 8 additions & 6 deletions clean/l/runtime/l_model.h
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: MIT
// Model L's model data in the runtime (#89): the stages and tensors of Model L in the converter's raw layout, and the
// raw loader for them (clean/runtime/model.h load_raw_stages). Names are the logical names of the default structure
// table (clean/l/reference/table.py: encoder_shapes, decoder_shapes); the ten Swin stages and post are Model M's.
// raw loader for them (clean/runtime/model.h load_raw_stages). Stages, names and shapes are those of the preset-L map
// (data/preset-L.v1.map.json), which the structure table (clean/l/reference/table.py param_shapes) groups the same way:
//
// enc0: Model M's W, b, T, sigma1, sigma2 (W is the projection, 64 x 128) + the two linears Wa / ba, Wb / bb + the
// width-32 block "blk.*"
// enc0: W (the projection, 64 x 128), b, T, sigma1, sigma2, and the two input linears Win1 / bin1 (32 x 20),
// Win2 / bin2 (32 x 32)
// enc0_block: the encoder's width-32 Swin block
// enc1 ... dec1: Model M's stages (clean/runtime/model.cpp stage_specs)
// dec0: Model M's W, b, S, T + the width-32 block "blk.*"
// dec0_block: the decoder's width-32 Swin block
// dec0: W (4 x 32 x 64), b (4 x 32), S, T
// post: Model M's s
#pragma once

Expand All @@ -18,7 +20,7 @@
namespace wgrt {
namespace l {

// The stages in load order: enc0, enc1 ... dec1 (model_stages() order), dec0, post.
// The stages in the map's order: enc0, enc0_block, enc1 ... dec1, dec0_block, dec0, post.
StageSpecs stage_specs();

// load_raw_stages with stage_specs(): <model_dir>[/raw]/<stage>/<name>.bin.
Expand Down
205 changes: 205 additions & 0 deletions clean/l/runtime/l_pack.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
// SPDX-License-Identifier: MIT
// Model L weight packing for the runtime; see l_pack.h.
#include "l_pack.h"

namespace wgrt {
namespace l {

namespace {

const Tensor* find(const StageTensors& t, const std::string& name) {
const auto it = t.find(name);
return it == t.end() ? nullptr : &it->second;
}

bool need(const StageTensors& t, const std::string& stage, const std::string& name, size_t n, const Tensor** out,
std::string* err) {
const Tensor* x = find(t, name);
if (!x) return *err = stage + ": missing tensor " + name, false;
if (x->f.size() != n)
return *err = stage + ": tensor " + name + " has " + std::to_string(x->f.size()) + " values, expected " +
std::to_string(n),
false;
*out = x;
return true;
}

const StageTensors* stage(const std::map<std::string, StageTensors>& p, const std::string& name, std::string* err) {
const auto it = p.find(name);
if (it == p.end()) return *err = "missing stage " + name, nullptr;
return &it->second;
}

int sum(const std::vector<int>& v) {
int n = 0;
for (int x : v) n += x;
return n;
}

Launch swin(const std::string& src, const std::string& offsets, const std::string& params) {
Launch l;
l.kind = "swin", l.blob = params, l.src = src, l.dst = src, l.C = 32, l.heads = 2, l.mlp = 128;
l.offsets = offsets, l.params = params;
return l;
}

} // namespace

const std::vector<Launch>& encoder_launches() {
static const std::vector<Launch> v = [] {
std::vector<Launch> l(5);
l[0].kind = "values", l[0].blob = "enc0.values", l[0].dst = "q", l[0].C = 32;
l[1].kind = "linear", l[1].blob = "enc0.Win1", l[1].src = "q", l[1].dst = "x";
l[1].C_in = 32, l[1].C_out = 32, l[1].w = "Win1", l[1].b = "bin1", l[1].acc = "f16", l[1].out_round = "relu_q8";
l[2].kind = "linear", l[2].blob = "enc0.Win2", l[2].src = "x", l[2].dst = "x";
l[2].C_in = 32, l[2].C_out = 32, l[2].w = "Win2", l[2].b = "bin2", l[2].acc = "f16", l[2].out_round = "q8";
l[3] = swin("x", "first", "enc0_block");
l[4].kind = "project", l[4].blob = "enc0.project", l[4].src = "x", l[4].groups = {32, 32, 32, 32};
l[4].w = "W", l[4].b = "b", l[4].clamp = 6.28125f;
return l;
}();
return v;
}

const std::vector<Launch>& decoder_launches() {
static const std::vector<Launch> v = [] {
std::vector<Launch> l(3);
l[0].kind = "dec_project", l[0].blob = "dec0.project", l[0].dst = "p", l[0].w = "W", l[0].b = "b";
l[0].groups = {32, 32};
l[1] = swin("p", "second", "dec0_block");
l[2].kind = "coef", l[2].blob = "dec0.coef", l[2].src = "p";
l[2].values = {0, 1, 10, 11, 12, 13, 14, 15, 16, 2, 3, 4, 5, 6, 7, 8, 9}; // table.DEC0_VALUES
return l;
}();
return v;
}

std::vector<int> kord_groups32(int n) {
std::vector<int> K(n);
for (int kb = 0; kb < n / 16; ++kb)
for (int h = 0; h < 2; ++h)
for (int j = 0; j < 8; ++j) K[16 * kb + 8 * h + j] = 32 * (kb / 2) + 16 * h + 8 * (kb % 2) + j;
return K;
}

bool pack_linear(const std::vector<float>& W, const std::vector<float>& b, int C_in, int C_out, int lo, const DRow& d,
std::vector<uint8_t>* out, std::string* err) {
out->clear();
if (C_in % 16 || C_out % 16 || C_out <= 0 || lo < 0 || b.size() != (size_t)C_out || W.size() % C_out)
return *err = "linear: bad shape (C_in " + std::to_string(C_in) + ", C_out " + std::to_string(C_out) + ")", false;
const int n = (int)(W.size() / C_out);
if (lo + n > C_in) return *err = "linear: input channels " + std::to_string(lo) + " + " + std::to_string(n) +
" outside " + std::to_string(C_in),
false;
std::vector<float> Wx((size_t)C_out * C_in, 0.0f); // zero weights outside lo .. lo + n
for (int r = 0; r < C_out; ++r)
for (int c = 0; c < n; ++c) Wx[(size_t)r * C_in + lo + c] = W[(size_t)r * n + c];
const pack_parts::Order RL = pack_parts::rows_layout_l(C_out, d);
pack_parts::put_tile_vector(out, b.data(), RL, d);
pack_parts::put_frags(out, pack_parts::frags(Wx.data(), C_out, C_in, RL, pack_parts::kord_layout_l(C_in)));
return true;
}

bool pack_project(const std::vector<float>& W, const std::vector<float>& b, const std::vector<int>& groups, int n_out,
const DRow& d, std::vector<uint8_t>* out, std::string* err) {
out->clear();
for (int g : groups)
if (g != 32) return *err = "project: groups of 32 channels only", false;
const int n = sum(groups);
if (n_out <= 0 || n_out % 16 || b.size() != (size_t)n_out || W.size() != (size_t)n_out * n)
return *err = "project: bad shape", false;
const pack_parts::Order RL = pack_parts::rows_layout_l(n_out, d);
pack_parts::put_tile_vector(out, b.data(), RL, d);
pack_parts::put_frags(out, pack_parts::frags(W.data(), n_out, n, RL, kord_groups32(n)));
return true;
}

bool pack_dec_project(const std::vector<float>& W, const std::vector<float>& b, const std::vector<int>& groups,
const DRow& d, std::vector<uint8_t>* out, std::string* err) {
out->clear();
for (int g : groups)
if (g != 32) return *err = "dec_project: groups of 32 channels only", false;
const int n = sum(groups);
if (b.size() != 128 || W.size() != (size_t)128 * n) return *err = "dec_project: bad shape", false;
const pack_parts::Order RE = pack_parts::rows_expand(32, d); // tile 2 s + g: sub-pixel s, channels 16 g ..
pack_parts::put_tile_vector(out, b.data(), RE, d);
pack_parts::put_frags(out, pack_parts::frags(W.data(), 128, n, RE, kord_groups32(n)));
return true;
}

bool pack_model(const std::map<std::string, StageTensors>& params, const DRow& d,
std::map<std::string, std::vector<uint8_t>>* out, std::string* err) {
out->clear();
const StageTensors* enc = stage(params, "enc0", err);
const StageTensors* dec = enc ? stage(params, "dec0", err) : nullptr;
if (!dec) return false;
auto block = [&](const Launch& ln, std::vector<uint8_t>* blob) {
const StageTensors* t = stage(params, ln.params, err);
if (!t) return false;
if (!pack_parts::pack_swin_block(*t, "", ln.C, ln.heads, ln.mlp, d, blob, err))
return *err = ln.params + ": " + *err, false;
return true;
};
for (const Launch& ln : encoder_launches()) {
std::vector<uint8_t> blob;
if (ln.kind == "values") {
// Model M's enc0 blob with a zero projection: only the dither table and the two scalars are used
StageTensors t;
t["W"].f.assign(64 * 80, 0.0f);
t["b"].f.assign(64, 0.0f);
const Tensor* T = find(*enc, "T");
if (T && T->u.size() != 8 * 16 * 16) return *err = "enc0: tensor T must hold 2048 uint8 values", false;
t["T"].u = T ? T->u : std::vector<uint8_t>(8 * 16 * 16, 0);
for (const char* s : {"sigma1", "sigma2"}) {
const Tensor* x = find(*enc, s);
if (x && x->f.size() != 1) return *err = std::string("enc0: tensor ") + s + " must hold one value", false;
t[s].f = {x ? x->f[0] : 0.0f};
}
if (!pack_stage_blob("enc0", t, d, &blob, err)) return false;
} else if (ln.kind == "linear") {
const Tensor *b, *W = find(*enc, ln.w);
if (!W) return *err = "enc0: missing tensor " + ln.w, false;
if (!need(*enc, "enc0", ln.b, (size_t)ln.C_out, &b, err)) return false;
if (!pack_linear(W->f, b->f, ln.C_in, ln.C_out, 0, d, &blob, err))
return *err = "enc0." + ln.w + ": " + *err, false;
} else if (ln.kind == "swin") {
if (!block(ln, &blob)) return false;
} else if (ln.kind == "project") {
const int n_out = 64;
const Tensor *W, *b;
if (!need(*enc, "enc0", ln.w, (size_t)n_out * sum(ln.groups), &W, err) ||
!need(*enc, "enc0", ln.b, (size_t)n_out, &b, err))
return false;
if (!pack_project(W->f, b->f, ln.groups, n_out, d, &blob, err)) return *err = "enc0: " + *err, false;
}
(*out)[ln.blob] = std::move(blob);
}
for (const Launch& ln : decoder_launches()) {
std::vector<uint8_t> blob;
if (ln.kind == "dec_project") {
const Tensor *W, *b;
if (!need(*dec, "dec0", ln.w, (size_t)128 * sum(ln.groups), &W, err) ||
!need(*dec, "dec0", ln.b, 128, &b, err))
return false;
if (!pack_dec_project(W->f, b->f, ln.groups, d, &blob, err)) return *err = "dec0: " + *err, false;
} else if (ln.kind == "swin") {
if (!block(ln, &blob)) return false;
} else if (ln.kind == "coef") {
// Model M's dec0 blob with a zero projection: the scalars S, T and the code tables
StageTensors t;
t["W"].f.assign(68 * 64, 0.0f);
t["b"].f.assign(68, 0.0f);
for (const char* s : {"S", "T"}) {
const Tensor* x;
if (!need(*dec, "dec0", s, 1, &x, err)) return false;
t[s] = *x;
}
if (!pack_stage_blob("dec0", t, d, &blob, err)) return false;
}
(*out)[ln.blob] = std::move(blob);
}
return true;
}

} // namespace l
} // namespace wgrt
63 changes: 63 additions & 0 deletions clean/l/runtime/l_pack.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// SPDX-License-Identifier: MIT
// Model L weight packing for the runtime (#89): C++ port of clean/l/runner/l_pack.py (pack_model_l) for the default
// structure table (clean/l/reference/table.py TABLE), on top of the runtime's packing building blocks
// (clean/runtime/model.h namespace pack_parts). The blobs are byte-identical to the Python packer's
// (clean/l/runtime/tests/test_l_runtime_pack.py). Kernels, arguments and launch order: clean/l/FORMATS.md.
//
// The ten Swin stages Model L shares with Model M use Model M's packer (pack_stage_blob) unchanged; this file packs
// Model L's own launches: the encoder (values, two input linears, the width-32 block, the projection of each token's
// 2 x 2 pixels) and the decoder (projection to 32 values per pixel, the width-32 block, the coefficient functions).
// Stage and tensor names are the preset-L map's (data/preset-L.v1.map.json).
#pragma once

#include <cstdint>
#include <map>
#include <string>
#include <vector>

#include "../../runtime/model.h"

namespace wgrt {
namespace l {

// One launch of Model L's encoder or decoder (l_pack.encoder_launches / decoder_launches). kind: "values", "linear",
// "swin", "project" (encoder); "dec_project", "swin", "coef" (decoder).
struct Launch {
std::string kind, blob, src, dst;
int C_in = 0, C_out = 0; // linear: input / output channels
std::string w, b, acc, out_round; // linear, project, dec_project: tensor names; linear: roundings
int C = 0, heads = 0, mlp = 0; // values: channels of its pixel map; swin: width, heads, MLP
std::string offsets, params; // swin: window-offset kind ("first", "second"), the block's stage
std::vector<int> groups; // project, dec_project: input channel groups
float clamp = 0.0f; // project: output clamp (0 = none)
std::vector<int> values; // coef: the value positions (table.DEC0_VALUES)
};

// The default table's launches, in order (mirrors of l_pack.encoder_launches(TABLE) / decoder_launches(TABLE); the
// test compares them field by field).
const std::vector<Launch>& encoder_launches();
const std::vector<Launch>& decoder_launches();

// K order of a projection: K[16 kb + 8 h + j] = 32 (kb / 2) + 16 h + 8 (kb % 2) + j (l_pack.kord_groups32).
std::vector<int> kord_groups32(int n);

// One linear blob: W [C_out][n] reading input channels lo .. lo + n of a C_in-wide map (zero weights elsewhere),
// bias [C_out] (l_pack.pack_linear).
bool pack_linear(const std::vector<float>& W, const std::vector<float>& b, int C_in, int C_out, int lo, const DRow& d,
std::vector<uint8_t>* out, std::string* err);
// The encoder's projection blob: W [n_out][sum(groups)], groups of 32 channels (l_pack.pack_project).
bool pack_project(const std::vector<float>& W, const std::vector<float>& b, const std::vector<int>& groups, int n_out,
const DRow& d, std::vector<uint8_t>* out, std::string* err);
// The decoder's projection blob: W [4][32][sum(groups)], b [4][32] (l_pack.pack_dec_project).
bool pack_dec_project(const std::vector<float>& W, const std::vector<float>& b, const std::vector<int>& groups,
const DRow& d, std::vector<uint8_t>* out, std::string* err);

// Every blob of Model L's own launches (l_pack.pack_model_l). `params` holds the map's stages by name: "enc0",
// "enc0_block", "dec0", "dec0_block" (float32 values in row-major order; enc0's dither table "T" as uint8 values
// 0..127 in Tensor::u, as the runtime's loader keeps it). Blob names as l_pack: "enc0.values", "enc0.Win1",
// "enc0.Win2", "enc0_block", "enc0.project", "dec0.project", "dec0_block", "dec0.coef".
bool pack_model(const std::map<std::string, StageTensors>& params, const DRow& d,
std::map<std::string, std::vector<uint8_t>>* out, std::string* err);

} // namespace l
} // namespace wgrt
Loading
Loading