Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 31 additions & 26 deletions clean/l/runtime/l_model.cpp
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// SPDX-License-Identifier: MIT
// Model L's model data in the runtime (see l_model.h). Shapes mirror clean/l/reference/table.py for the default table
// (checked by clean/l/runtime/tests/test_l_runtime_load.py).
// Model L's model data in the runtime (see l_model.h). Checked against the preset-L map and the structure table by
// clean/l/runtime/tests/test_l_runtime_load.py.
#include "l_model.h"

namespace wgrt {
Expand All @@ -10,38 +10,43 @@ namespace {

const int kHeadDim = 32, kWindowTokens = 16;

// One Swin block's tensors with a name prefix (table.block_shapes).
void add_block(std::vector<TensorSpec>* v, const std::string& prefix, int C, int heads, int mlp) {
const std::vector<TensorSpec> b = {{prefix + "gamma1", {C}, false},
{prefix + "gamma2", {C}, false},
{prefix + "Wqk", {heads, kHeadDim, C}, false},
{prefix + "Wv", {heads, kHeadDim, C}, false},
{prefix + "B", {heads, kWindowTokens, kWindowTokens}, false},
{prefix + "Wp", {C, kHeadDim * heads}, false},
{prefix + "bp", {C}, false},
{prefix + "W1", {mlp, C}, false},
{prefix + "b1", {mlp}, false},
{prefix + "W2", {C, mlp}, false},
{prefix + "b2", {C}, false}};
v->insert(v->end(), b.begin(), b.end());
// One Swin block's tensors (table.block_shapes; the map's block record set).
std::vector<TensorSpec> block_specs(int C, int heads, int mlp) {
return {{"gamma1", {C}, false},
{"gamma2", {C}, false},
{"Wqk", {heads, kHeadDim, C}, false},
{"Wv", {heads, kHeadDim, C}, false},
{"B", {heads, kWindowTokens, kWindowTokens}, false},
{"Wp", {C, kHeadDim * heads}, false},
{"bp", {C}, false},
{"W1", {mlp, C}, false},
{"b1", {mlp}, false},
{"W2", {C, mlp}, false},
{"b2", {C}, false}};
}

} // namespace

StageSpecs stage_specs() {
StageSpecs s;
// enc0: the projection reads [raw (80) | x (32) | 16 zeros] = 128 inputs; the first linear reads the 20 values of
// one sub-pixel (table: select 0..20, linear 20 -> 32), the second 32 -> 32; then the width-32 block.
std::vector<TensorSpec> enc0 = {{"W", {64, 128}, false}, {"b", {64}, false}, {"T", {8, 16, 16}, true},
{"sigma1", {1}, false}, {"sigma2", {1}, false}, {"Wa", {32, 20}, false},
{"ba", {32}, false}, {"Wb", {32, 32}, false}, {"bb", {32}, false}};
add_block(&enc0, "blk.", 32, 2, 128);
s.emplace_back("enc0", enc0);
// enc0: the projection reads the 2 x 2 pixels of 32 channels of each token (128 inputs); the first input linear
// reads the 20 values of a pixel, the second 32.
s.emplace_back("enc0", std::vector<TensorSpec>{{"W", {64, 128}, false},
{"b", {64}, false},
{"T", {8, 16, 16}, true},
{"sigma1", {1}, false},
{"sigma2", {1}, false},
{"Win1", {32, 20}, false},
{"bin1", {32}, false},
{"Win2", {32, 32}, false},
{"bin2", {32}, false}});
s.emplace_back("enc0_block", block_specs(32, 2, 128));
for (const std::string& st : model_stages())
if (st != "enc0" && st != "dec0" && st != "post") s.emplace_back(st, wgrt::stage_specs(st));
std::vector<TensorSpec> dec0 = wgrt::stage_specs("dec0");
add_block(&dec0, "blk.", 32, 2, 128);
s.emplace_back("dec0", dec0);
s.emplace_back("dec0_block", block_specs(32, 2, 128));
// dec0: 32 outputs per sub-pixel (Model M: 17), then Model M's scalars
s.emplace_back("dec0", std::vector<TensorSpec>{
{"W", {4, 32, 64}, false}, {"b", {4, 32}, false}, {"S", {1}, false}, {"T", {1}, false}});
s.emplace_back("post", wgrt::stage_specs("post"));
return s;
}
Expand Down
14 changes: 8 additions & 6 deletions clean/l/runtime/l_model.h
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
// SPDX-License-Identifier: MIT
// Model L's model data in the runtime (#89): the stages and tensors of Model L in the converter's raw layout, and the
// raw loader for them (clean/runtime/model.h load_raw_stages). Names are the logical names of the default structure
// table (clean/l/reference/table.py: encoder_shapes, decoder_shapes); the ten Swin stages and post are Model M's.
// raw loader for them (clean/runtime/model.h load_raw_stages). Stages, names and shapes are those of the preset-L map
// (data/preset-L.v1.map.json), which the structure table (clean/l/reference/table.py param_shapes) groups the same way:
//
// enc0: Model M's W, b, T, sigma1, sigma2 (W is the projection, 64 x 128) + the two linears Wa / ba, Wb / bb + the
// width-32 block "blk.*"
// enc0: W (the projection, 64 x 128), b, T, sigma1, sigma2, and the two input linears Win1 / bin1 (32 x 20),
// Win2 / bin2 (32 x 32)
// enc0_block: the encoder's width-32 Swin block
// enc1 ... dec1: Model M's stages (clean/runtime/model.cpp stage_specs)
// dec0: Model M's W, b, S, T + the width-32 block "blk.*"
// dec0_block: the decoder's width-32 Swin block
// dec0: W (4 x 32 x 64), b (4 x 32), S, T
// post: Model M's s
#pragma once

Expand All @@ -18,7 +20,7 @@
namespace wgrt {
namespace l {

// The stages in load order: enc0, enc1 ... dec1 (model_stages() order), dec0, post.
// The stages in the map's order: enc0, enc0_block, enc1 ... dec1, dec0_block, dec0, post.
StageSpecs stage_specs();

// load_raw_stages with stage_specs(): <model_dir>[/raw]/<stage>/<name>.bin.
Expand Down
42 changes: 26 additions & 16 deletions clean/l/runtime/tests/test_l_runtime_load.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# SPDX-License-Identifier: MIT
"""Model L's raw model loading in the runtime (clean/l/runtime/l_model.cpp) on the CPU (#89):

- the tensor list against the default table (clean/l/reference/table.py encoder_shapes / decoder_shapes: same names,
same number of values) and, for the ten Swin stages and post, against Model M's raw tensors
(clean/runner/synthetic_model.py);
- the tensor list against the preset-L map (data/preset-L.v1.map.json: stages, names, shapes, types, in order),
against the structure table (clean/l/reference/table.py param_shapes: names, number of values) and, for the ten
Swin stages and post, against Model M's raw tensors (clean/runner/synthetic_model.py);
- a synthetic Model L in the raw layout (clean/l/reference/oracle.py random_params) loads with every tensor's bytes
unchanged;
- the refusals (missing tensor, wrong size, T above 127).
Expand All @@ -15,6 +15,7 @@
python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_load.py
"""
import hashlib
import json
import os
import shutil
import subprocess
Expand Down Expand Up @@ -116,25 +117,34 @@ def numel(shape):


class Specs(unittest.TestCase):
def test_map(self):
"""Stages, names, shapes and types of the preset-L map (data/preset-L.v1.map.json), in its order."""
with open(os.path.join(ROOT, "data", "preset-L.v1.map.json"), encoding="utf-8") as f:
m = json.load(f)
want = {st["stage"]: {t["name"]: (tuple(t["shape"]), t["dtype"] == "uint8") for t in st["tensors"]}
for st in m["stages"]}
s = specs()
self.assertEqual(list(s), list(want))
for stage in want:
self.assertEqual(list(s[stage].items()), list(want[stage].items()), stage)

def test_table(self):
"""The structure table's parameters (table.param_shapes): same stages and names, same number of values."""
s = specs()
self.assertEqual(list(s), ["enc0"] + list(SHARED) + ["dec0", "post"])
for stage, want in (("enc0", T.encoder_shapes()), ("dec0", T.decoder_shapes())):
for stage, want in T.param_shapes().items():
self.assertEqual(sorted(s[stage]), sorted(want), stage)
for name, shape in want.items():
self.assertEqual(numel(s[stage][name][0]), numel(shape), "%s/%s" % (stage, name))
self.assertEqual(s[stage][name][1], name == "T" and stage == "enc0", "%s/%s" % (stage, name))
self.assertEqual(list(s)[0], "enc0")
self.assertEqual(list(s["enc0"])[0], "W") # the raw root is found by enc0/W.bin, as for Model M

def test_shared_stages_are_model_m(self):
s, m = specs(), SM.synthetic_tensors(1)
for stage in SHARED + ("post",):
self.assertEqual({k: v[0] for k, v in s[stage].items()}, {k: a.shape for k, a in m[stage].items()}, stage)
for name, a in m["dec0"].items(): # Model M's dec0 tensors keep their raw shapes inside L's dec0
self.assertEqual(s["dec0"][name][0], a.shape, name)
for name, a in m["enc0"].items():
if name != "W": # L's projection is 64 x 128 (Model M: 64 x 80)
self.assertEqual(s["enc0"][name][0], a.shape, name)
for stage, names in (("enc0", ("b", "T", "sigma1", "sigma2")), ("dec0", ("S", "T"))):
for name in names: # Model M's raw shapes (L: projection 64 x 128, dec0 32 outputs per sub-pixel)
self.assertEqual(s[stage][name][0], m[stage][name].shape, name)


class Load(unittest.TestCase):
Expand All @@ -160,13 +170,13 @@ def test_refusals(self):
try:
write_raw(d, model)
raw = d + "/raw" # as the loader names it
f = os.path.join(d, "raw", "dec0", "blk.W1.bin")
f = os.path.join(d, "raw", "dec0_block", "W1.bin")
os.remove(f)
self.assertEqual(dump("load", d), ["ERR model data missing: " + raw + "/dec0/blk.W1.bin"])
model["dec0"]["blk.W1"][:64].tofile(f)
self.assertEqual(dump("load", d), ["ERR model data damaged: " + raw + "/dec0/blk.W1.bin has 8192 bytes,"
self.assertEqual(dump("load", d), ["ERR model data missing: " + raw + "/dec0_block/W1.bin"])
model["dec0_block"]["W1"][:64].tofile(f)
self.assertEqual(dump("load", d), ["ERR model data damaged: " + raw + "/dec0_block/W1.bin has 8192 bytes,"
" expected 16384"])
model["dec0"]["blk.W1"].tofile(f)
model["dec0_block"]["W1"].tofile(f)
t = model["enc0"]["T"].copy()
t.flat[7] = 200
t.tofile(os.path.join(d, "raw", "enc0", "T.bin"))
Expand Down
Loading