From 405957f49626bdd5ab98bf1007b0b7d0a0a046e5 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:52:11 -0500 Subject: [PATCH] Runtime: model.wgcm loading for any stage list; Model L's model.wgcm loader load_wgcm_stages checks a model.wgcm against a map for a given stage list (load_wgcm_model is Model M's list through it, unchanged); wgrt::l::load_wgcm_model uses Model L's list, which is the preset-L map's stages, names and shapes. synthetic_model.write_wgcm takes another model's stage order (Model M's output unchanged). Refs #89 Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/l/runtime/l_model.cpp | 5 +++ clean/l/runtime/l_model.h | 5 +++ clean/l/runtime/tests/l_load_dump.cpp | 8 +++- clean/l/runtime/tests/test_l_runtime_load.py | 47 +++++++++++++++++++- clean/runner/synthetic_model.py | 13 ++++-- clean/runtime/model.cpp | 11 ++++- clean/runtime/model.h | 4 ++ 7 files changed, 84 insertions(+), 9 deletions(-) diff --git a/clean/l/runtime/l_model.cpp b/clean/l/runtime/l_model.cpp index 6691db8b..4dde6e36 100644 --- a/clean/l/runtime/l_model.cpp +++ b/clean/l/runtime/l_model.cpp @@ -55,5 +55,10 @@ bool load_raw_model(const std::string& model_dir, std::map* out, + std::string* err, bool* other_map) { + return load_wgcm_stages(path, map_path, stage_specs(), out, err, other_map); +} + } // namespace l } // namespace wgrt diff --git a/clean/l/runtime/l_model.h b/clean/l/runtime/l_model.h index c5fb9a4f..7cf760d0 100644 --- a/clean/l/runtime/l_model.h +++ b/clean/l/runtime/l_model.h @@ -26,5 +26,10 @@ StageSpecs stage_specs(); // load_raw_stages with stage_specs(): [/raw]//.bin. bool load_raw_model(const std::string& model_dir, std::map* out, std::string* err); +// load_wgcm_stages with stage_specs(): the converter's model.wgcm, checked against the preset-L map at map_path +// (whole-file hash, every stage's keyed digest, names, types, shapes, values; clean/runtime/model.h). +bool load_wgcm_model(const std::string& path, const std::string& map_path, std::map* out, + std::string* err, bool* other_map = nullptr); + } // namespace l } // namespace wgrt diff --git a/clean/l/runtime/tests/l_load_dump.cpp b/clean/l/runtime/tests/l_load_dump.cpp index 2f782ac8..8a1905b7 100644 --- a/clean/l/runtime/tests/l_load_dump.cpp +++ b/clean/l/runtime/tests/l_load_dump.cpp @@ -6,6 +6,8 @@ // l_load_dump load // wgrt::l::load_raw_model; prints "OK " and one line per loaded tensor, " // ", or "ERR " +// l_load_dump wgcm +// wgrt::l::load_wgcm_model; the same output #include #include #include @@ -24,10 +26,12 @@ int main(int argc, char** argv) { } return 0; } - if (mode == "load" && argc > 2) { + if ((mode == "load" && argc > 2) || (mode == "wgcm" && argc > 3)) { std::map model; std::string err; - if (!wgrt::l::load_raw_model(argv[2], &model, &err)) return std::printf("ERR %s\n", err.c_str()), 0; + if (mode == "load" ? !wgrt::l::load_raw_model(argv[2], &model, &err) + : !wgrt::l::load_wgcm_model(argv[2], argv[3], &model, &err)) + return std::printf("ERR %s\n", err.c_str()), 0; std::printf("OK %zu\n", model.size()); for (const auto& [stage, tensors] : model) for (const auto& [name, t] : tensors) { diff --git a/clean/l/runtime/tests/test_l_runtime_load.py b/clean/l/runtime/tests/test_l_runtime_load.py index 385ee441..0987c3fd 100644 --- a/clean/l/runtime/tests/test_l_runtime_load.py +++ b/clean/l/runtime/tests/test_l_runtime_load.py @@ -6,7 +6,9 @@ Swin stages and post, against Model M's raw tensors (clean/runner/synthetic_model.py); - a synthetic Model L in the raw layout (clean/l/reference/oracle.py random_params) loads with every tensor's bytes unchanged; -- the refusals (missing tensor, wrong size, T above 127). +- the refusals (missing tensor, wrong size, T above 127); +- model.wgcm with Model L's stages (synthetic file and map): the same tensors as from the raw layout; refused: a map + shape not kept, a Model M file, a file whose digests the shipped preset-L map does not list. No GPU: clean/l/runtime/tests/l_load_dump.cpp is built with the runtime's model.cpp by the host C++ compiler (CXX, clang++ or g++; on Windows MSVC cl after them, from PATH or through vcvars64.bat; skipped without one). @@ -185,5 +187,48 @@ def test_refusals(self): shutil.rmtree(d, ignore_errors=True) +class Wgcm(unittest.TestCase): + """model.wgcm (clean/converter/OUTPUT-FORMAT.md) with Model L's stages: a synthetic file and its synthetic map + (synthetic_model.write_wgcm in the map's stage order), as the converter writes with the preset-L map.""" + + def setUp(self): + self.d = tempfile.mkdtemp(prefix="wg_lw_") + + def tearDown(self): + shutil.rmtree(self.d, ignore_errors=True) + + def write(self, model, sub="L"): + """(model.wgcm path, synthetic map path) in /.""" + d = os.path.join(self.d, sub) + return os.path.join(d, "model.wgcm"), SM.write_wgcm(d, model, order=list(specs())) + + def test_bytes(self): + model = raw_model(7) + path, map_path = self.write(model) + lines = dump("wgcm", path, map_path) + self.assertEqual(lines[0], "OK %d" % len(model)) + got = {tuple(ln.split(" ")[:2]): ln.split(" ")[2:] for ln in lines[1:]} + want = {(st, n): [str(a.size), hashlib.sha256(np.ascontiguousarray(a).tobytes()).hexdigest()] + for st, ts in model.items() for n, a in ts.items()} + self.assertEqual(got, want) + write_raw(os.path.join(self.d, "raw_copy"), model) # the same tensors as from the raw layout + self.assertEqual(dump("load", os.path.join(self.d, "raw_copy")), lines) + + def test_refusals(self): + model = raw_model(8) + bad = {st: dict(ts) for st, ts in model.items()} + bad["dec0"]["W"] = bad["dec0"]["W"].reshape(8, 16, 64) # same values, not the map's shape + path, map_path = self.write(bad) + self.assertEqual(dump("wgcm", path, map_path), + ["ERR model data damaged: %s: tensor dec0/W has the wrong shape or type" % path]) + m_dir = os.path.join(self.d, "M") # a Model M file (with its map) is not a Model L file + m_map, m_path = SM.write_wgcm(m_dir, SM.synthetic_tensors(1)), os.path.join(m_dir, "model.wgcm") + self.assertEqual(dump("wgcm", m_path, m_map), + ["ERR model data damaged: %s: tensor enc0/W has the wrong shape or type" % m_path]) + path, _ = self.write(model) # the shipped preset-L map: its digests are not those of synthetic data + self.assertEqual(dump("wgcm", path, os.path.join(ROOT, "data", "preset-L.v1.map.json")), + ["ERR model data does not match this installation (converted with a different map): " + path]) + + if __name__ == "__main__": unittest.main() diff --git a/clean/runner/synthetic_model.py b/clean/runner/synthetic_model.py index df2825a7..cf5877ad 100644 --- a/clean/runner/synthetic_model.py +++ b/clean/runner/synthetic_model.py @@ -115,7 +115,7 @@ def read_raw(model_dir): def write_wgcm(model_dir, tensors, key=None, other_filter_variants=1, filter_variant=None, map_file=None, - list_filter=True): + list_filter=True, order=None): """Write /model.wgcm (clean/converter/OUTPUT-FORMAT.md) from a tensor set, and a synthetic map file /synthetic.map.json that carries what the runtime reads from a map: the stage list with the keyed digests. Returns the map's path (Pipeline(map_path=...)). Test aid only: the real file comes from the converter. @@ -125,9 +125,14 @@ def write_wgcm(model_dir, tensors, key=None, other_filter_variants=1, filter_var (default: last). The map always lists the group, also when the tensor set has none (as the shipped map does). map_file: no synthetic map is written; the file records that it was converted with this existing map (its digests will not match: for tests of how the runtime reads a given map). - list_filter=False: a map as before spec 08 draft 2 ("version 1"): thirteen stages, no entry for the group.""" + list_filter=False: a map as before spec 08 draft 2 ("version 1"): thirteen stages, no entry for the group. + order: the stages of another model (e.g. Model L's map order) for the file and the map, in place of Model M's.""" key = hashlib.sha256(b"WrongGPU synthetic model key").digest() if key is None else bytes(key) - order = stage_order(tensors) + stage_list = list(order) if order is not None else MAP_ORDER + if order is None: + order = stage_order(tensors) + else: + order = [s for s in stage_list if s in tensors] + ([FILTER_STAGE] if FILTER_STAGE in tensors else []) data, index, digests = bytearray(), bytearray(), {} def put_str(text): @@ -160,7 +165,7 @@ def put_str(text): data += raw pos += len(raw) assert len(index) == isize - stages = [{"stage": s, "digest": digests[s].hex()} for s in MAP_ORDER] + stages = [{"stage": s, "digest": digests[s].hex()} for s in stage_list if s in digests] others = [hashlib.sha256(b"another variant %d" % i).hexdigest() for i in range(int(other_filter_variants))] own = [digests[FILTER_STAGE].hex()] if FILTER_STAGE in tensors else [] at = len(others) if filter_variant is None else int(filter_variant) diff --git a/clean/runtime/model.cpp b/clean/runtime/model.cpp index efa239ff..cfbe6050 100644 --- a/clean/runtime/model.cpp +++ b/clean/runtime/model.cpp @@ -497,6 +497,13 @@ bool check_values(const std::string& where, const Tensor& t, std::string* err) { bool load_wgcm_model(const std::string& path, const std::string& map_path, std::map* out, std::string* err, bool* other_map) { + StageSpecs stages; + for (const std::string& stage : model_stages()) stages.emplace_back(stage, stage_specs(stage)); + return load_wgcm_stages(path, map_path, stages, out, err, other_map); +} + +bool load_wgcm_stages(const std::string& path, const std::string& map_path, const StageSpecs& specs, + std::map* out, std::string* err, bool* other_map) { if (other_map) *other_map = false; std::vector d; if (!read_all(path, &d)) return *err = "model data missing: " + path, false; @@ -608,10 +615,10 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std:: if (!ix.ok) return *err = bad + ": truncated index", false; // Every stage and tensor the runtime needs, with the shapes of spec 08 section 7.2. - for (const std::string& stage : model_stages()) { + for (const auto& [stage, stage_tensors] : specs) { auto it = got.find(stage); if (it == got.end()) return *err = same_map ? bad + ": stage " + stage + " missing" : not_this_map, false; - for (const TensorSpec& ts : stage_specs(stage)) { + for (const TensorSpec& ts : stage_tensors) { auto t = it->second.find(ts.name); const std::string where = stage + "/" + ts.name; if (t == it->second.end()) return *err = bad + ": tensor " + where + " missing", false; diff --git a/clean/runtime/model.h b/clean/runtime/model.h index 5254acc6..804b7275 100644 --- a/clean/runtime/model.h +++ b/clean/runtime/model.h @@ -70,6 +70,10 @@ bool load_raw_stages(const std::string& model_dir, const StageSpecs& stages, bool load_wgcm_model(const std::string& path, const std::string& map_path, std::map* out, std::string* err, bool* other_map = nullptr); +// The same for any list of stages (another model's, with its map): load_wgcm_model is Model M's list through it. +bool load_wgcm_stages(const std::string& path, const std::string& map_path, const StageSpecs& stages, + std::map* out, std::string* err, bool* other_map = nullptr); + // Packed weight blob of one stage (enc0, enc1..dec1, dec0); post has no blob of its own (its scalar s is passed to // the post kernels). bool pack_stage_blob(const std::string& stage, const StageTensors& t, const DRow& drow, std::vector* out,