diff --git a/clean/converter/README.md b/clean/converter/README.md index dd3635b2..edb5d323 100644 --- a/clean/converter/README.md +++ b/clean/converter/README.md @@ -47,8 +47,20 @@ with that preset's message. It also converts an **L-shaped** map (`tests/cases.py` `l_shaped_map`): preset M's map layout with the input encoder's output linear 64 x 128, and two new stages of L's record kinds (two linears and a C = 32 Swin block before `enc0`, a C = 32 Swin block before `dec0`), filled with our own random data. Its stage names and placement are stand-ins; the real map defines its own. A map of -versions 1-4 must still have the sources W, D and K: should preset L have no dither table or constants, that rule -becomes per preset. +versions 1-4 must have the sources its preset requires (`required_sources` in `known_presets`): W, D and K for presets +M and L. + +**Preset FG** (frame generation, issue #96): `"preset": "FG"` in a map of version 1. Its sources are the weight blobs of +the frame-generation library's two convolution networks, each a block source with its own name, length and hashes +(no keyed source, no constants, no dither table); its tensors use preset M's record kinds (`f16` reads, strides that skip +a padding area or take one copy of a stored-several-times bias). The preset requires no source by name. The source +the key is made from is searched first: when it is missing, the file has no frame-generation model (exit 12, preset FG's +message); any other missing blob is exit 13. A map without the field is still read as preset M (so it must have W, D +and K) unless the caller names the preset (`--preset FG`). The spec side's map is not delivered yet; `tests/run_tests.py` +(`test_fg`) converts a synthetic library with our own random blobs in the frame-generation layout +([`clean/fg/layout.py`](../fg/layout.py): every blob's real size, shuffled order, aligned, even and odd offsets, a decoy), +with a stand-in map (`tests/cases.py` `fg_map`) whose tensor names and axis order are placeholders. The padding areas +and the extra bias copies of the synthetic blobs hold NaN, so a map that read them would fail (exit 22). ## Preset F (map version 5, spec 57) @@ -134,7 +146,8 @@ Preset K, for the spec team (48-T1, 48-T6): | 12 | M: W not found. K: T not found | This DLSS file has no preset-M model. Version 310.5 or newer is needed. / This DLSS file has no preset-K model. Version 310.1 or newer is needed. (+ "This file is version a.b.c.d." when the file has a version resource) | | 12 | F: W not found (spec 57 section 8) | This DLSS file has no preset-F (Ray Reconstruction) model. Version 310.7 or newer is needed. (+ the file version when there is one) | | 12 | L: W not found (a map with `"preset": "L"`) | This DLSS file has no preset-L model. Version 310.5 or newer is needed. (+ the file version when there is one) | -| 13 | M: W found, D or K not found. K: T found, O not found (spec 48 draft 4 section 8). F: W found, another source (or every variant of IMG-IN) not found. The log names the missing source | This DLSS build is not supported yet (model data found, but other parts differ). | +| 12 | FG: the key's source not found (a map with `"preset": "FG"`) | This DLSS Frame Generation file has no frame-generation model. Version 310.1 or newer is needed. (+ the file version when there is one) | +| 13 | M: W found, D or K not found. FG: the key's source found, another blob not found. K: T found, O not found (spec 48 draft 4 section 8). F: W found, another source (or every variant of IMG-IN) not found. The log names the missing source | This DLSS build is not supported yet (model data found, but other parts differ). | | 14 | F only (spec 57 section 8): a constant's sites disagree, a `const_f16x2` with unequal halves, a `const_f16lo` with a non-zero high half, a non-finite value where 57.6.1 or 57.6.2 forbids it, an RN16 overflow, the two reads of an `also` tensor differ, a `require` or check fails, the attention constants fail 57.6.4. The log names the tensor or check (never a value). Not an installer error: the next file is tried | This DLSS build is not supported yet (model data found, but other parts differ). | | 20 | map invalid: unreadable, bad JSON, bad format or version, unknown op / locate / element / encoding / dtype (or one its version does not define), missing field, unknown perm, perm not dividing its axis, `axes` length ≠ `shape` length, a segment that is not a positive multiple of 4 or runs past the end of T, K sources other than T (and O), `capped_softmax` without 8 × 8 last axes, a source F that is not a keyed optional `lz4-block` block source with variants, a field outside the decoded text, variants with different numbers of fields, a `digest` list whose length is not the number of variants (or a list for a group without variants), a group that mixes tensors from F and from other sources, … Also (spec 08 section 4.3): a variant of F matched its hashes, but the block does not decompress to `decoded_length`, does not match `decoded_sha256`, or a field is not eight hexadecimal digits | Installer error: the converter data is damaged. … | | 20 | F: every structural failure of 57 section 5.4 (an index outside its source included) and a defensive bounds failure at run time | Installer error: the converter data is damaged. … | diff --git a/clean/converter/src/api.cpp b/clean/converter/src/api.cpp index e3a355e6..c1318c37 100644 --- a/clean/converter/src/api.cpp +++ b/clean/converter/src/api.cpp @@ -25,6 +25,9 @@ const char* const kMsgNoPresetF = "This DLSS file has no preset-F (Ray Reconstruction) model. Version 310.7 or newer is needed."; // Model L (#89). The minimum version is the first DLSS version confirmed to carry a preset-L model. const char* const kMsgNoPresetL = "This DLSS file has no preset-L model. Version 310.5 or newer is needed."; +// Frame generation (#96): the first frame-generation library version known to carry its two networks' weights. +const char* const kMsgNoPresetFG = + "This DLSS Frame Generation file has no frame-generation model. Version 310.1 or newer is needed."; const char* const kMsgUnsupported = "This DLSS build is not supported yet (model data found, but other parts differ)."; const char* const kMsgMapDamaged = "Installer error: the converter data is damaged. Please reinstall WrongGPU."; const char* const kMsgMapDoesNotFit = diff --git a/clean/converter/src/api.h b/clean/converter/src/api.h index ff753c91..a7e8d26c 100644 --- a/clean/converter/src/api.h +++ b/clean/converter/src/api.h @@ -43,6 +43,7 @@ extern const char* const kMsgNoPresetM; extern const char* const kMsgNoPresetK; extern const char* const kMsgNoPresetF; extern const char* const kMsgNoPresetL; +extern const char* const kMsgNoPresetFG; extern const char* const kMsgUnsupported; extern const char* const kMsgMapDamaged; extern const char* const kMsgMapDoesNotFit; diff --git a/clean/converter/src/main.cpp b/clean/converter/src/main.cpp index 865d519b..f12be2ce 100644 --- a/clean/converter/src/main.cpp +++ b/clean/converter/src/main.cpp @@ -31,7 +31,7 @@ void log_line(const std::string& line) { std::cerr << "[wronggpu-convert] " << l void usage() { std::cout << "usage: wronggpu-convert --map --dll [--dll ...]\n" - " --out [--raw] [--threads N] [--preset M|K|F|L]\n" + " --out [--raw] [--threads N] [--preset M|K|F|L|FG]\n" " --dll may be given several times; the files are tried in order: the first that converts\n" " completely is used (for a map with an optional group, such as preset M's output filter: the\n" " first that has the group, if any has it). The map names its preset (field \"preset\", or by its\n" diff --git a/clean/converter/src/map.cpp b/clean/converter/src/map.cpp index 9dfca8f4..5658e741 100644 --- a/clean/converter/src/map.cpp +++ b/clean/converter/src/map.cpp @@ -166,7 +166,10 @@ bool json_number_exact_f32(const std::string& lit, float& out) { class Loader { public: - explicit Loader(MapError& err) : err_(err) {} + // `hint`: the preset the caller loads the map for ("" : none). A map without the field "preset" is read as that + // preset when the preset uses the map's version, so that its own required sources are checked (load_map below makes + // every other check of the caller's name). + Loader(MapError& err, const std::string& hint) : err_(err), hint_(hint) {} bool damaged(const std::string& what) { err_.kind = MapErrorKind::Damaged; @@ -259,7 +262,8 @@ class Loader { m.preset_name = p->name; m.preset_named = true; } else { - m.preset_name = preset_for_version(version_); + const PresetInfo* h = hint_.empty() ? nullptr : find_preset(hint_); + m.preset_name = h && h->allows(version_) ? std::string(h->name) : preset_for_version(version_); } std::string order; if (!need_string(root, "channel_order", "map", order)) return false; @@ -274,9 +278,13 @@ class Loader { if (!load_source(kv.first, kv.second, m)) return false; } if (!k_) { - for (const char* required : {"W", "D", "K"}) { - if (m.find_source(required) < 0) return damaged(std::string("sources: missing source ") + required); + // The sources the map's preset needs (map.h PresetInfo): W, D and K for presets M and L, none by name for FG. + const PresetInfo* p = find_preset(m.preset_name); + if (!p) return damaged("map: no preset"); + for (const std::string& required : p->required_sources) { + if (m.find_source(required) < 0) return damaged("sources: missing source " + required); } + if (m.sources.empty()) return damaged("sources: the map has no source"); } else if (!check_k_sources(m)) { return false; } @@ -1367,6 +1375,7 @@ class Loader { private: MapError& err_; + std::string hint_; int version_ = 0; bool k_ = false; // versions 2 and 3 (preset K, spec 48) }; @@ -1376,12 +1385,17 @@ class Loader { const std::vector& known_presets() { // the map versions each preset's maps use (bit v: version v), as they were before maps named their preset static const std::vector presets = { - {"M", (1u << 1) | (1u << 4), kMsgNoPresetM}, // spec 08 (version 4 adds the source F) - {"K", (1u << 2) | (1u << 3), kMsgNoPresetK}, // spec 48 - {"F", 1u << 5, kMsgNoPresetF}, // spec 57 + {"M", (1u << 1) | (1u << 4), kMsgNoPresetM, {"W", "D", "K"}}, // spec 08 (version 4 adds the source F) + {"K", (1u << 2) | (1u << 3), kMsgNoPresetK, {}}, // spec 48 (its own source rules: T, O) + {"F", 1u << 5, kMsgNoPresetF, {}}, // spec 57 (its own source rules) // Model L (#89): the same record kinds as preset M's map (no new element format, op or source kind), so the // same map versions; its parameters are a block of their own in the same library file, found by its own hashes - {"L", (1u << 1) | (1u << 4), kMsgNoPresetL}, + {"L", (1u << 1) | (1u << 4), kMsgNoPresetL, {"W", "D", "K"}}, + // Frame generation (#96): the weights of its two convolution networks, a fixed set of separate float16 blobs in + // the frame-generation library, each a block source located by its own length and hashes (no keyed source, no + // constants, no dither table). Map version 1 record kinds only; the map names its own sources. The source the key + // is made from is searched first: when it is missing, the file has no frame-generation model (exit 12). + {"FG", 1u << 1, kMsgNoPresetFG, {}}, }; return presets; } @@ -1398,8 +1412,27 @@ std::string preset_for_version(int version) { return ""; } +namespace { + +bool load_map_hinted(const std::string& file_bytes, Map& out, MapError& err, const std::string& hint) { + out = Map(); + err = MapError(); + out.file_sha256 = Sha256::hash(file_bytes.data(), file_bytes.size()); + Json root; + std::string jerr; + if (!parse_json(file_bytes, root, jerr)) { + err.kind = MapErrorKind::Damaged; + err.detail = "not valid JSON: " + jerr; + return false; + } + Loader loader(err, hint); + return loader.load(root, out); +} + +} // namespace + bool load_map(const std::string& file_bytes, Map& out, MapError& err, const std::string& expected_preset) { - if (!load_map(file_bytes, out, err)) return false; + if (!load_map_hinted(file_bytes, out, err, expected_preset)) return false; if (expected_preset.empty()) return true; auto damaged = [&](const std::string& detail) { err.kind = MapErrorKind::Damaged; @@ -1420,19 +1453,6 @@ bool load_map(const std::string& file_bytes, Map& out, MapError& err, const std: return true; } -bool load_map(const std::string& file_bytes, Map& out, MapError& err) { - out = Map(); - err = MapError(); - out.file_sha256 = Sha256::hash(file_bytes.data(), file_bytes.size()); - Json root; - std::string jerr; - if (!parse_json(file_bytes, root, jerr)) { - err.kind = MapErrorKind::Damaged; - err.detail = "not valid JSON: " + jerr; - return false; - } - Loader loader(err); - return loader.load(root, out); -} +bool load_map(const std::string& file_bytes, Map& out, MapError& err) { return load_map_hinted(file_bytes, out, err, ""); } } // namespace wgc diff --git a/clean/converter/src/map.h b/clean/converter/src/map.h index 2e8b560c..c8287098 100644 --- a/clean/converter/src/map.h +++ b/clean/converter/src/map.h @@ -221,9 +221,12 @@ struct Perm { // model of that preset. A map names its preset in the optional top-level field "preset"; a map without it gets the preset // its version had before the field existed (versions 1 and 4: M; 2 and 3: K; 5: F), so the shipped maps load as before. struct PresetInfo { - const char* name; // "M", "K", "F", "L" + const char* name; // "M", "K", "F", "L", "FG" unsigned versions; // bit v set: map version v is allowed for this preset const char* no_model_message; // api.cpp kMsgNoPreset*: the DLL has no model of this preset + // Map versions 1 and 4: the sources a map of this preset must have, by name (M and L: W, D and K). Preset FG names + // none: its sources are the frame-generation library's weight blobs, each with its own name. + std::vector required_sources; bool allows(int version) const { return version >= 0 && version < 32 && (versions >> version & 1u) != 0; } }; // The known presets, in a fixed order; find_preset: by name (exact, upper case), nullptr when unknown. @@ -253,7 +256,8 @@ struct Map { int find_source(const std::string& name) const; int find_array(const std::string& name) const; // The preset the player messages name, as one letter (the first of preset_name; without one, by the version as before: - // 'M' for versions 1 and 4, 'K' for versions 2 and 3, 'F' for 5). + // 'M' for versions 1 and 4, 'K' for versions 2 and 3, 'F' for 5). Not unique: preset FG also gives 'F'; code that + // must tell presets apart uses preset_name or preset_info(). char preset() const { if (!preset_name.empty()) return preset_name[0]; const std::string p = preset_for_version(version); diff --git a/clean/converter/tests/cases.py b/clean/converter/tests/cases.py index 104d9538..3c56addc 100644 --- a/clean/converter/tests/cases.py +++ b/clean/converter/tests/cases.py @@ -589,3 +589,115 @@ def flip_bit(data, offset, bit=0): b = bytearray(data) b[offset] ^= 1 << bit return bytes(b) + + +# ---------------------------------------------------------------- frame generation (#96): preset FG + +def _fg_layout(): + """clean/fg/layout.py: the frame-generation weight layout (structure only).""" + import os + import sys + root = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "..")) + if root not in sys.path: + sys.path.insert(0, root) + from clean.fg import layout + return layout + + +def _small_halves(rng, n): + """n random finite binary16 patterns with |x| < 2 (exponent field at most 15), fast.""" + from array import array + a = array("H", rng.randbytes(2 * n)) + a = array("H", (x & 0xBFFF for x in a)) + if struct.pack(" converter data damaged", + code == EXIT["map_damaged"] and MSG["map_damaged"] in so and "missing source W" in se, so + se[-300:]) + out2 = r.path("fg", "out-hint", "x")[:-2] + code, so, se = r.run(pp, [dll], out2, extra=["--preset", "FG"]) + r.check("FG map without its field, --preset FG: converts", code == 0 and "preset FG" in se, so + se[-300:]) + with open(pp, "rb") as f: + check_output(r, "FG via --preset", case, out2, f.read()) + for asked in ("L", "M"): + code, so, se = r.run(mp, [dll], r.path("fg", "out-fail", "x")[:-2], extra=["--preset", asked]) + r.check("FG map, --preset %s: converter data damaged" % asked, code == EXIT["map_damaged"], so) + v4 = copy.deepcopy(case.map) + v4["version"] = 4 + code, so, se = r.run(r.write("fg/map-v4.json", v4), [dll], r.path("fg", "out-fail", "x")[:-2]) + r.check("\"preset\": \"FG\" in a version-4 map: converter data damaged", code == EXIT["map_damaged"], so) + + # missing blobs: the key's blob missing -> no frame-generation model (exit 12, FG's message); another blob missing -> + # not supported yet (exit 13), the log names it + no_key = cases.fg_case(seed=41, drop=("net1_01",)) + code, so, se = r.run(mp, [r.write("fg/no-key.dll", no_key.file)], r.path("fg", "out-fail", "x")[:-2]) + r.check("FG: a file without the key's blob: exit 12 with FG's message", code == 12 and MSG_NO_PRESET_FG in so, + "code %d, out %r" % (code, so)) + one_less = cases.fg_case(seed=41, drop=("net2_25_b",)) + code, so, se = r.run(mp, [r.write("fg/one-less.dll", one_less.file)], r.path("fg", "out-fail", "x")[:-2]) + r.check("FG: a file without one blob: exit 13, the log names it", + code == EXIT["unsupported"] and MSG["unsupported"] in so and "missing sources: net2_25_b" in se, so + se[-300:]) + r.check("FG: after the failures the first output is unchanged", os.path.exists(os.path.join(out, "model.wgcm"))) + + # a map that reads net1's zero area (or a bias copy other than 0) meets the test's NaN: exit 22. This shows the + # success case above really skipped them. + for label, fn in (("zero area", lambda t: t.update(base=t["shape"][0] * t["shape"][1])), + ("bias copy 1", lambda t: t.update(base=t["base"] + 8))): + bad = copy.deepcopy(case.map) + st = bad["stages"][0] + fn(st["tensors"][0] if label == "zero area" else st["tensors"][1]) + code, so, se = r.run(r.write("fg/reads-%s.json" % label.replace(" ", "-"), bad), [dll], r.path("fg", "out-fail", "x")[:-2]) + r.check("FG map reading net1's %s: exit 22 (NaN)" % label, code == EXIT["bad_value"], so + se[-300:]) + + def mutate(m, fn): m2 = copy.deepcopy(m) fn(m2) @@ -1153,6 +1219,7 @@ def main(): t0 = time.time() test_errors(r) test_preset_l(r) + test_fg(r) test_filter(r, args.t7) test_k_errors(r) if os.path.exists(args.map): diff --git a/clean/converter/tests/unit_tests.cpp b/clean/converter/tests/unit_tests.cpp index 9913cd7a..45a29dc1 100644 --- a/clean/converter/tests/unit_tests.cpp +++ b/clean/converter/tests/unit_tests.cpp @@ -600,13 +600,21 @@ static void test_api_files(const fs::path& work) { CHECK(preset_for_version(1) == "M" && preset_for_version(4) == "M" && preset_for_version(2) == "K" && preset_for_version(3) == "K" && preset_for_version(5) == "F" && preset_for_version(0).empty() && preset_for_version(6).empty()); CHECK(find_preset("M") && find_preset("K") && find_preset("F") && !find_preset("m") && !find_preset("") && !find_preset("X") && - known_presets().size() == 4); + known_presets().size() == 5); // preset L (#89): the map versions of preset M, its own message; never the preset of a map without the field CHECK(find_preset("L") && find_preset("L")->allows(1) && find_preset("L")->allows(4) && !find_preset("L")->allows(2) && !find_preset("L")->allows(3) && !find_preset("L")->allows(5) && std::string(find_preset("L")->no_model_message) == kMsgNoPresetL && std::string(kMsgNoPresetL) == "This DLSS file has no preset-L model. Version 310.5 or newer is needed."); CHECK(std::string(find_preset("M")->no_model_message) == kMsgNoPresetM && std::string(find_preset("K")->no_model_message) == kMsgNoPresetK && std::string(find_preset("F")->no_model_message) == kMsgNoPresetF); + // preset FG (frame generation, #96): map version 1 only, its own message, no source required by name; M and L need + // W, D and K; never the preset of a map without the field + CHECK(find_preset("FG") && find_preset("FG")->allows(1) && !find_preset("FG")->allows(2) && !find_preset("FG")->allows(3) && + !find_preset("FG")->allows(4) && !find_preset("FG")->allows(5) && find_preset("FG")->required_sources.empty() && + std::string(find_preset("FG")->no_model_message) == kMsgNoPresetFG && !find_preset("fg") && !find_preset("G") && + std::string(kMsgNoPresetFG) == "This DLSS Frame Generation file has no frame-generation model. Version 310.1 or newer is needed."); + CHECK(find_preset("M")->required_sources == std::vector({"W", "D", "K"}) && + find_preset("L")->required_sources == std::vector({"W", "D", "K"}) && preset_for_version(1) != "FG"); { auto with_preset = [&](const std::string& map, const std::string& version, const std::string& field) { std::string t = map; @@ -642,6 +650,13 @@ static void test_api_files(const fs::path& work) { LoadedMap hm; CHECK(load_map_bytes(with_preset(u.map_v1, "1", "\"M\""), hm, sink, "L").code == kMapDamaged && !hm.map); CHECK(logged("the map names preset M, preset L was asked for") && logged("preset \"X\" asked for is not known")); + // preset FG: a version-1 map with or without the field (W, D and K are allowed, just not required); version 4 is not FG's + LoadedMap g1, g2, g3, g4; + CHECK(load_map_bytes(with_preset(u.map_v1, "1", "\"FG\""), g1, sink).code == kOk && g1.map && g1.map->preset_name == "FG" && + g1.map->preset_info() == find_preset("FG")); + CHECK(load_map_bytes(u.map_v1, g2, sink, "FG").code == kOk && g2.map && g2.map->preset_name == "FG" && !g2.map->preset_named); + CHECK(load_map_bytes(with_preset(u.map_v4, "4", "\"FG\""), g3, sink).code == kMapDamaged && !g3.map); + CHECK(load_map_bytes(u.map_v4, g4, sink, "FG").code == kMapDamaged && !g4.map); } for (const char* bad : {"\"K\"", "\"F\"", "\"X\"", "\"m\"", "\"l\"", "\"\"", "7", "null"}) { LoadedMap bm; diff --git a/clean/fg/FORMATS.md b/clean/fg/FORMATS.md new file mode 100644 index 00000000..b3eda42a --- /dev/null +++ b/clean/fg/FORMATS.md @@ -0,0 +1,36 @@ +# Frame generation: formats and open items + +What the clean side has for frame generation, and what it still waits for. **OPEN** marks behaviour that has not been +delivered by the spec side; the code never guesses it (it raises, or takes the choice as an explicit argument). + +## Weights + +- One weight set for every configuration: both FG presets (UI recomposition off and on), every multi-frame count and + index, every combination of optional inputs. The same blobs are in every 310.x frame-generation library. +- Two networks, float16 elements, ReLU activations, no attention, no FP8: + - **net1**: a six-level encoder-decoder on a 2 x 2-folded grid; 17 blobs, each `[weights][zero area][bias x 16]`. + - **net2**: two small encoder-decoders (sub-nets A and B) of 3 x 3 convolutions; 25 layers, a weight blob and a bias + blob each. +- Shapes and byte sizes per layer: [`layout.py`](layout.py) (checked by `tests/test_fg_layout.py`). + +## Converter map + +- Map version 1 with `"preset": "FG"` (converter README, "Preset FG"): one block source per blob, located by length + and hashes, `f16` reads. net1's zero area is skipped by the tensor's base; its bias is read as copy 0 of each group of + eight (`layout.net1_bias_index`). +- The spec side's map file is **OPEN** (not delivered). Until it arrives the converter is tested with our own random + blobs and a stand-in map (`clean/converter/tests/cases.py`, `fg_map`), whose stage and tensor names and axis order are + placeholders. +- A conversion of a real library will be checked by digests only (the converter compares every stage digest the map + gives); no value or byte of it is ever written into the repository or any public text. + +## OPEN (waiting for the spec side) + +1. The converter map file. +2. The in-blob axis order of the weights (tap, input channel, output channel, any tiling), for both networks. +3. Which launch reads which blob (upload order is known; launch order is not). +4. net1's blob 2: a 1 x 1 convolution or a 2 x 2 one. +5. Both networks' grid sizes relative to the output size. +6. What the UI-recomposition preset changes in the stage order. +7. Launch order, the motion-field scatter, the pull pyramid and the UI handling (part 3). +8. Whether the networks run once per real frame or once per generated frame. diff --git a/clean/fg/README.md b/clean/fg/README.md new file mode 100644 index 00000000..9d4f704f --- /dev/null +++ b/clean/fg/README.md @@ -0,0 +1,16 @@ +# Frame generation (issue #96, milestone "Later") + +Groundwork for running the two convolution networks of frame generation. Nothing here is used by the runtime, the +front end or the launcher yet. Status, open items and formats: [FORMATS.md](FORMATS.md). + +| Path | Content | +|---|---| +| `layout.py` | the weight layout of both networks as structure only: per layer its shape, and the byte size and inner layout of each blob, in the library's upload order; totals | +| `tests/test_fg_layout.py` | the table against the totals the spec side gives | +| `../converter` (preset FG) | the converter reads a version-1 map with `"preset": "FG"`: one block source per weight blob; see the converter's README, "Preset FG" | + +``` +python -m unittest discover -s clean/fg/tests -t . +``` + +Test data is synthetic. Model data and file bytes never go into this repository. diff --git a/clean/fg/__init__.py b/clean/fg/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/fg/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/fg/layout.py b/clean/fg/layout.py new file mode 100644 index 00000000..7b11bdb5 --- /dev/null +++ b/clean/fg/layout.py @@ -0,0 +1,170 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (issue #96): the weight layout of its two convolution networks, as structure only. + +Written for this project from the spec side's frame-generation scoping (parts 1 and 2). No weights, no file bytes: +only layer shapes and the byte sizes of the blobs that hold them. One weight set serves every configuration (both FG +presets, every multi-frame count and index, every optional input), and the same blobs are in every 310.x library, so +this one table describes them all. + +Network 1 ("net1"): a six-level encoder-decoder on a 2 x 2-folded grid, 17 blobs. Each blob is + [weights: out * K float16][a zero area of the same size][bias: out * 16 float16] +where K = taps * input channels, and the bias holds the output channels in groups of 8, each group stored 16 times +(group g, copy r, channel j at element g * 128 + r * 8 + j of the bias area). A converter takes copy 0 and skips the +zero area. + +Network 2 ("net2"): two small encoder-decoders (sub-nets A and B), 25 layers, all 3 x 3, each a weight blob of +9 * in * out float16 followed by a bias blob of out float16 (50 blobs). + +The blob order below is the order the library uploads them, which is also their data order. OPEN (not delivered yet; +see FORMATS.md): the in-blob axis order of the weights (tap, input and output channel, any tiling), which launch reads +which blob, whether net1's blob 2 is a 1 x 1 or a 2 x 2 convolution, the grid sizes of both networks relative to the +output, and the converter map itself. Nothing here guesses them. +""" +from dataclasses import dataclass + +F16 = 2 # bytes per element +NET1_BIAS_GROUP = 8 # output channels per bias group +NET1_BIAS_COPIES = 16 # copies of each bias group + + +@dataclass(frozen=True) +class Net1Layer: + index: int # 1-based, upload order + out: int # output channels (= distinct bias values) + k: int # taps * input channels + taps: object # 9 (3 x 3), 1 (1 x 1), or None where the reading is OPEN + + @property + def cin(self): + """Input channels, or None while the tap count is OPEN.""" + return None if self.taps is None else self.k // self.taps + + @property + def weight_elements(self): + return self.out * self.k + + @property + def bias_elements(self): + """Elements of the stored (replicated) bias area.""" + return self.out * NET1_BIAS_COPIES + + @property + def blob_bytes(self): + return F16 * (2 * self.weight_elements + self.bias_elements) + + @property + def zero_offset(self): + """First element of the zero area.""" + return self.weight_elements + + @property + def bias_offset(self): + """First element of the bias area.""" + return 2 * self.weight_elements + + @property + def params(self): + """Weights and distinct biases (the zero area and the bias copies excluded).""" + return self.weight_elements + self.out + + @property + def name(self): + return "net1_%02d" % self.index + + +@dataclass(frozen=True) +class Net2Layer: + index: int # 1-based, upload order + subnet: str # "A" or "B" + cin: int # stored input channels + cin_used: int # input channels that carry data (layer 1 of A stores 16, uses 10) + out: int + + @property + def weight_elements(self): + return 9 * self.cin * self.out + + @property + def weight_bytes(self): + return F16 * self.weight_elements + + @property + def bias_bytes(self): + return F16 * self.out + + @property + def params(self): + return self.weight_elements + self.out + + @property + def name(self): + return "net2_%02d" % self.index + + +def _net1(): + rows = [ # (out, K, taps) + (32, 432, 9), # 3 x 3, 48 -> 32 (level 0) + (64, 32, None), # 32 -> 64: 1 x 1, or 2 x 2 over 8 channels (OPEN) + (128, 576, 9), + (128, 1152, 9), + (256, 1152, 9), + (512, 256, 1), # bottom + (256, 512, 1), # bottom + (256, 256, 1), + (128, 256, 1), + (128, 128, 1), + (128, 128, 1), + (128, 128, 1), + (64, 128, 1), + (64, 64, 1), + (32, 64, 1), + (32, 32, 1), + (16, 32, 1), # network output + ] + return tuple(Net1Layer(i + 1, o, k, t) for i, (o, k, t) in enumerate(rows)) + + +def _net2(): + rows = [("A", 16, 10, 32), ("A", 32, 32, 32), ("A", 32, 32, 64)] + rows += [("A", 64, 64, 64)] * 8 # the eight-layer block + rows += [("A", 64, 64, 96), ("A", 32, 32, 8)] + rows += [("B", 18, 18, 16), ("B", 16, 16, 32)] + rows += [("B", 32, 32, 32)] * 8 # the eight-layer block + rows += [("B", 32, 32, 48), ("B", 16, 16, 8)] + return tuple(Net2Layer(i + 1, s, c, cu, o) for i, (s, c, cu, o) in enumerate(rows)) + + +NET1 = _net1() +NET2 = _net2() + + +@dataclass(frozen=True) +class Blob: + name: str # net1_NN, net2_NN_w, net2_NN_b + nbytes: int + layer: object # the Net1Layer / Net2Layer it belongs to + + +def blobs(): + """All weight blobs in upload (= data) order: the 17 of net1, then net2's weight and bias blob per layer.""" + out = [Blob(l.name, l.blob_bytes, l) for l in NET1] + for l in NET2: + out.append(Blob(l.name + "_w", l.weight_bytes, l)) + out.append(Blob(l.name + "_b", l.bias_bytes, l)) + return out + + +def net1_bias_index(channel): + """Element of copy 0 of an output channel's bias, relative to the bias area.""" + g, j = divmod(channel, NET1_BIAS_GROUP) + return g * NET1_BIAS_GROUP * NET1_BIAS_COPIES + j + + +def totals(): + net1 = sum(l.blob_bytes for l in NET1) + net2 = sum(l.weight_bytes + l.bias_bytes for l in NET2) + return { + "net1_blobs": len(NET1), "net2_blobs": 2 * len(NET2), + "net1_bytes": net1, "net2_bytes": net2, "bytes": net1 + net2, + "net1_params": sum(l.params for l in NET1), "net2_params": sum(l.params for l in NET2), + } diff --git a/clean/fg/tests/__init__.py b/clean/fg/tests/__init__.py new file mode 100644 index 00000000..548d2d44 --- /dev/null +++ b/clean/fg/tests/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: MIT diff --git a/clean/fg/tests/test_fg_layout.py b/clean/fg/tests/test_fg_layout.py new file mode 100644 index 00000000..06714639 --- /dev/null +++ b/clean/fg/tests/test_fg_layout.py @@ -0,0 +1,58 @@ +# SPDX-License-Identifier: MIT +"""Frame generation (#96): the weight layout table (clean/fg/layout.py) against the totals the spec side gives for it. + + python -m unittest clean.fg.tests.test_fg_layout +""" +import unittest + +from clean.fg import layout as L + + +class LayoutTotals(unittest.TestCase): + def test_counts_and_bytes(self): + t = L.totals() + self.assertEqual((t["net1_blobs"], t["net2_blobs"]), (17, 50)) + self.assertEqual(len(L.blobs()), 67) + self.assertEqual(t["net1_bytes"], 3905024) + self.assertEqual(t["net2_bytes"], 963552) + self.assertEqual(t["bytes"], 4868576) + + def test_params(self): + t = L.totals() + self.assertEqual(t["net1_params"], 959792) + self.assertEqual(sum(l.weight_elements for l in L.NET1), 957440) + self.assertEqual(sum(l.out for l in L.NET1), 2352) + self.assertEqual(t["net2_params"], 481776) + + def test_net1_blob_sizes(self): + want = [56320, 10240, 299008, 593920, 1187840, 540672, 532480, 270336, 135168, 69632, 69632, 69632, 34816, + 18432, 9216, 5120, 2560] + self.assertEqual([l.blob_bytes for l in L.NET1], want) + for l in L.NET1: + self.assertEqual(l.out % L.NET1_BIAS_GROUP, 0) + if l.taps is not None: + self.assertEqual(l.k % l.taps, 0) + self.assertIsNone(L.NET1[1].taps) # blob 2: 1 x 1 or 2 x 2, OPEN + self.assertEqual(L.NET1[0].cin, 48) + + def test_net2_blob_sizes(self): + want_w = [9216, 18432, 36864] + [73728] * 8 + [110592, 4608, 5184, 9216] + [18432] * 8 + [27648, 2304] + self.assertEqual([l.weight_bytes for l in L.NET2], want_w) + self.assertEqual([l.bias_bytes for l in L.NET2], [2 * l.out for l in L.NET2]) + self.assertEqual([l.subnet for l in L.NET2].count("A"), 13) + self.assertEqual(L.NET2[0].cin_used, 10) + + def test_blob_order(self): + names = [b.name for b in L.blobs()] + self.assertEqual(names[:17], ["net1_%02d" % i for i in range(1, 18)]) + self.assertEqual(names[17:21], ["net2_01_w", "net2_01_b", "net2_02_w", "net2_02_b"]) + self.assertEqual(names[-1], "net2_25_b") + + def test_net1_bias_copy0(self): + self.assertEqual([L.net1_bias_index(c) for c in range(10)], [0, 1, 2, 3, 4, 5, 6, 7, 128, 129]) + l = L.NET1[0] + self.assertEqual(L.net1_bias_index(l.out - 1) + 15 * 8, l.bias_elements - 1) # last channel's last copy ends the blob + + +if __name__ == "__main__": + unittest.main()