Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions clean/converter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ versions 1-4 that uses a version-5-only feature (`groups`, `field`, `encoding: n
(spec 57 section 5.1); the op names `f16` and `exp` mean spec 08's operations in versions 1-4 and spec 57's in
version 5.

**The preset, by name.** A map may name its preset in the optional top-level field `"preset"` (text: `"M"`, `"K"` or
`"F"`; the table `known_presets` in `src/map.cpp`). The name decides the player's "no preset-X model" message (exit 12);
the version still decides the map's features. A map without the field gets the preset its version always meant
(versions 1 and 4: M; 2 and 3: K; 5: F), so the shipped maps are read exactly as before. A preset name that is not known,
not text, or whose preset does not use the map's version (for example `"preset": "K"` in a version-1 map) is "converter
data damaged". This is how a further preset that reuses an existing map version is added: one row in that table.

## Preset F (map version 5, spec 57)

wronggpu-convert --map data\preset-F.v1.map.json --dll <nvngx_dlssd.dll> [--dll <other> ...] --out <dir> [--raw]
Expand Down
4 changes: 3 additions & 1 deletion clean/converter/src/api.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,9 @@ Result convert_file(const Map& map, const fs::path& dll, const SearchOptions& op
Found fw = locate_source(data.data(), data.size(), ks, nullptr, opt);
if (!fw.found) {
emit(log, "source " + ks.name + " not found (" + hex64(ks.fnv) + ")");
std::string msg = map.preset() == 'F' ? kMsgNoPresetF : map.preset() == 'K' ? kMsgNoPresetK : kMsgNoPresetM;
// the message of the map's preset, by name (map.h known_presets)
const PresetInfo* preset = map.preset_info();
std::string msg = preset ? preset->no_model_message : kMsgNoPresetM;
if (!out.version.empty()) msg += " This file is version " + out.version + ".";
return fail(kNoPresetM, msg);
}
Expand Down
4 changes: 3 additions & 1 deletion clean/converter/src/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ void usage() {
std::cout << "usage: wronggpu-convert --map <map.json> --dll <model dll> [--dll <other> ...]\n"
" --out <dir> [--raw] [--threads N]\n"
" --dll may be given several times; the files are tried in order: the first that converts\n"
" completely is used (for preset M: the first with the output-filter weights, if any has them).\n"
" completely is used (for a map with an optional group, such as preset M's output filter: the\n"
" first that has the group, if any has it). The map names its preset (field \"preset\", or by its\n"
" version: 1 and 4 M, 2 and 3 K, 5 F).\n"
" --raw also writes <dir>/raw/<group>/<name>.bin for every tensor.\n";
}

Expand Down
34 changes: 34 additions & 0 deletions clean/converter/src/map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#include <initializer_list>
#include <set>

#include "api.h"
#include "json.h"
#include "numeric.h"

Expand Down Expand Up @@ -248,6 +249,17 @@ class Loader {
m.version = version_ = static_cast<int>(version);
k_ = version_ == 2 || version_ == 3;
if (!need_string(root, "model", "map", m.model)) return false;
// The preset, by name: the optional field "preset", else the one this version always meant (the shipped maps).
if (const Json* pj = root.find("preset")) {
if (!pj->is_string()) return damaged("map: field \"preset\" must be text");
const PresetInfo* p = find_preset(pj->text);
if (!p) return damaged("map: unknown preset \"" + pj->text + "\"");
if (!p->allows(version_))
return damaged("map: preset " + pj->text + " does not use map version " + std::to_string(version_));
m.preset_name = p->name;
} else {
m.preset_name = preset_for_version(version_);
}
std::string order;
if (!need_string(root, "channel_order", "map", order)) return false;
if (order != "natural") return damaged("unknown channel_order \"" + order + "\"");
Expand Down Expand Up @@ -1360,6 +1372,28 @@ class Loader {

} // namespace

const std::vector<PresetInfo>& known_presets() {
// the map versions each preset's maps use (bit v: version v), as they were before maps named their preset
static const std::vector<PresetInfo> presets = {
{"M", (1u << 1) | (1u << 4), kMsgNoPresetM}, // spec 08 (version 4 adds the source F)
{"K", (1u << 2) | (1u << 3), kMsgNoPresetK}, // spec 48
{"F", 1u << 5, kMsgNoPresetF}, // spec 57
};
return presets;
}

const PresetInfo* find_preset(const std::string& name) {
for (const PresetInfo& p : known_presets())
if (name == p.name) return &p;
return nullptr;
}

std::string preset_for_version(int version) {
for (const PresetInfo& p : known_presets())
if (p.allows(version)) return p.name;
return "";
}

bool load_map(const std::string& file_bytes, Map& out, MapError& err) {
out = Map();
err = MapError();
Expand Down
29 changes: 27 additions & 2 deletions clean/converter/src/map.h
Original file line number Diff line number Diff line change
Expand Up @@ -217,11 +217,29 @@ struct Perm {
std::vector<std::int64_t> table;
};

// A preset the converter knows, by name: the map versions its maps may have, and the player's message when a DLL has no
// model of that preset. A map names its preset in the optional top-level field "preset"; a map without it gets the preset
// its version had before the field existed (versions 1 and 4: M; 2 and 3: K; 5: F), so the shipped maps load as before.
struct PresetInfo {
const char* name; // "M", "K", "F"
unsigned versions; // bit v set: map version v is allowed for this preset
const char* no_model_message; // api.cpp kMsgNoPreset*: the DLL has no model of this preset
bool allows(int version) const { return version >= 0 && version < 32 && (versions >> version & 1u) != 0; }
};
// The known presets, in a fixed order; find_preset: by name (exact, upper case), nullptr when unknown.
const std::vector<PresetInfo>& known_presets();
const PresetInfo* find_preset(const std::string& name);
// The preset a map without a "preset" field has: the one its version always meant ("" for no known version).
std::string preset_for_version(int version);

struct Map {
// 1 or 4 (preset M, spec 08; 4 adds the source F), 2 or 3 (preset K, spec 48), 5 (preset F, spec 57: its
// output groups are kept in `stages` too)
int version = 0;
std::string model;
// The preset, by name ("M", "K", "F"): the map's "preset" field, or preset_for_version(version) without one.
// load_map checks it is a known preset that allows the map's version.
std::string preset_name;
std::vector<Source> sources; // in file order
std::vector<Array> arrays; // what tensors index
int key_source = -1;
Expand All @@ -232,8 +250,15 @@ struct Map {

int find_source(const std::string& name) const;
int find_array(const std::string& name) const;
// The preset the player messages name: 'M' for versions 1 and 4, 'K' for versions 2 and 3, 'F' for 5.
char preset() const { return version == 5 ? 'F' : version == 2 || version == 3 ? 'K' : 'M'; }
// The preset the player messages name, as one letter (the first of preset_name; without one, by the version as before:
// 'M' for versions 1 and 4, 'K' for versions 2 and 3, 'F' for 5).
char preset() const {
if (!preset_name.empty()) return preset_name[0];
const std::string p = preset_for_version(version);
return p.empty() ? 'M' : p[0];
}
// The preset's entry in known_presets() (nullptr only for a map load_map did not accept).
const PresetInfo* preset_info() const { return find_preset(preset_name.empty() ? preset_for_version(version) : preset_name); }
};

enum class MapErrorKind {
Expand Down
31 changes: 31 additions & 0 deletions clean/converter/tests/unit_tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,37 @@ static void test_api_files(const fs::path& work) {
CHECK(load_map_bytes(u.map_v1, m1, sink).code == kOk);
if (!m4.map || !m1.map) return;
CHECK(m4.map->version == 4 && m4.map->preset() == 'M' && m1.map->preset() == 'M');
// The preset by name: a map without the field gets its version's preset as before; with it, the name must be known
// and allow the map's version (map.h known_presets)
CHECK(m4.map->preset_name == "M" && m1.map->preset_name == "M" && m1.map->preset_info() == find_preset("M"));
CHECK(preset_for_version(1) == "M" && preset_for_version(4) == "M" && preset_for_version(2) == "K" && preset_for_version(3) == "K" &&
preset_for_version(5) == "F" && preset_for_version(0).empty() && preset_for_version(6).empty());
CHECK(find_preset("M") && find_preset("K") && find_preset("F") && !find_preset("m") && !find_preset("") && !find_preset("X") &&
known_presets().size() == 3);
CHECK(std::string(find_preset("M")->no_model_message) == kMsgNoPresetM && std::string(find_preset("K")->no_model_message) == kMsgNoPresetK &&
std::string(find_preset("F")->no_model_message) == kMsgNoPresetF);
{
auto with_preset = [&](const std::string& map, const std::string& version, const std::string& field) {
std::string t = map;
const std::string v = "\"version\":" + version;
const std::size_t at = t.find(v);
if (at != std::string::npos) t.insert(at + v.size(), ",\"preset\":" + field);
return t;
};
LoadedMap pm1, pm4;
CHECK(load_map_bytes(with_preset(u.map_v1, "1", "\"M\""), pm1, sink).code == kOk && pm1.map && pm1.map->preset_name == "M" &&
pm1.map->preset() == 'M');
CHECK(load_map_bytes(with_preset(u.map_v4, "4", "\"M\""), pm4, sink).code == kOk && pm4.map && pm4.map->preset_info() == find_preset("M"));
// the same map bytes with and without "preset":"M" describe the same tensors (only the file hash differs)
CHECK(pm4.map && pm4.map->stages.size() == m4.map->stages.size() && pm4.map->sources.size() == m4.map->sources.size());
for (const char* bad : {"\"K\"", "\"F\"", "\"X\"", "\"m\"", "\"\"", "7", "null"}) {
LoadedMap bm;
const Result r = load_map_bytes(with_preset(u.map_v1, "1", bad), bm, sink);
if (r.code != kMapDamaged) std::printf("preset field %s\n", bad);
CHECK(r.code == kMapDamaged && !bm.map);
}
CHECK(logged("unknown preset \"X\"") && logged("preset K does not use map version 1") && logged("field \"preset\" must be text"));
}
const int fi = m4.map->find_source("F");
CHECK(fi >= 0 && m4.map->sources[static_cast<std::size_t>(fi)].variants.size() == 2 && m4.map->stages.size() == 2);
CHECK(m4.map->stages[1].optional() && m4.map->stages[1].variant_source == fi && !m4.map->stages[0].optional());
Expand Down
Loading