Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions clean/converter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,24 @@ the frame-generation library's two convolution networks, each a block source wit
a padding area or take one copy of a stored-several-times bias). The preset requires no source by name. The source
the key is made from is searched first: when it is missing, the file has no frame-generation model (exit 12, preset FG's
message); any other missing blob is exit 13. A map without the field is still read as preset M (so it must have W, D
and K) unless the caller names the preset (`--preset FG`). The spec side's map is not delivered yet; `tests/run_tests.py`
(`test_fg`) converts a synthetic library with our own random blobs in the frame-generation layout
([`clean/fg/layout.py`](../fg/layout.py): every blob's real size, shuffled order, aligned, even and odd offsets, a decoy),
with a stand-in map (`tests/cases.py` `fg_map`) whose tensor names and axis order are placeholders. The padding areas
and the extra bias copies of the synthetic blobs hold NaN, so a map that read them would fail (exit 22).
and K) unless the caller names the preset (`--preset FG`).

Its map is `data/fg.v1.map.json` (the spec side's v1; 67 sources `A01`..`A17`, `B01w`/`B01b`..`B25w`/`B25b`; 42
stages `dl1.l01`..`dl2.l25`): `wronggpu-convert --map data\fg.v1.map.json --dll <frame-generation library> --out <dir>`.
It uses two map features of its own:

- `"channel_order": "storage"`: the weight tensors (`W_storage`) are the blobs' weight spans as stored, 1-D, not yet
reshaped to natural axes (that follows in a later map revision). Accepted only in a version-1 map of preset FG; the
converter reads the tensors the same way either way.
- An axis given by **explicit offsets** (`{"offsets": [o0, o1, ...]}`, one per index, in place of `stride` and `perm`;
map versions 1 and 4): the axis contributes `offsets[i]` to the element index. The net1 biases use it to take copy 0
of each group of eight. The index range is checked when the map is loaded (smallest and largest offset); a list of
the wrong length, a non-integer, or `offsets` together with `stride` or `perm` is "converter data damaged".

`tests/run_tests.py` pins the map's SHA-256 (`FG_MAP_SHA256`), checks its sources and stages against
[`clean/fg/layout.py`](../fg/layout.py), and converts a synthetic library with our own random blobs in exactly the map's
layout (`tests/cases.py` `fg_case`: every blob's real size, shuffled order, aligned, even and odd offsets, a decoy).
The zero areas and the extra bias copies of the synthetic blobs hold NaN, so a map that read them would fail (exit 22).

## Preset F (map version 5, spec 57)

Expand Down
4 changes: 4 additions & 0 deletions clean/converter/src/convert.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ bool decode_tensor(const Map& map, const Tensor& t, const std::vector<std::uint8
const Axis& ax = t.axes[a];
contrib[a].resize(static_cast<std::size_t>(ax.length));
for (std::int64_t i = 0; i < ax.length; ++i) {
if (!ax.offsets.empty()) { // explicit offsets (map versions 1 and 4; range-checked by load_map)
contrib[a][static_cast<std::size_t>(i)] = ax.offsets[static_cast<std::size_t>(i)];
continue;
}
std::int64_t pi = i;
if (ax.perm >= 0) {
const std::vector<std::int64_t>& q = map.perms[static_cast<std::size_t>(ax.perm)].table;
Expand Down
35 changes: 33 additions & 2 deletions clean/converter/src/map.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,15 @@ class Loader {
}
std::string order;
if (!need_string(root, "channel_order", "map", order)) return false;
if (order != "natural") return damaged("unknown channel_order \"" + order + "\"");
// "storage" (weights as stored, not yet reshaped to natural axes): preset FG's map v1 only. It names what the
// tensors hold; the converter reads them the same way.
if (order == "storage") {
if (m.preset_name != "FG" || version_ != 1)
return damaged("channel_order \"storage\" is only for preset FG (map version 1)");
} else if (order != "natural") {
return damaged("unknown channel_order \"" + order + "\"");
}
m.channel_order = order;
if (version_ == 5) return load_v5(root, m);
// Spec 57 section 5.1: a map of versions 1-4 that uses a version-5 feature is invalid.
if (root.find("groups")) return damaged("map: \"groups\" is a version-5 feature");
Expand Down Expand Up @@ -681,8 +689,24 @@ class Loader {
if (!aj.is_object()) return damaged(where + ": axis must be an object");
Axis ax;
ax.length = t.shape[a];
if (!need_int(aj, "stride", where + ", axis " + std::to_string(a), ax.stride)) return false;
const std::string wa = where + ", axis " + std::to_string(a);
if (aj.find("limit")) return damaged(where + ": an axis \"limit\" is a version-5 feature");
if (const Json* oj = aj.find("offsets")) {
// Explicit element offsets, one per index (map versions 1 and 4): no stride, no perm with them.
if (k_) return damaged(wa + ": \"offsets\" is not a feature of map versions 2 and 3");
if (aj.find("stride") || aj.find("perm")) return damaged(wa + ": \"offsets\" with \"stride\" or \"perm\"");
if (!oj->is_array() || oj->items.size() != static_cast<std::size_t>(ax.length))
return damaged(wa + ": offsets must be a list with one entry per index of the axis");
for (const Json& v : oj->items) {
std::int64_t o;
if (!v.as_int64(o)) return damaged(wa + ": an offset is not an integer");
if (o >= (std::int64_t(1) << 62) || o <= -(std::int64_t(1) << 62)) return out_of_range(wa + ": offset overflow");
ax.offsets.push_back(o);
}
t.axes.push_back(ax);
continue;
}
if (!need_int(aj, "stride", wa, ax.stride)) return false;
if (const Json* pj = aj.find("perm")) {
if (!pj->is_string()) return damaged(where + ": perm must be text");
for (std::size_t p = 0; p < m.perms.size(); ++p)
Expand All @@ -698,6 +722,12 @@ class Loader {
// the values stride * j, j = 0..n-1, and the extremes of e are reached.
std::int64_t lo = t.base, hi = t.base;
for (const Axis& ax : t.axes) {
if (!ax.offsets.empty()) { // every combination of offsets occurs, so the extremes are min and max
const auto mm = std::minmax_element(ax.offsets.begin(), ax.offsets.end());
if (!checked_add(lo, *mm.first, lo) || !checked_add(hi, *mm.second, hi))
return out_of_range(where + ": index overflow");
continue;
}
std::int64_t span;
if (!checked_mul(ax.stride, ax.length - 1, span)) return out_of_range(where + ": index overflow");
std::int64_t& end = (span < 0) ? lo : hi;
Expand Down Expand Up @@ -1128,6 +1158,7 @@ class Loader {
if (!aj.is_object()) return damaged(wa + ": must be an object");
Axis ax;
ax.length = t.shape[a];
if (aj.find("offsets")) return damaged(wa + ": \"offsets\" is not a version-5 feature");
if (!need_int(aj, "stride", wa, ax.stride)) return false;
if (ax.stride < 0) return damaged(wa + ": negative strides are not a version-5 feature");
if (aj.find("limit") && !need_nonneg(aj, "limit", wa, ax.limit)) return false;
Expand Down
5 changes: 5 additions & 0 deletions clean/converter/src/map.h
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,9 @@ struct Axis {
std::int64_t stride = 0;
int perm = -1; // index into Map::perms, or -1
std::int64_t limit = -1; // map version 5 (57 section 5.3): elements with pi(i) >= limit are zero; -1 = none
// Map versions 1 and 4: an axis given by explicit element offsets (`offsets`, one per index i, in place of
// `stride` and `perm`): its contribution to the element index is offsets[i]. Empty: the stride form.
std::vector<std::int64_t> offsets;
};

// Map version 5: a second read of the same elements from another source and base (`also`).
Expand Down Expand Up @@ -245,6 +248,8 @@ struct Map {
// load_map checks it is a known preset that allows the map's version.
std::string preset_name;
bool preset_named = false; // the map has the field "preset" (else the name came from the version, or the caller)
// "natural", or "storage" (preset FG map v1: weight tensors are the stored spans, not yet natural axes).
std::string channel_order;
std::vector<Source> sources; // in file order
std::vector<Array> arrays; // what tensors index
int key_source = -1;
Expand Down
62 changes: 25 additions & 37 deletions clean/converter/tests/cases.py
Original file line number Diff line number Diff line change
Expand Up @@ -638,45 +638,33 @@ def fg_blobs(rng):
return out


def fg_map(blobs, label="synthetic FG label", key_from="net1_01"):
"""A version-1 map with "preset": "FG" for the blobs of fg_blobs: one block source per blob (map order = upload
order) and one stage per layer. The tensor layout is a stand-in (weights as [out, K] rows, net1's bias as copy 0
of each group of 8, net2's bias as [out]); the spec side's map defines the real names, axis order and stages."""
def fg_map(fg_map_obj, layout_blobs):
"""The delivered preset-FG map (data/fg.v1.map.json, parsed) with our own blobs in place of the real ones: every
stage, tensor, base, axis and offset stays as delivered; each source keeps its name and text but gets the hashes of
our blob. The map's sources are in upload order, so the n-th source is layout blob n (clean/fg/layout.py; the
lengths are checked). Returns (map, {source name: bytes})."""
L = _fg_layout()
m = {
"format": "wronggpu-converter-map",
"version": 1,
"model": "synthetic frame-generation case",
"preset": "FG",
"channel_order": "natural",
"sources": {b.name: source_entry(b.name, "fake frame-generation weight blob", "f16le", blobs[b.name])
for b in L.blobs()},
"key": {"label": label, "from": key_from},
"perms": {},
"stages": [],
}
f16 = {"name": "f16"}
for l in L.NET1:
m["stages"].append({"stage": l.name, "digest": "", "tensors": [
tensor("W", [l.out, l.k], "float32", l.name, f16, 0, [lin(l.k), lin(1)]),
tensor("b", [l.out // L.NET1_BIAS_GROUP, L.NET1_BIAS_GROUP], "float32", l.name, f16, l.bias_offset,
[lin(L.NET1_BIAS_GROUP * L.NET1_BIAS_COPIES), lin(1)]),
]})
for l in L.NET2:
m["stages"].append({"stage": l.name, "digest": "", "tensors": [
tensor("W", [l.out, 9 * l.cin], "float32", l.name + "_w", f16, 0, [lin(9 * l.cin), lin(1)]),
tensor("b", [l.out], "float32", l.name + "_b", f16, 0, [lin(1)]),
]})
return m


def fg_case(seed=41, drop=(), decoy="net2_07_w"):
"""A synthetic frame-generation library: random bytes holding the 67 blobs of fg_blobs in a shuffled order, at
16-aligned, even and odd offsets, with a decoy of one blob (its first 64 bytes, then other bytes) before it.
drop: blob names left out of the file (the map still lists them). Returns the Case (placements: {blob: offset})."""
m = copy.deepcopy(fg_map_obj)
names = list(m["sources"])
assert len(names) == len(L.blobs())
blobs = {}
for name, b in zip(names, L.blobs()):
data = layout_blobs[b.name]
assert m["sources"][name]["length"] == len(data), name
m["sources"][name] = source_entry(name, m["sources"][name]["what"], "f16le", data)
blobs[name] = data
for st in m["stages"]:
st["digest"] = ""
return m, blobs


def fg_case(fg_map_obj, seed=41, drop=(), decoy="B07w"):
"""A synthetic frame-generation library for the delivered map's layout: random bytes holding our own 67 blobs
(fg_blobs) in a shuffled order, at 16-aligned, even and odd offsets, with a decoy of one blob (its first 64 bytes,
then other bytes) before it. drop: source names left out of the file (the map still lists them). Returns the Case
(placements: {source: offset})."""
rng = random.Random(seed)
blobs = fg_blobs(rng)
m = fg_map(blobs)
m, blobs = fg_map(fg_map_obj, fg_blobs(rng))
order = list(blobs)
rng.shuffle(order)
buf = bytearray(rng.randbytes(4096))
Expand Down
Loading
Loading