Skip to content

Commit 42e2bf9

Browse files
Converter: the delivered preset-FG map, storage order and offset axes (#177)
2 parents e6a98fa + 55c01de commit 42e2bf9

12 files changed

Lines changed: 4877 additions & 64 deletions

File tree

‎clean/converter/README.md‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,24 @@ the frame-generation library's two convolution networks, each a block source wit
5656
a padding area or take one copy of a stored-several-times bias). The preset requires no source by name. The source
5757
the key is made from is searched first: when it is missing, the file has no frame-generation model (exit 12, preset FG's
5858
message); any other missing blob is exit 13. A map without the field is still read as preset M (so it must have W, D
59-
and K) unless the caller names the preset (`--preset FG`). The spec side's map is not delivered yet; `tests/run_tests.py`
60-
(`test_fg`) converts a synthetic library with our own random blobs in the frame-generation layout
61-
([`clean/fg/layout.py`](../fg/layout.py): every blob's real size, shuffled order, aligned, even and odd offsets, a decoy),
62-
with a stand-in map (`tests/cases.py` `fg_map`) whose tensor names and axis order are placeholders. The padding areas
63-
and the extra bias copies of the synthetic blobs hold NaN, so a map that read them would fail (exit 22).
59+
and K) unless the caller names the preset (`--preset FG`).
60+
61+
Its map is `data/fg.v1.map.json` (the spec side's v1; 67 sources `A01`..`A17`, `B01w`/`B01b`..`B25w`/`B25b`; 42
62+
stages `dl1.l01`..`dl2.l25`): `wronggpu-convert --map data\fg.v1.map.json --dll <frame-generation library> --out <dir>`.
63+
It uses two map features of its own:
64+
65+
- `"channel_order": "storage"`: the weight tensors (`W_storage`) are the blobs' weight spans as stored, 1-D, not yet
66+
reshaped to natural axes (that follows in a later map revision). Accepted only in a version-1 map of preset FG; the
67+
converter reads the tensors the same way either way.
68+
- An axis given by **explicit offsets** (`{"offsets": [o0, o1, ...]}`, one per index, in place of `stride` and `perm`;
69+
map versions 1 and 4): the axis contributes `offsets[i]` to the element index. The net1 biases use it to take copy 0
70+
of each group of eight. The index range is checked when the map is loaded (smallest and largest offset); a list of
71+
the wrong length, a non-integer, or `offsets` together with `stride` or `perm` is "converter data damaged".
72+
73+
`tests/run_tests.py` pins the map's SHA-256 (`FG_MAP_SHA256`), checks its sources and stages against
74+
[`clean/fg/layout.py`](../fg/layout.py), and converts a synthetic library with our own random blobs in exactly the map's
75+
layout (`tests/cases.py` `fg_case`: every blob's real size, shuffled order, aligned, even and odd offsets, a decoy).
76+
The zero areas and the extra bias copies of the synthetic blobs hold NaN, so a map that read them would fail (exit 22).
6477

6578
## Preset F (map version 5, spec 57)
6679

‎clean/converter/src/convert.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -155,6 +155,10 @@ bool decode_tensor(const Map& map, const Tensor& t, const std::vector<std::uint8
155155
const Axis& ax = t.axes[a];
156156
contrib[a].resize(static_cast<std::size_t>(ax.length));
157157
for (std::int64_t i = 0; i < ax.length; ++i) {
158+
if (!ax.offsets.empty()) { // explicit offsets (map versions 1 and 4; range-checked by load_map)
159+
contrib[a][static_cast<std::size_t>(i)] = ax.offsets[static_cast<std::size_t>(i)];
160+
continue;
161+
}
158162
std::int64_t pi = i;
159163
if (ax.perm >= 0) {
160164
const std::vector<std::int64_t>& q = map.perms[static_cast<std::size_t>(ax.perm)].table;

‎clean/converter/src/map.cpp‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,15 @@ class Loader {
267267
}
268268
std::string order;
269269
if (!need_string(root, "channel_order", "map", order)) return false;
270-
if (order != "natural") return damaged("unknown channel_order \"" + order + "\"");
270+
// "storage" (weights as stored, not yet reshaped to natural axes): preset FG's map v1 only. It names what the
271+
// tensors hold; the converter reads them the same way.
272+
if (order == "storage") {
273+
if (m.preset_name != "FG" || version_ != 1)
274+
return damaged("channel_order \"storage\" is only for preset FG (map version 1)");
275+
} else if (order != "natural") {
276+
return damaged("unknown channel_order \"" + order + "\"");
277+
}
278+
m.channel_order = order;
271279
if (version_ == 5) return load_v5(root, m);
272280
// Spec 57 section 5.1: a map of versions 1-4 that uses a version-5 feature is invalid.
273281
if (root.find("groups")) return damaged("map: \"groups\" is a version-5 feature");
@@ -681,8 +689,24 @@ class Loader {
681689
if (!aj.is_object()) return damaged(where + ": axis must be an object");
682690
Axis ax;
683691
ax.length = t.shape[a];
684-
if (!need_int(aj, "stride", where + ", axis " + std::to_string(a), ax.stride)) return false;
692+
const std::string wa = where + ", axis " + std::to_string(a);
685693
if (aj.find("limit")) return damaged(where + ": an axis \"limit\" is a version-5 feature");
694+
if (const Json* oj = aj.find("offsets")) {
695+
// Explicit element offsets, one per index (map versions 1 and 4): no stride, no perm with them.
696+
if (k_) return damaged(wa + ": \"offsets\" is not a feature of map versions 2 and 3");
697+
if (aj.find("stride") || aj.find("perm")) return damaged(wa + ": \"offsets\" with \"stride\" or \"perm\"");
698+
if (!oj->is_array() || oj->items.size() != static_cast<std::size_t>(ax.length))
699+
return damaged(wa + ": offsets must be a list with one entry per index of the axis");
700+
for (const Json& v : oj->items) {
701+
std::int64_t o;
702+
if (!v.as_int64(o)) return damaged(wa + ": an offset is not an integer");
703+
if (o >= (std::int64_t(1) << 62) || o <= -(std::int64_t(1) << 62)) return out_of_range(wa + ": offset overflow");
704+
ax.offsets.push_back(o);
705+
}
706+
t.axes.push_back(ax);
707+
continue;
708+
}
709+
if (!need_int(aj, "stride", wa, ax.stride)) return false;
686710
if (const Json* pj = aj.find("perm")) {
687711
if (!pj->is_string()) return damaged(where + ": perm must be text");
688712
for (std::size_t p = 0; p < m.perms.size(); ++p)
@@ -698,6 +722,12 @@ class Loader {
698722
// the values stride * j, j = 0..n-1, and the extremes of e are reached.
699723
std::int64_t lo = t.base, hi = t.base;
700724
for (const Axis& ax : t.axes) {
725+
if (!ax.offsets.empty()) { // every combination of offsets occurs, so the extremes are min and max
726+
const auto mm = std::minmax_element(ax.offsets.begin(), ax.offsets.end());
727+
if (!checked_add(lo, *mm.first, lo) || !checked_add(hi, *mm.second, hi))
728+
return out_of_range(where + ": index overflow");
729+
continue;
730+
}
701731
std::int64_t span;
702732
if (!checked_mul(ax.stride, ax.length - 1, span)) return out_of_range(where + ": index overflow");
703733
std::int64_t& end = (span < 0) ? lo : hi;
@@ -1128,6 +1158,7 @@ class Loader {
11281158
if (!aj.is_object()) return damaged(wa + ": must be an object");
11291159
Axis ax;
11301160
ax.length = t.shape[a];
1161+
if (aj.find("offsets")) return damaged(wa + ": \"offsets\" is not a version-5 feature");
11311162
if (!need_int(aj, "stride", wa, ax.stride)) return false;
11321163
if (ax.stride < 0) return damaged(wa + ": negative strides are not a version-5 feature");
11331164
if (aj.find("limit") && !need_nonneg(aj, "limit", wa, ax.limit)) return false;

‎clean/converter/src/map.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,9 @@ struct Axis {
141141
std::int64_t stride = 0;
142142
int perm = -1; // index into Map::perms, or -1
143143
std::int64_t limit = -1; // map version 5 (57 section 5.3): elements with pi(i) >= limit are zero; -1 = none
144+
// Map versions 1 and 4: an axis given by explicit element offsets (`offsets`, one per index i, in place of
145+
// `stride` and `perm`): its contribution to the element index is offsets[i]. Empty: the stride form.
146+
std::vector<std::int64_t> offsets;
144147
};
145148

146149
// Map version 5: a second read of the same elements from another source and base (`also`).
@@ -245,6 +248,8 @@ struct Map {
245248
// load_map checks it is a known preset that allows the map's version.
246249
std::string preset_name;
247250
bool preset_named = false; // the map has the field "preset" (else the name came from the version, or the caller)
251+
// "natural", or "storage" (preset FG map v1: weight tensors are the stored spans, not yet natural axes).
252+
std::string channel_order;
248253
std::vector<Source> sources; // in file order
249254
std::vector<Array> arrays; // what tensors index
250255
int key_source = -1;

‎clean/converter/tests/cases.py‎

Lines changed: 25 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -638,45 +638,33 @@ def fg_blobs(rng):
638638
return out
639639

640640

641-
def fg_map(blobs, label="synthetic FG label", key_from="net1_01"):
642-
"""A version-1 map with "preset": "FG" for the blobs of fg_blobs: one block source per blob (map order = upload
643-
order) and one stage per layer. The tensor layout is a stand-in (weights as [out, K] rows, net1's bias as copy 0
644-
of each group of 8, net2's bias as [out]); the spec side's map defines the real names, axis order and stages."""
641+
def fg_map(fg_map_obj, layout_blobs):
642+
"""The delivered preset-FG map (data/fg.v1.map.json, parsed) with our own blobs in place of the real ones: every
643+
stage, tensor, base, axis and offset stays as delivered; each source keeps its name and text but gets the hashes of
644+
our blob. The map's sources are in upload order, so the n-th source is layout blob n (clean/fg/layout.py; the
645+
lengths are checked). Returns (map, {source name: bytes})."""
645646
L = _fg_layout()
646-
m = {
647-
"format": "wronggpu-converter-map",
648-
"version": 1,
649-
"model": "synthetic frame-generation case",
650-
"preset": "FG",
651-
"channel_order": "natural",
652-
"sources": {b.name: source_entry(b.name, "fake frame-generation weight blob", "f16le", blobs[b.name])
653-
for b in L.blobs()},
654-
"key": {"label": label, "from": key_from},
655-
"perms": {},
656-
"stages": [],
657-
}
658-
f16 = {"name": "f16"}
659-
for l in L.NET1:
660-
m["stages"].append({"stage": l.name, "digest": "", "tensors": [
661-
tensor("W", [l.out, l.k], "float32", l.name, f16, 0, [lin(l.k), lin(1)]),
662-
tensor("b", [l.out // L.NET1_BIAS_GROUP, L.NET1_BIAS_GROUP], "float32", l.name, f16, l.bias_offset,
663-
[lin(L.NET1_BIAS_GROUP * L.NET1_BIAS_COPIES), lin(1)]),
664-
]})
665-
for l in L.NET2:
666-
m["stages"].append({"stage": l.name, "digest": "", "tensors": [
667-
tensor("W", [l.out, 9 * l.cin], "float32", l.name + "_w", f16, 0, [lin(9 * l.cin), lin(1)]),
668-
tensor("b", [l.out], "float32", l.name + "_b", f16, 0, [lin(1)]),
669-
]})
670-
return m
671-
672-
673-
def fg_case(seed=41, drop=(), decoy="net2_07_w"):
674-
"""A synthetic frame-generation library: random bytes holding the 67 blobs of fg_blobs in a shuffled order, at
675-
16-aligned, even and odd offsets, with a decoy of one blob (its first 64 bytes, then other bytes) before it.
676-
drop: blob names left out of the file (the map still lists them). Returns the Case (placements: {blob: offset})."""
647+
m = copy.deepcopy(fg_map_obj)
648+
names = list(m["sources"])
649+
assert len(names) == len(L.blobs())
650+
blobs = {}
651+
for name, b in zip(names, L.blobs()):
652+
data = layout_blobs[b.name]
653+
assert m["sources"][name]["length"] == len(data), name
654+
m["sources"][name] = source_entry(name, m["sources"][name]["what"], "f16le", data)
655+
blobs[name] = data
656+
for st in m["stages"]:
657+
st["digest"] = ""
658+
return m, blobs
659+
660+
661+
def fg_case(fg_map_obj, seed=41, drop=(), decoy="B07w"):
662+
"""A synthetic frame-generation library for the delivered map's layout: random bytes holding our own 67 blobs
663+
(fg_blobs) in a shuffled order, at 16-aligned, even and odd offsets, with a decoy of one blob (its first 64 bytes,
664+
then other bytes) before it. drop: source names left out of the file (the map still lists them). Returns the Case
665+
(placements: {source: offset})."""
677666
rng = random.Random(seed)
678-
blobs = fg_blobs(rng)
679-
m = fg_map(blobs)
667+
m, blobs = fg_map(fg_map_obj, fg_blobs(rng))
680668
order = list(blobs)
681669
rng.shuffle(order)
682670
buf = bytearray(rng.randbytes(4096))

0 commit comments

Comments
 (0)