Skip to content

Commit 3d66154

Browse files
Model loaders: size, offset and shape checks cannot overflow or reach past the file (#84)
2 parents b3a07df + a4c398c commit 3d66154

5 files changed

Lines changed: 401 additions & 11 deletions

File tree

‎clean/rr_f/runtime/src/rr_model.cpp‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,15 +67,16 @@ static bool load_own(const std::vector<uint8_t>& d, Model* out, std::string* err
6767
for (uint32_t i = 0; i < nb; ++i, o += 64) {
6868
std::string name((const char*)d.data() + o, strnlen((const char*)d.data() + o, 48));
6969
const uint64_t off = u64(o + 48), size = u64(o + 56);
70-
if (off + size > isize) return *err = "model file damaged (blob " + name + ")", false;
70+
if (off > isize || size > isize - off) return *err = "model file damaged (blob " + name + ")", false;
7171
out->blobs[name] = {off, size};
7272
}
7373
for (uint32_t i = 0; i < ns; ++i, o += 52) {
7474
std::string name((const char*)d.data() + o, strnlen((const char*)d.data() + o, 48));
7575
out->scalars[name] = u32(o + 48);
7676
}
7777
o = (o + 255) / 256 * 256;
78-
if (o + isize + 32 != d.size() || isize % 256) return *err = "model file damaged (image size)", false;
78+
// (d.size() >= 96 here; written so that neither sum can wrap, and so that `o` is inside the file before it is used)
79+
if (o > d.size() - 32 || isize != d.size() - 32 - o || isize % 256) return *err = "model file damaged (image size)", false;
7980
out->image.assign(d.begin() + o, d.begin() + o + isize);
8081
out->kind = "wgrrmod1";
8182
return true;
@@ -192,14 +193,17 @@ bool read_wgcm(const std::vector<uint8_t>& d, std::vector<WgcmGroup>* groups, st
192193
if (tn.dtype > 6) return *err = "model.wgcm damaged: unknown dtype in " + grp.name + "/" + tn.name, false;
193194
if (!need(4ull * rank + 16)) return *err = "model.wgcm damaged: index entry of " + grp.name + "/" + tn.name, false;
194195
uint64_t count = 1;
196+
bool count_ok = true; // the element count never above the data's size (no wrap)
195197
for (uint32_t r = 0; r < rank; ++r) {
196-
tn.shape.push_back(u32((size_t)pos + 4 * r));
197-
count *= tn.shape.back();
198+
const uint32_t dim = u32((size_t)pos + 4 * r);
199+
tn.shape.push_back(dim);
200+
if (dim && count > body / dim) count_ok = false;
201+
else count *= dim;
198202
}
199203
pos += 4ull * rank;
200204
const uint64_t off = u64((size_t)pos), size = u64((size_t)pos + 8);
201205
pos += 16;
202-
if (off % 64 || off < end || off + size > body || size != count * kDtypeSize[tn.dtype])
206+
if (!count_ok || off % 64 || off < end || off > body || size > body - off || size != count * kDtypeSize[tn.dtype])
203207
return *err = "model.wgcm damaged: bad data range of " + grp.name + "/" + tn.name, false;
204208
if (grp.find(tn.name)) return *err = "model.wgcm damaged: duplicate tensor " + grp.name + "/" + tn.name, false;
205209
tn.data = d.data() + off;
Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
// SPDX-License-Identifier: MIT
2+
// Bounds checks of the RR runtime's model loaders (src/rr_model.cpp), CPU only, no GPU and no model data: crafted files with
3+
// a correct whole-file SHA-256 (the hash is unkeyed, so it only catches accidents) whose offsets, sizes and shapes wrap
4+
// around 2^64. Driven by test_model_bounds.py, one case per run:
5+
//
6+
// test_model_bounds <case> <scratch folder>
7+
//
8+
// Prints "OK" or "FAIL <what>". A loader that reads through a wrapped range may also crash: that is a failure too.
9+
#define WIN32_LEAN_AND_MEAN
10+
#define NOMINMAX
11+
#include <windows.h>
12+
13+
#include <stdint.h>
14+
#include <stdio.h>
15+
#include <string.h>
16+
17+
#include <string>
18+
#include <vector>
19+
20+
#include "../src/rr_model.h"
21+
22+
using namespace wgrr_rt;
23+
24+
// load_model refers to the packer of rr_pack.cpp, which needs the generated tables; the loaders under test do not use it.
25+
namespace wgrr_rt {
26+
bool pack_wgcm(const std::vector<WgcmGroup>&, Model*, std::string* err) {
27+
*err = "packing is not part of this test";
28+
return false;
29+
}
30+
} // namespace wgrr_rt
31+
32+
typedef std::vector<uint8_t> Bytes;
33+
34+
static void put(Bytes& b, uint64_t v, int n) {
35+
for (int i = 0; i < n; ++i) b.push_back((uint8_t)(v >> (8 * i)));
36+
}
37+
static void put_at(Bytes& b, size_t at, uint64_t v, int n) {
38+
for (int i = 0; i < n; ++i) b[at + i] = (uint8_t)(v >> (8 * i));
39+
}
40+
static void pad_to(Bytes& b, size_t n) {
41+
while (b.size() < n) b.push_back(0);
42+
}
43+
static void put_str(Bytes& b, const std::string& s) {
44+
put(b, s.size(), 2);
45+
b.insert(b.end(), s.begin(), s.end());
46+
}
47+
static Bytes digest(const Bytes& data) {
48+
const std::string h = sha256_hex(data.data(), data.size());
49+
Bytes out;
50+
for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back((uint8_t)strtoul(h.substr(i, 2).c_str(), nullptr, 16));
51+
return out;
52+
}
53+
static void seal(Bytes& b) { // the trailing whole-file hash
54+
const Bytes h = digest(b);
55+
b.insert(b.end(), h.begin(), h.end());
56+
}
57+
58+
// ---- model.wgcm: header (88), index, data at 64-byte offsets, trailing hash ----------------------------------------
59+
struct WgcmTensorSpec {
60+
uint8_t dtype;
61+
std::vector<uint32_t> dims;
62+
uint64_t size;
63+
bool at_data_start; // offset = the first 64-aligned offset past the index
64+
uint64_t off; // else this offset
65+
Bytes data; // placed at the data start, when it is used
66+
};
67+
68+
// One group "g" with one tensor "t". `own_digest`: the group digest is the right one for `data` (else zeros).
69+
static Bytes make_wgcm(const WgcmTensorSpec& t) {
70+
Bytes key(32);
71+
for (int i = 0; i < 32; ++i) key[i] = (uint8_t)(0x40 + i);
72+
Bytes index;
73+
put_str(index, "g");
74+
Bytes keyed = key;
75+
keyed.insert(keyed.end(), t.data.begin(), t.data.end());
76+
const Bytes dg = digest(keyed);
77+
index.insert(index.end(), dg.begin(), dg.end());
78+
put(index, 1, 4); // one tensor
79+
put_str(index, "t");
80+
put(index, t.dtype, 1);
81+
put(index, t.dims.size(), 1);
82+
for (uint32_t d : t.dims) put(index, d, 4);
83+
const uint64_t end = 88 + index.size() + 16; // the index ends after this tensor's offset and size
84+
const uint64_t start = (end + 63) / 64 * 64;
85+
put(index, t.at_data_start ? start : t.off, 8);
86+
put(index, t.size, 8);
87+
Bytes f;
88+
const char magic[] = "WGCMODEL";
89+
f.insert(f.end(), magic, magic + 8);
90+
put(f, 2, 4); // container version 2
91+
put(f, 1, 4); // groups
92+
f.insert(f.end(), key.begin(), key.end());
93+
pad_to(f, 80);
94+
put(f, index.size(), 8);
95+
f.insert(f.end(), index.begin(), index.end());
96+
pad_to(f, (size_t)start);
97+
f.insert(f.end(), t.data.begin(), t.data.end());
98+
pad_to(f, (size_t)start + ((t.data.size() + 63) / 64 * 64));
99+
seal(f);
100+
return f;
101+
}
102+
103+
// ---- WGRRMOD1: header (64), blob table (64 each), scalar table (52 each), image at a 256-byte offset, hash ----------
104+
static Bytes own_header(uint32_t nb, uint32_t ns, uint64_t isize) {
105+
Bytes f;
106+
const char magic[] = "WGRRMOD1";
107+
f.insert(f.end(), magic, magic + 8);
108+
put(f, 1, 4);
109+
put(f, 0, 4);
110+
put(f, nb, 4);
111+
put(f, ns, 4);
112+
put(f, isize, 8);
113+
pad_to(f, 64);
114+
return f;
115+
}
116+
static void own_blob(Bytes& f, const char* name, uint64_t off, uint64_t size) {
117+
const size_t at = f.size();
118+
pad_to(f, at + 48);
119+
memcpy(&f[at], name, strlen(name));
120+
put(f, off, 8);
121+
put(f, size, 8);
122+
}
123+
static void own_scalar(Bytes& f, const char* name, uint32_t v) {
124+
const size_t at = f.size();
125+
pad_to(f, at + 48);
126+
memcpy(&f[at], name, strlen(name));
127+
put(f, v, 4);
128+
}
129+
130+
static bool load_file(const std::wstring& dir, const Bytes& file, std::string* err, Model* m) {
131+
const std::wstring path = dir + L"\\model_bounds_case.bin";
132+
FILE* fp = _wfopen(path.c_str(), L"wb");
133+
if (!fp) {
134+
*err = "cannot write the scratch file";
135+
return false;
136+
}
137+
fwrite(file.data(), 1, file.size(), fp);
138+
fclose(fp);
139+
const bool ok = load_model(path, m, err);
140+
DeleteFileW(path.c_str());
141+
return ok;
142+
}
143+
144+
static int fail(const std::string& what) {
145+
printf("FAIL %s\n", what.c_str());
146+
return 0;
147+
}
148+
static bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; }
149+
150+
int wmain(int argc, wchar_t** argv) {
151+
if (argc < 3) {
152+
printf("FAIL usage\n");
153+
return 2;
154+
}
155+
const std::wstring which = argv[1], dir = argv[2];
156+
std::string err;
157+
std::vector<WgcmGroup> groups;
158+
159+
if (which == L"wgcm_valid") {
160+
// control: a well-formed file is still accepted (f32, shape 2x3, 24 bytes of data)
161+
WgcmTensorSpec t{0, {2, 3}, 24, true, 0, Bytes(24, 0x11)};
162+
if (!read_wgcm(make_wgcm(t), &groups, &err)) return fail("a valid file was refused: " + err);
163+
if (groups.size() != 1 || groups[0].tensors.size() != 1 || groups[0].tensors[0].size != 24) return fail("wrong content");
164+
} else if (which == L"wgcm_offset_wraps") {
165+
// off + size wraps around 2^64 to a value inside the file: the range must still be refused, before anything reads it
166+
WgcmTensorSpec t{1, {64}, 64, false, 0xFFFFFFFFFFFFFFC0ull, Bytes()};
167+
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range");
168+
if (!has(err, "bad data range")) return fail("wrong message: " + err);
169+
} else if (which == L"wgcm_size_past_end_after_wrap") {
170+
// the same with a size that is itself the product of the shape (a 2^32-1 square) and an offset above 2^33
171+
WgcmTensorSpec t{1, {0xFFFFFFFFu, 0xFFFFFFFFu}, 0xFFFFFFFE00000001ull, false, 1ull << 33, Bytes()};
172+
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range");
173+
if (!has(err, "bad data range")) return fail("wrong message: " + err);
174+
} else if (which == L"wgcm_shape_wraps") {
175+
// the shape's element count wraps around 2^64 (2^30 * 2^30 * 4 elements of 4 bytes = 0 bytes): refused
176+
WgcmTensorSpec t{0, {1u << 30, 1u << 30, 4}, 0, true, 0, Bytes()};
177+
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a shape whose element count wraps");
178+
if (!has(err, "bad data range")) return fail("wrong message: " + err);
179+
} else if (which == L"own_valid") {
180+
// control: a well-formed WGRRMOD1 with one blob and one scalar is still accepted
181+
Bytes f = own_header(1, 1, 256);
182+
own_blob(f, "a", 0, 256);
183+
own_scalar(f, "s", 7);
184+
pad_to(f, 256 + 256);
185+
seal(f);
186+
Model m;
187+
if (!load_file(dir, f, &err, &m)) return fail("a valid file was refused: " + err);
188+
if (m.image.size() != 256 || m.blob("a") != 0 || m.scalar("s") != 7) return fail("wrong content");
189+
} else if (which == L"own_blob_wraps") {
190+
// off + size wraps around 2^64 to a small value: the blob lies outside the image and must be refused
191+
Bytes f = own_header(1, 0, 256);
192+
own_blob(f, "a", 0xFFFFFFFFFFFFFFF0ull, 32);
193+
pad_to(f, 256 + 256);
194+
seal(f);
195+
Model m;
196+
if (load_file(dir, f, &err, &m)) return fail("accepted a blob whose range wraps");
197+
if (!has(err, "blob a")) return fail("wrong message: " + err);
198+
} else if (which == L"own_image_start_past_end") {
199+
// the tables end (rounded up to 256) behind the end of the file, and isize wraps so that offset + isize + 32 equals the
200+
// file size: the image's range would be reversed
201+
Bytes f = own_header(1, 3, 0xFFFFFFFFFFFFFF00ull);
202+
own_blob(f, "a", 0, 0);
203+
own_scalar(f, "s0", 0);
204+
own_scalar(f, "s1", 0);
205+
own_scalar(f, "s2", 0);
206+
f.resize(256); // the file is 256 bytes and its hash: 288 in all (the scalar table runs into the hash, as it may)
207+
seal(f);
208+
Model m;
209+
if (load_file(dir, f, &err, &m)) return fail("accepted an image that starts behind the end of the file");
210+
if (!has(err, "image size")) return fail("wrong message: " + err);
211+
} else {
212+
return fail("unknown case");
213+
}
214+
printf("OK\n");
215+
return 0;
216+
}
Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
# SPDX-License-Identifier: MIT
2+
"""Bounds checks of the RR runtime's model loaders (src/rr_model.cpp: read_wgcm for model.wgcm, load_own for WGRRMOD1).
3+
No GPU and no model data: clean/rr_f/runtime/tests/test_model_bounds.cpp is built with MSVC cl (on PATH, e.g. a developer
4+
prompt or CI, or through vcvars64.bat; skipped without one) together with src/rr_model.cpp (a stub stands in for the
5+
packer of src/rr_pack.cpp, which needs the generated tables), and loads
6+
crafted files whose whole-file SHA-256 is correct but whose offsets, sizes and shapes wrap around 2^64. Each case runs in
7+
its own process, since a loader that reads through a wrapped range may crash.
8+
9+
Run: python -m unittest clean.rr_f.runtime.tests.test_model_bounds
10+
"""
11+
import os
12+
import shutil
13+
import subprocess
14+
import sys
15+
import tempfile
16+
import unittest
17+
18+
ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
19+
RT = os.path.join(ROOT, "clean", "rr_f", "runtime")
20+
SOURCES = [os.path.join(RT, "tests", "test_model_bounds.cpp"), os.path.join(RT, "src", "rr_model.cpp")]
21+
_EXE = None
22+
23+
24+
def _vcvars64():
25+
"""vcvars64.bat of the newest Visual Studio (vswhere), or None."""
26+
vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
27+
"Microsoft Visual Studio", "Installer", "vswhere.exe")
28+
if not os.path.exists(vswhere):
29+
return None
30+
r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True,
31+
text=True)
32+
vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat")
33+
return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None
34+
35+
36+
def bounds_exe():
37+
global _EXE
38+
if _EXE is None:
39+
if os.name != "nt":
40+
raise unittest.SkipTest("the loaders use Windows (bcrypt): Windows only")
41+
vcvars = None if shutil.which("cl") else _vcvars64()
42+
if not shutil.which("cl") and not vcvars:
43+
raise unittest.SkipTest("no MSVC cl (not on PATH, no Visual Studio found)")
44+
work = tempfile.mkdtemp(prefix="wg_mb_")
45+
exe = os.path.join(work, "test_model_bounds.exe")
46+
cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s /Fo"%s\\\\" /Fe"%s"'
47+
% (" ".join('"%s"' % s for s in SOURCES), work, exe))
48+
bat = os.path.join(work, "build.bat")
49+
with open(bat, "w") as f:
50+
f.write("@echo off\r\n")
51+
if vcvars:
52+
f.write('call "%s" >nul || exit /b 1\r\n' % vcvars)
53+
f.write(cl + " || exit /b 1\r\n")
54+
r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True)
55+
if r.returncode != 0:
56+
raise RuntimeError("build of test_model_bounds failed:\n" + r.stdout + r.stderr)
57+
_EXE = exe
58+
return _EXE
59+
60+
61+
class ModelBounds(unittest.TestCase):
62+
def run_case(self, name):
63+
scratch = tempfile.mkdtemp(prefix="wg_mb_")
64+
r = subprocess.run([bounds_exe(), name, scratch], capture_output=True, text=True, timeout=120)
65+
self.assertEqual(r.returncode, 0, "case %s ended abnormally (%s): %s" % (name, r.returncode, r.stdout + r.stderr))
66+
self.assertEqual(r.stdout.strip(), "OK", "case %s: %s" % (name, r.stdout.strip()))
67+
68+
# model.wgcm (read_wgcm)
69+
def test_wgcm_valid_file_still_loads(self):
70+
self.run_case("wgcm_valid")
71+
72+
def test_wgcm_offset_plus_size_wraps(self):
73+
self.run_case("wgcm_offset_wraps")
74+
75+
def test_wgcm_size_from_a_huge_shape_and_offset_above_2_33(self):
76+
self.run_case("wgcm_size_past_end_after_wrap")
77+
78+
def test_wgcm_shape_element_count_wraps(self):
79+
self.run_case("wgcm_shape_wraps")
80+
81+
# WGRRMOD1 (load_own, through load_model)
82+
def test_own_valid_file_still_loads(self):
83+
self.run_case("own_valid")
84+
85+
def test_own_blob_range_wraps(self):
86+
self.run_case("own_blob_wraps")
87+
88+
def test_own_image_start_behind_the_end_of_the_file(self):
89+
self.run_case("own_image_start_past_end")
90+
91+
92+
if __name__ == "__main__":
93+
unittest.main()

‎clean/runtime/model.cpp‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,8 @@ struct Reader {
454454
size_t pos, end;
455455
bool ok = true;
456456
bool need(size_t n) {
457-
if (!ok || pos + n > end || pos + n < pos) ok = false;
457+
// Written so that no sum can wrap, and so that no range reaches past the buffer whatever `end` was set to.
458+
if (!ok || end > d.size() || pos > end || n > end - pos) ok = false;
458459
return ok;
459460
}
460461
uint64_t le(int n) {
@@ -505,7 +506,8 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
505506
r.bytes(key.data(), 32);
506507
r.bytes(map_hash.data(), 32);
507508
const uint64_t isize = r.le(8);
508-
if (!r.ok || isize > d.size()) return *err = bad + ": bad header", false;
509+
// The index lies between the 88-byte header and the trailing hash (d.size() >= 88 + 32 is known here).
510+
if (!r.ok || isize > d.size() - 32 - 88) return *err = bad + ": bad header", false;
509511

510512
// The map shipped with the runtime: its digests are the reference.
511513
std::vector<uint8_t> mbytes;
@@ -559,12 +561,18 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
559561
const uint64_t dtype = ix.le(1), rank = ix.le(1);
560562
Tensor ten;
561563
size_t n = 1;
562-
for (uint64_t a = 0; a < rank && ix.ok; ++a) {
563-
ten.shape.push_back((int)ix.le(4));
564-
n *= (size_t)ten.shape.back();
564+
bool shape_ok = true; // every dimension above 0, and the element count never above the file's size (no wrap)
565+
for (uint64_t a = 0; a < rank && ix.ok && shape_ok; ++a) {
566+
const int dim = (int)ix.le(4);
567+
if (dim <= 0 || n > d.size() / (size_t)dim) {
568+
shape_ok = false;
569+
break;
570+
}
571+
ten.shape.push_back(dim);
572+
n *= (size_t)dim;
565573
}
566574
const uint64_t off = ix.le(8), size = ix.le(8);
567-
if (!ix.ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 ||
575+
if (!ix.ok || !shape_ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 ||
568576
size > d.size() - 32 - off)
569577
return *err = bad + ": bad tensor entry " + stage + "/" + name, false;
570578
h.update(&d[off], (size_t)size);

0 commit comments

Comments
 (0)