Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions clean/rr_f/runtime/src/rr_model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,15 +67,16 @@ static bool load_own(const std::vector<uint8_t>& d, Model* out, std::string* err
for (uint32_t i = 0; i < nb; ++i, o += 64) {
std::string name((const char*)d.data() + o, strnlen((const char*)d.data() + o, 48));
const uint64_t off = u64(o + 48), size = u64(o + 56);
if (off + size > isize) return *err = "model file damaged (blob " + name + ")", false;
if (off > isize || size > isize - off) return *err = "model file damaged (blob " + name + ")", false;
out->blobs[name] = {off, size};
}
for (uint32_t i = 0; i < ns; ++i, o += 52) {
std::string name((const char*)d.data() + o, strnlen((const char*)d.data() + o, 48));
out->scalars[name] = u32(o + 48);
}
o = (o + 255) / 256 * 256;
if (o + isize + 32 != d.size() || isize % 256) return *err = "model file damaged (image size)", false;
// (d.size() >= 96 here; written so that neither sum can wrap, and so that `o` is inside the file before it is used)
if (o > d.size() - 32 || isize != d.size() - 32 - o || isize % 256) return *err = "model file damaged (image size)", false;
out->image.assign(d.begin() + o, d.begin() + o + isize);
out->kind = "wgrrmod1";
return true;
Expand Down Expand Up @@ -192,14 +193,17 @@ bool read_wgcm(const std::vector<uint8_t>& d, std::vector<WgcmGroup>* groups, st
if (tn.dtype > 6) return *err = "model.wgcm damaged: unknown dtype in " + grp.name + "/" + tn.name, false;
if (!need(4ull * rank + 16)) return *err = "model.wgcm damaged: index entry of " + grp.name + "/" + tn.name, false;
uint64_t count = 1;
bool count_ok = true; // the element count never above the data's size (no wrap)
for (uint32_t r = 0; r < rank; ++r) {
tn.shape.push_back(u32((size_t)pos + 4 * r));
count *= tn.shape.back();
const uint32_t dim = u32((size_t)pos + 4 * r);
tn.shape.push_back(dim);
if (dim && count > body / dim) count_ok = false;
else count *= dim;
}
pos += 4ull * rank;
const uint64_t off = u64((size_t)pos), size = u64((size_t)pos + 8);
pos += 16;
if (off % 64 || off < end || off + size > body || size != count * kDtypeSize[tn.dtype])
if (!count_ok || off % 64 || off < end || off > body || size > body - off || size != count * kDtypeSize[tn.dtype])
return *err = "model.wgcm damaged: bad data range of " + grp.name + "/" + tn.name, false;
if (grp.find(tn.name)) return *err = "model.wgcm damaged: duplicate tensor " + grp.name + "/" + tn.name, false;
tn.data = d.data() + off;
Expand Down
216 changes: 216 additions & 0 deletions clean/rr_f/runtime/tests/test_model_bounds.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,216 @@
// SPDX-License-Identifier: MIT
// Bounds checks of the RR runtime's model loaders (src/rr_model.cpp), CPU only, no GPU and no model data: crafted files with
// a correct whole-file SHA-256 (the hash is unkeyed, so it only catches accidents) whose offsets, sizes and shapes wrap
// around 2^64. Driven by test_model_bounds.py, one case per run:
//
// test_model_bounds <case> <scratch folder>
//
// Prints "OK" or "FAIL <what>". A loader that reads through a wrapped range may also crash: that is a failure too.
#define WIN32_LEAN_AND_MEAN
#define NOMINMAX
#include <windows.h>

#include <stdint.h>
#include <stdio.h>
#include <string.h>

#include <string>
#include <vector>

#include "../src/rr_model.h"

using namespace wgrr_rt;

// load_model refers to the packer of rr_pack.cpp, which needs the generated tables; the loaders under test do not use it.
namespace wgrr_rt {
bool pack_wgcm(const std::vector<WgcmGroup>&, Model*, std::string* err) {
*err = "packing is not part of this test";
return false;
}
} // namespace wgrr_rt

typedef std::vector<uint8_t> Bytes;

static void put(Bytes& b, uint64_t v, int n) {
for (int i = 0; i < n; ++i) b.push_back((uint8_t)(v >> (8 * i)));
}
static void put_at(Bytes& b, size_t at, uint64_t v, int n) {
for (int i = 0; i < n; ++i) b[at + i] = (uint8_t)(v >> (8 * i));
}
static void pad_to(Bytes& b, size_t n) {
while (b.size() < n) b.push_back(0);
}
static void put_str(Bytes& b, const std::string& s) {
put(b, s.size(), 2);
b.insert(b.end(), s.begin(), s.end());
}
static Bytes digest(const Bytes& data) {
const std::string h = sha256_hex(data.data(), data.size());
Bytes out;
for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back((uint8_t)strtoul(h.substr(i, 2).c_str(), nullptr, 16));
return out;
}
static void seal(Bytes& b) { // the trailing whole-file hash
const Bytes h = digest(b);
b.insert(b.end(), h.begin(), h.end());
}

// ---- model.wgcm: header (88), index, data at 64-byte offsets, trailing hash ----------------------------------------
struct WgcmTensorSpec {
uint8_t dtype;
std::vector<uint32_t> dims;
uint64_t size;
bool at_data_start; // offset = the first 64-aligned offset past the index
uint64_t off; // else this offset
Bytes data; // placed at the data start, when it is used
};

// One group "g" with one tensor "t". `own_digest`: the group digest is the right one for `data` (else zeros).
static Bytes make_wgcm(const WgcmTensorSpec& t) {
Bytes key(32);
for (int i = 0; i < 32; ++i) key[i] = (uint8_t)(0x40 + i);
Bytes index;
put_str(index, "g");
Bytes keyed = key;
keyed.insert(keyed.end(), t.data.begin(), t.data.end());
const Bytes dg = digest(keyed);
index.insert(index.end(), dg.begin(), dg.end());
put(index, 1, 4); // one tensor
put_str(index, "t");
put(index, t.dtype, 1);
put(index, t.dims.size(), 1);
for (uint32_t d : t.dims) put(index, d, 4);
const uint64_t end = 88 + index.size() + 16; // the index ends after this tensor's offset and size
const uint64_t start = (end + 63) / 64 * 64;
put(index, t.at_data_start ? start : t.off, 8);
put(index, t.size, 8);
Bytes f;
const char magic[] = "WGCMODEL";
f.insert(f.end(), magic, magic + 8);
put(f, 2, 4); // container version 2
put(f, 1, 4); // groups
f.insert(f.end(), key.begin(), key.end());
pad_to(f, 80);
put(f, index.size(), 8);
f.insert(f.end(), index.begin(), index.end());
pad_to(f, (size_t)start);
f.insert(f.end(), t.data.begin(), t.data.end());
pad_to(f, (size_t)start + ((t.data.size() + 63) / 64 * 64));
seal(f);
return f;
}

// ---- WGRRMOD1: header (64), blob table (64 each), scalar table (52 each), image at a 256-byte offset, hash ----------
static Bytes own_header(uint32_t nb, uint32_t ns, uint64_t isize) {
Bytes f;
const char magic[] = "WGRRMOD1";
f.insert(f.end(), magic, magic + 8);
put(f, 1, 4);
put(f, 0, 4);
put(f, nb, 4);
put(f, ns, 4);
put(f, isize, 8);
pad_to(f, 64);
return f;
}
static void own_blob(Bytes& f, const char* name, uint64_t off, uint64_t size) {
const size_t at = f.size();
pad_to(f, at + 48);
memcpy(&f[at], name, strlen(name));
put(f, off, 8);
put(f, size, 8);
}
static void own_scalar(Bytes& f, const char* name, uint32_t v) {
const size_t at = f.size();
pad_to(f, at + 48);
memcpy(&f[at], name, strlen(name));
put(f, v, 4);
}

static bool load_file(const std::wstring& dir, const Bytes& file, std::string* err, Model* m) {
const std::wstring path = dir + L"\\model_bounds_case.bin";
FILE* fp = _wfopen(path.c_str(), L"wb");
if (!fp) {
*err = "cannot write the scratch file";
return false;
}
fwrite(file.data(), 1, file.size(), fp);
fclose(fp);
const bool ok = load_model(path, m, err);
DeleteFileW(path.c_str());
return ok;
}

static int fail(const std::string& what) {
printf("FAIL %s\n", what.c_str());
return 0;
}
static bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; }

int wmain(int argc, wchar_t** argv) {
if (argc < 3) {
printf("FAIL usage\n");
return 2;
}
const std::wstring which = argv[1], dir = argv[2];
std::string err;
std::vector<WgcmGroup> groups;

if (which == L"wgcm_valid") {
// control: a well-formed file is still accepted (f32, shape 2x3, 24 bytes of data)
WgcmTensorSpec t{0, {2, 3}, 24, true, 0, Bytes(24, 0x11)};
if (!read_wgcm(make_wgcm(t), &groups, &err)) return fail("a valid file was refused: " + err);
if (groups.size() != 1 || groups[0].tensors.size() != 1 || groups[0].tensors[0].size != 24) return fail("wrong content");
} else if (which == L"wgcm_offset_wraps") {
// off + size wraps around 2^64 to a value inside the file: the range must still be refused, before anything reads it
WgcmTensorSpec t{1, {64}, 64, false, 0xFFFFFFFFFFFFFFC0ull, Bytes()};
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range");
if (!has(err, "bad data range")) return fail("wrong message: " + err);
} else if (which == L"wgcm_size_past_end_after_wrap") {
// the same with a size that is itself the product of the shape (a 2^32-1 square) and an offset above 2^33
WgcmTensorSpec t{1, {0xFFFFFFFFu, 0xFFFFFFFFu}, 0xFFFFFFFE00000001ull, false, 1ull << 33, Bytes()};
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range");
if (!has(err, "bad data range")) return fail("wrong message: " + err);
} else if (which == L"wgcm_shape_wraps") {
// the shape's element count wraps around 2^64 (2^30 * 2^30 * 4 elements of 4 bytes = 0 bytes): refused
WgcmTensorSpec t{0, {1u << 30, 1u << 30, 4}, 0, true, 0, Bytes()};
if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a shape whose element count wraps");
if (!has(err, "bad data range")) return fail("wrong message: " + err);
} else if (which == L"own_valid") {
// control: a well-formed WGRRMOD1 with one blob and one scalar is still accepted
Bytes f = own_header(1, 1, 256);
own_blob(f, "a", 0, 256);
own_scalar(f, "s", 7);
pad_to(f, 256 + 256);
seal(f);
Model m;
if (!load_file(dir, f, &err, &m)) return fail("a valid file was refused: " + err);
if (m.image.size() != 256 || m.blob("a") != 0 || m.scalar("s") != 7) return fail("wrong content");
} else if (which == L"own_blob_wraps") {
// off + size wraps around 2^64 to a small value: the blob lies outside the image and must be refused
Bytes f = own_header(1, 0, 256);
own_blob(f, "a", 0xFFFFFFFFFFFFFFF0ull, 32);
pad_to(f, 256 + 256);
seal(f);
Model m;
if (load_file(dir, f, &err, &m)) return fail("accepted a blob whose range wraps");
if (!has(err, "blob a")) return fail("wrong message: " + err);
} else if (which == L"own_image_start_past_end") {
// the tables end (rounded up to 256) behind the end of the file, and isize wraps so that offset + isize + 32 equals the
// file size: the image's range would be reversed
Bytes f = own_header(1, 3, 0xFFFFFFFFFFFFFF00ull);
own_blob(f, "a", 0, 0);
own_scalar(f, "s0", 0);
own_scalar(f, "s1", 0);
own_scalar(f, "s2", 0);
f.resize(256); // the file is 256 bytes and its hash: 288 in all (the scalar table runs into the hash, as it may)
seal(f);
Model m;
if (load_file(dir, f, &err, &m)) return fail("accepted an image that starts behind the end of the file");
if (!has(err, "image size")) return fail("wrong message: " + err);
} else {
return fail("unknown case");
}
printf("OK\n");
return 0;
}
93 changes: 93 additions & 0 deletions clean/rr_f/runtime/tests/test_model_bounds.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# SPDX-License-Identifier: MIT
"""Bounds checks of the RR runtime's model loaders (src/rr_model.cpp: read_wgcm for model.wgcm, load_own for WGRRMOD1).
No GPU and no model data: clean/rr_f/runtime/tests/test_model_bounds.cpp is built with MSVC cl (on PATH, e.g. a developer
prompt or CI, or through vcvars64.bat; skipped without one) together with src/rr_model.cpp (a stub stands in for the
packer of src/rr_pack.cpp, which needs the generated tables), and loads
crafted files whose whole-file SHA-256 is correct but whose offsets, sizes and shapes wrap around 2^64. Each case runs in
its own process, since a loader that reads through a wrapped range may crash.

Run: python -m unittest clean.rr_f.runtime.tests.test_model_bounds
"""
import os
import shutil
import subprocess
import sys
import tempfile
import unittest

ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
RT = os.path.join(ROOT, "clean", "rr_f", "runtime")
SOURCES = [os.path.join(RT, "tests", "test_model_bounds.cpp"), os.path.join(RT, "src", "rr_model.cpp")]
_EXE = None


def _vcvars64():
"""vcvars64.bat of the newest Visual Studio (vswhere), or None."""
vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"),
"Microsoft Visual Studio", "Installer", "vswhere.exe")
if not os.path.exists(vswhere):
return None
r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True,
text=True)
vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat")
return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None


def bounds_exe():
global _EXE
if _EXE is None:
if os.name != "nt":
raise unittest.SkipTest("the loaders use Windows (bcrypt): Windows only")
vcvars = None if shutil.which("cl") else _vcvars64()
if not shutil.which("cl") and not vcvars:
raise unittest.SkipTest("no MSVC cl (not on PATH, no Visual Studio found)")
work = tempfile.mkdtemp(prefix="wg_mb_")
exe = os.path.join(work, "test_model_bounds.exe")
cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s /Fo"%s\\\\" /Fe"%s"'
% (" ".join('"%s"' % s for s in SOURCES), work, exe))
bat = os.path.join(work, "build.bat")
with open(bat, "w") as f:
f.write("@echo off\r\n")
if vcvars:
f.write('call "%s" >nul || exit /b 1\r\n' % vcvars)
f.write(cl + " || exit /b 1\r\n")
r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True)
if r.returncode != 0:
raise RuntimeError("build of test_model_bounds failed:\n" + r.stdout + r.stderr)
_EXE = exe
return _EXE


class ModelBounds(unittest.TestCase):
def run_case(self, name):
scratch = tempfile.mkdtemp(prefix="wg_mb_")
r = subprocess.run([bounds_exe(), name, scratch], capture_output=True, text=True, timeout=120)
self.assertEqual(r.returncode, 0, "case %s ended abnormally (%s): %s" % (name, r.returncode, r.stdout + r.stderr))
self.assertEqual(r.stdout.strip(), "OK", "case %s: %s" % (name, r.stdout.strip()))

# model.wgcm (read_wgcm)
def test_wgcm_valid_file_still_loads(self):
self.run_case("wgcm_valid")

def test_wgcm_offset_plus_size_wraps(self):
self.run_case("wgcm_offset_wraps")

def test_wgcm_size_from_a_huge_shape_and_offset_above_2_33(self):
self.run_case("wgcm_size_past_end_after_wrap")

def test_wgcm_shape_element_count_wraps(self):
self.run_case("wgcm_shape_wraps")

# WGRRMOD1 (load_own, through load_model)
def test_own_valid_file_still_loads(self):
self.run_case("own_valid")

def test_own_blob_range_wraps(self):
self.run_case("own_blob_wraps")

def test_own_image_start_behind_the_end_of_the_file(self):
self.run_case("own_image_start_past_end")


if __name__ == "__main__":
unittest.main()
20 changes: 14 additions & 6 deletions clean/runtime/model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -454,7 +454,8 @@ struct Reader {
size_t pos, end;
bool ok = true;
bool need(size_t n) {
if (!ok || pos + n > end || pos + n < pos) ok = false;
// Written so that no sum can wrap, and so that no range reaches past the buffer whatever `end` was set to.
if (!ok || end > d.size() || pos > end || n > end - pos) ok = false;
return ok;
}
uint64_t le(int n) {
Expand Down Expand Up @@ -505,7 +506,8 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
r.bytes(key.data(), 32);
r.bytes(map_hash.data(), 32);
const uint64_t isize = r.le(8);
if (!r.ok || isize > d.size()) return *err = bad + ": bad header", false;
// The index lies between the 88-byte header and the trailing hash (d.size() >= 88 + 32 is known here).
if (!r.ok || isize > d.size() - 32 - 88) return *err = bad + ": bad header", false;

// The map shipped with the runtime: its digests are the reference.
std::vector<uint8_t> mbytes;
Expand Down Expand Up @@ -559,12 +561,18 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std::
const uint64_t dtype = ix.le(1), rank = ix.le(1);
Tensor ten;
size_t n = 1;
for (uint64_t a = 0; a < rank && ix.ok; ++a) {
ten.shape.push_back((int)ix.le(4));
n *= (size_t)ten.shape.back();
bool shape_ok = true; // every dimension above 0, and the element count never above the file's size (no wrap)
for (uint64_t a = 0; a < rank && ix.ok && shape_ok; ++a) {
const int dim = (int)ix.le(4);
if (dim <= 0 || n > d.size() / (size_t)dim) {
shape_ok = false;
break;
}
ten.shape.push_back(dim);
n *= (size_t)dim;
}
const uint64_t off = ix.le(8), size = ix.le(8);
if (!ix.ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 ||
if (!ix.ok || !shape_ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 ||
size > d.size() - 32 - off)
return *err = bad + ": bad tensor entry " + stage + "/" + name, false;
h.update(&d[off], (size_t)size);
Expand Down
Loading
Loading