diff --git a/clean/rr_f/runtime/src/rr_model.cpp b/clean/rr_f/runtime/src/rr_model.cpp index 1c6babd0..da56cdd7 100644 --- a/clean/rr_f/runtime/src/rr_model.cpp +++ b/clean/rr_f/runtime/src/rr_model.cpp @@ -67,7 +67,7 @@ static bool load_own(const std::vector& d, Model* out, std::string* err for (uint32_t i = 0; i < nb; ++i, o += 64) { std::string name((const char*)d.data() + o, strnlen((const char*)d.data() + o, 48)); const uint64_t off = u64(o + 48), size = u64(o + 56); - if (off + size > isize) return *err = "model file damaged (blob " + name + ")", false; + if (off > isize || size > isize - off) return *err = "model file damaged (blob " + name + ")", false; out->blobs[name] = {off, size}; } for (uint32_t i = 0; i < ns; ++i, o += 52) { @@ -75,7 +75,8 @@ static bool load_own(const std::vector& d, Model* out, std::string* err out->scalars[name] = u32(o + 48); } o = (o + 255) / 256 * 256; - if (o + isize + 32 != d.size() || isize % 256) return *err = "model file damaged (image size)", false; + // (d.size() >= 96 here; written so that neither sum can wrap, and so that `o` is inside the file before it is used) + if (o > d.size() - 32 || isize != d.size() - 32 - o || isize % 256) return *err = "model file damaged (image size)", false; out->image.assign(d.begin() + o, d.begin() + o + isize); out->kind = "wgrrmod1"; return true; @@ -192,14 +193,17 @@ bool read_wgcm(const std::vector& d, std::vector* groups, st if (tn.dtype > 6) return *err = "model.wgcm damaged: unknown dtype in " + grp.name + "/" + tn.name, false; if (!need(4ull * rank + 16)) return *err = "model.wgcm damaged: index entry of " + grp.name + "/" + tn.name, false; uint64_t count = 1; + bool count_ok = true; // the element count never above the data's size (no wrap) for (uint32_t r = 0; r < rank; ++r) { - tn.shape.push_back(u32((size_t)pos + 4 * r)); - count *= tn.shape.back(); + const uint32_t dim = u32((size_t)pos + 4 * r); + tn.shape.push_back(dim); + if (dim && count > body / dim) count_ok = false; + else count *= dim; } pos += 4ull * rank; const uint64_t off = u64((size_t)pos), size = u64((size_t)pos + 8); pos += 16; - if (off % 64 || off < end || off + size > body || size != count * kDtypeSize[tn.dtype]) + if (!count_ok || off % 64 || off < end || off > body || size > body - off || size != count * kDtypeSize[tn.dtype]) return *err = "model.wgcm damaged: bad data range of " + grp.name + "/" + tn.name, false; if (grp.find(tn.name)) return *err = "model.wgcm damaged: duplicate tensor " + grp.name + "/" + tn.name, false; tn.data = d.data() + off; diff --git a/clean/rr_f/runtime/tests/test_model_bounds.cpp b/clean/rr_f/runtime/tests/test_model_bounds.cpp new file mode 100644 index 00000000..7ed7a662 --- /dev/null +++ b/clean/rr_f/runtime/tests/test_model_bounds.cpp @@ -0,0 +1,216 @@ +// SPDX-License-Identifier: MIT +// Bounds checks of the RR runtime's model loaders (src/rr_model.cpp), CPU only, no GPU and no model data: crafted files with +// a correct whole-file SHA-256 (the hash is unkeyed, so it only catches accidents) whose offsets, sizes and shapes wrap +// around 2^64. Driven by test_model_bounds.py, one case per run: +// +// test_model_bounds +// +// Prints "OK" or "FAIL ". A loader that reads through a wrapped range may also crash: that is a failure too. +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include + +#include +#include +#include + +#include +#include + +#include "../src/rr_model.h" + +using namespace wgrr_rt; + +// load_model refers to the packer of rr_pack.cpp, which needs the generated tables; the loaders under test do not use it. +namespace wgrr_rt { +bool pack_wgcm(const std::vector&, Model*, std::string* err) { + *err = "packing is not part of this test"; + return false; +} +} // namespace wgrr_rt + +typedef std::vector Bytes; + +static void put(Bytes& b, uint64_t v, int n) { + for (int i = 0; i < n; ++i) b.push_back((uint8_t)(v >> (8 * i))); +} +static void put_at(Bytes& b, size_t at, uint64_t v, int n) { + for (int i = 0; i < n; ++i) b[at + i] = (uint8_t)(v >> (8 * i)); +} +static void pad_to(Bytes& b, size_t n) { + while (b.size() < n) b.push_back(0); +} +static void put_str(Bytes& b, const std::string& s) { + put(b, s.size(), 2); + b.insert(b.end(), s.begin(), s.end()); +} +static Bytes digest(const Bytes& data) { + const std::string h = sha256_hex(data.data(), data.size()); + Bytes out; + for (size_t i = 0; i + 1 < h.size(); i += 2) out.push_back((uint8_t)strtoul(h.substr(i, 2).c_str(), nullptr, 16)); + return out; +} +static void seal(Bytes& b) { // the trailing whole-file hash + const Bytes h = digest(b); + b.insert(b.end(), h.begin(), h.end()); +} + +// ---- model.wgcm: header (88), index, data at 64-byte offsets, trailing hash ---------------------------------------- +struct WgcmTensorSpec { + uint8_t dtype; + std::vector dims; + uint64_t size; + bool at_data_start; // offset = the first 64-aligned offset past the index + uint64_t off; // else this offset + Bytes data; // placed at the data start, when it is used +}; + +// One group "g" with one tensor "t". `own_digest`: the group digest is the right one for `data` (else zeros). +static Bytes make_wgcm(const WgcmTensorSpec& t) { + Bytes key(32); + for (int i = 0; i < 32; ++i) key[i] = (uint8_t)(0x40 + i); + Bytes index; + put_str(index, "g"); + Bytes keyed = key; + keyed.insert(keyed.end(), t.data.begin(), t.data.end()); + const Bytes dg = digest(keyed); + index.insert(index.end(), dg.begin(), dg.end()); + put(index, 1, 4); // one tensor + put_str(index, "t"); + put(index, t.dtype, 1); + put(index, t.dims.size(), 1); + for (uint32_t d : t.dims) put(index, d, 4); + const uint64_t end = 88 + index.size() + 16; // the index ends after this tensor's offset and size + const uint64_t start = (end + 63) / 64 * 64; + put(index, t.at_data_start ? start : t.off, 8); + put(index, t.size, 8); + Bytes f; + const char magic[] = "WGCMODEL"; + f.insert(f.end(), magic, magic + 8); + put(f, 2, 4); // container version 2 + put(f, 1, 4); // groups + f.insert(f.end(), key.begin(), key.end()); + pad_to(f, 80); + put(f, index.size(), 8); + f.insert(f.end(), index.begin(), index.end()); + pad_to(f, (size_t)start); + f.insert(f.end(), t.data.begin(), t.data.end()); + pad_to(f, (size_t)start + ((t.data.size() + 63) / 64 * 64)); + seal(f); + return f; +} + +// ---- WGRRMOD1: header (64), blob table (64 each), scalar table (52 each), image at a 256-byte offset, hash ---------- +static Bytes own_header(uint32_t nb, uint32_t ns, uint64_t isize) { + Bytes f; + const char magic[] = "WGRRMOD1"; + f.insert(f.end(), magic, magic + 8); + put(f, 1, 4); + put(f, 0, 4); + put(f, nb, 4); + put(f, ns, 4); + put(f, isize, 8); + pad_to(f, 64); + return f; +} +static void own_blob(Bytes& f, const char* name, uint64_t off, uint64_t size) { + const size_t at = f.size(); + pad_to(f, at + 48); + memcpy(&f[at], name, strlen(name)); + put(f, off, 8); + put(f, size, 8); +} +static void own_scalar(Bytes& f, const char* name, uint32_t v) { + const size_t at = f.size(); + pad_to(f, at + 48); + memcpy(&f[at], name, strlen(name)); + put(f, v, 4); +} + +static bool load_file(const std::wstring& dir, const Bytes& file, std::string* err, Model* m) { + const std::wstring path = dir + L"\\model_bounds_case.bin"; + FILE* fp = _wfopen(path.c_str(), L"wb"); + if (!fp) { + *err = "cannot write the scratch file"; + return false; + } + fwrite(file.data(), 1, file.size(), fp); + fclose(fp); + const bool ok = load_model(path, m, err); + DeleteFileW(path.c_str()); + return ok; +} + +static int fail(const std::string& what) { + printf("FAIL %s\n", what.c_str()); + return 0; +} +static bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; } + +int wmain(int argc, wchar_t** argv) { + if (argc < 3) { + printf("FAIL usage\n"); + return 2; + } + const std::wstring which = argv[1], dir = argv[2]; + std::string err; + std::vector groups; + + if (which == L"wgcm_valid") { + // control: a well-formed file is still accepted (f32, shape 2x3, 24 bytes of data) + WgcmTensorSpec t{0, {2, 3}, 24, true, 0, Bytes(24, 0x11)}; + if (!read_wgcm(make_wgcm(t), &groups, &err)) return fail("a valid file was refused: " + err); + if (groups.size() != 1 || groups[0].tensors.size() != 1 || groups[0].tensors[0].size != 24) return fail("wrong content"); + } else if (which == L"wgcm_offset_wraps") { + // off + size wraps around 2^64 to a value inside the file: the range must still be refused, before anything reads it + WgcmTensorSpec t{1, {64}, 64, false, 0xFFFFFFFFFFFFFFC0ull, Bytes()}; + if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range"); + if (!has(err, "bad data range")) return fail("wrong message: " + err); + } else if (which == L"wgcm_size_past_end_after_wrap") { + // the same with a size that is itself the product of the shape (a 2^32-1 square) and an offset above 2^33 + WgcmTensorSpec t{1, {0xFFFFFFFFu, 0xFFFFFFFFu}, 0xFFFFFFFE00000001ull, false, 1ull << 33, Bytes()}; + if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a wrapped data range"); + if (!has(err, "bad data range")) return fail("wrong message: " + err); + } else if (which == L"wgcm_shape_wraps") { + // the shape's element count wraps around 2^64 (2^30 * 2^30 * 4 elements of 4 bytes = 0 bytes): refused + WgcmTensorSpec t{0, {1u << 30, 1u << 30, 4}, 0, true, 0, Bytes()}; + if (read_wgcm(make_wgcm(t), &groups, &err)) return fail("accepted a shape whose element count wraps"); + if (!has(err, "bad data range")) return fail("wrong message: " + err); + } else if (which == L"own_valid") { + // control: a well-formed WGRRMOD1 with one blob and one scalar is still accepted + Bytes f = own_header(1, 1, 256); + own_blob(f, "a", 0, 256); + own_scalar(f, "s", 7); + pad_to(f, 256 + 256); + seal(f); + Model m; + if (!load_file(dir, f, &err, &m)) return fail("a valid file was refused: " + err); + if (m.image.size() != 256 || m.blob("a") != 0 || m.scalar("s") != 7) return fail("wrong content"); + } else if (which == L"own_blob_wraps") { + // off + size wraps around 2^64 to a small value: the blob lies outside the image and must be refused + Bytes f = own_header(1, 0, 256); + own_blob(f, "a", 0xFFFFFFFFFFFFFFF0ull, 32); + pad_to(f, 256 + 256); + seal(f); + Model m; + if (load_file(dir, f, &err, &m)) return fail("accepted a blob whose range wraps"); + if (!has(err, "blob a")) return fail("wrong message: " + err); + } else if (which == L"own_image_start_past_end") { + // the tables end (rounded up to 256) behind the end of the file, and isize wraps so that offset + isize + 32 equals the + // file size: the image's range would be reversed + Bytes f = own_header(1, 3, 0xFFFFFFFFFFFFFF00ull); + own_blob(f, "a", 0, 0); + own_scalar(f, "s0", 0); + own_scalar(f, "s1", 0); + own_scalar(f, "s2", 0); + f.resize(256); // the file is 256 bytes and its hash: 288 in all (the scalar table runs into the hash, as it may) + seal(f); + Model m; + if (load_file(dir, f, &err, &m)) return fail("accepted an image that starts behind the end of the file"); + if (!has(err, "image size")) return fail("wrong message: " + err); + } else { + return fail("unknown case"); + } + printf("OK\n"); + return 0; +} diff --git a/clean/rr_f/runtime/tests/test_model_bounds.py b/clean/rr_f/runtime/tests/test_model_bounds.py new file mode 100644 index 00000000..a1361c75 --- /dev/null +++ b/clean/rr_f/runtime/tests/test_model_bounds.py @@ -0,0 +1,93 @@ +# SPDX-License-Identifier: MIT +"""Bounds checks of the RR runtime's model loaders (src/rr_model.cpp: read_wgcm for model.wgcm, load_own for WGRRMOD1). +No GPU and no model data: clean/rr_f/runtime/tests/test_model_bounds.cpp is built with MSVC cl (on PATH, e.g. a developer +prompt or CI, or through vcvars64.bat; skipped without one) together with src/rr_model.cpp (a stub stands in for the +packer of src/rr_pack.cpp, which needs the generated tables), and loads +crafted files whose whole-file SHA-256 is correct but whose offsets, sizes and shapes wrap around 2^64. Each case runs in +its own process, since a loader that reads through a wrapped range may crash. + +Run: python -m unittest clean.rr_f.runtime.tests.test_model_bounds +""" +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) +RT = os.path.join(ROOT, "clean", "rr_f", "runtime") +SOURCES = [os.path.join(RT, "tests", "test_model_bounds.cpp"), os.path.join(RT, "src", "rr_model.cpp")] +_EXE = None + + +def _vcvars64(): + """vcvars64.bat of the newest Visual Studio (vswhere), or None.""" + vswhere = os.path.join(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)"), + "Microsoft Visual Studio", "Installer", "vswhere.exe") + if not os.path.exists(vswhere): + return None + r = subprocess.run([vswhere, "-latest", "-products", "*", "-property", "installationPath"], capture_output=True, + text=True) + vcvars = os.path.join(r.stdout.strip(), "VC", "Auxiliary", "Build", "vcvars64.bat") + return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None + + +def bounds_exe(): + global _EXE + if _EXE is None: + if os.name != "nt": + raise unittest.SkipTest("the loaders use Windows (bcrypt): Windows only") + vcvars = None if shutil.which("cl") else _vcvars64() + if not shutil.which("cl") and not vcvars: + raise unittest.SkipTest("no MSVC cl (not on PATH, no Visual Studio found)") + work = tempfile.mkdtemp(prefix="wg_mb_") + exe = os.path.join(work, "test_model_bounds.exe") + cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s /Fo"%s\\\\" /Fe"%s"' + % (" ".join('"%s"' % s for s in SOURCES), work, exe)) + bat = os.path.join(work, "build.bat") + with open(bat, "w") as f: + f.write("@echo off\r\n") + if vcvars: + f.write('call "%s" >nul || exit /b 1\r\n' % vcvars) + f.write(cl + " || exit /b 1\r\n") + r = subprocess.run(["cmd", "/d", "/c", bat], capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError("build of test_model_bounds failed:\n" + r.stdout + r.stderr) + _EXE = exe + return _EXE + + +class ModelBounds(unittest.TestCase): + def run_case(self, name): + scratch = tempfile.mkdtemp(prefix="wg_mb_") + r = subprocess.run([bounds_exe(), name, scratch], capture_output=True, text=True, timeout=120) + self.assertEqual(r.returncode, 0, "case %s ended abnormally (%s): %s" % (name, r.returncode, r.stdout + r.stderr)) + self.assertEqual(r.stdout.strip(), "OK", "case %s: %s" % (name, r.stdout.strip())) + + # model.wgcm (read_wgcm) + def test_wgcm_valid_file_still_loads(self): + self.run_case("wgcm_valid") + + def test_wgcm_offset_plus_size_wraps(self): + self.run_case("wgcm_offset_wraps") + + def test_wgcm_size_from_a_huge_shape_and_offset_above_2_33(self): + self.run_case("wgcm_size_past_end_after_wrap") + + def test_wgcm_shape_element_count_wraps(self): + self.run_case("wgcm_shape_wraps") + + # WGRRMOD1 (load_own, through load_model) + def test_own_valid_file_still_loads(self): + self.run_case("own_valid") + + def test_own_blob_range_wraps(self): + self.run_case("own_blob_wraps") + + def test_own_image_start_behind_the_end_of_the_file(self): + self.run_case("own_image_start_past_end") + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runtime/model.cpp b/clean/runtime/model.cpp index 9d6345bf..fe417d7f 100644 --- a/clean/runtime/model.cpp +++ b/clean/runtime/model.cpp @@ -454,7 +454,8 @@ struct Reader { size_t pos, end; bool ok = true; bool need(size_t n) { - if (!ok || pos + n > end || pos + n < pos) ok = false; + // Written so that no sum can wrap, and so that no range reaches past the buffer whatever `end` was set to. + if (!ok || end > d.size() || pos > end || n > end - pos) ok = false; return ok; } uint64_t le(int n) { @@ -505,7 +506,8 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std:: r.bytes(key.data(), 32); r.bytes(map_hash.data(), 32); const uint64_t isize = r.le(8); - if (!r.ok || isize > d.size()) return *err = bad + ": bad header", false; + // The index lies between the 88-byte header and the trailing hash (d.size() >= 88 + 32 is known here). + if (!r.ok || isize > d.size() - 32 - 88) return *err = bad + ": bad header", false; // The map shipped with the runtime: its digests are the reference. std::vector mbytes; @@ -559,12 +561,18 @@ bool load_wgcm_model(const std::string& path, const std::string& map_path, std:: const uint64_t dtype = ix.le(1), rank = ix.le(1); Tensor ten; size_t n = 1; - for (uint64_t a = 0; a < rank && ix.ok; ++a) { - ten.shape.push_back((int)ix.le(4)); - n *= (size_t)ten.shape.back(); + bool shape_ok = true; // every dimension above 0, and the element count never above the file's size (no wrap) + for (uint64_t a = 0; a < rank && ix.ok && shape_ok; ++a) { + const int dim = (int)ix.le(4); + if (dim <= 0 || n > d.size() / (size_t)dim) { + shape_ok = false; + break; + } + ten.shape.push_back(dim); + n *= (size_t)dim; } const uint64_t off = ix.le(8), size = ix.le(8); - if (!ix.ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 || + if (!ix.ok || !shape_ok || dtype > 1 || off % 64 || size != n * (dtype ? 1 : 4) || off > d.size() - 32 || size > d.size() - 32 - off) return *err = bad + ": bad tensor entry " + stage + "/" + name, false; h.update(&d[off], (size_t)size); diff --git a/clean/tests/test_model_filter.py b/clean/tests/test_model_filter.py index d7217dae..c1cd9cbf 100644 --- a/clean/tests/test_model_filter.py +++ b/clean/tests/test_model_filter.py @@ -12,6 +12,7 @@ import hashlib import os import shutil +import struct import subprocess import tempfile import unittest @@ -268,5 +269,73 @@ def test_shipped_map_is_readable(self): self.assertRegex(self.case(t, map_file=DEFAULT_MAP), r"^ERR .*stage enc0 does not match the map's digest") +class LoaderBounds(unittest.TestCase): + """Issue #66: bounds checks of clean/runtime/model.cpp against crafted files with a correct whole-file SHA-256 (the + hash is unkeyed): the index may not reach into the hash or past the file, a shape may not be zero, negative or wrap + the element count.""" + + @staticmethod + def crafted(dims, size, data=b""): + """A model.wgcm with one stage "enc0" holding one float32 tensor "t" of the given dimensions (as 32-bit words), + `size` bytes at a 64-aligned offset (`data` there), a zero stage digest, and a correct whole-file hash.""" + name, stage = b"t", b"enc0" + entry = struct.pack("