From c1a6e78d976eeb8411cf4ab2cb6cf09e41eea358 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 01:40:31 -0500 Subject: [PATCH] Runtime build: the built-in files as string literals instead of byte lists write_embedded wrote every built-in file (all code objects, each also under its gfx1201 name, and every DXIL of dx12/) as a brace list of decimal bytes: tens of millions of integer elements in one generated embedded_files.cpp, one compiler node each. Compiling that one file was the memory peak of the runtime build. Each file is now one string literal in adjacent 64-byte pieces (3-digit octal escapes, printable bytes as themselves): the same bytes in the library, the same table line, a smaller file and far less compiler memory. A runtime built by MSVC would hit its 64 KB string-literal limit: the generated file says so with #error (clang++ and g++ build the runtime). clean/tests/test_embedded_table.py: the runtime's own lookup (wg_read_file, through clean/tests/embed_dump.cpp) walks the compiled table; every file's SHA-256 equals its source file's (synthetic files with every byte value, an empty file, quotes / backslashes / trigraph text; a mutant with one changed byte is caught), and a --build mode for the table of a real build folder. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/runner/build.py | 29 ++++- clean/tests/embed_dump.cpp | 45 ++++++++ clean/tests/test_embedded_table.py | 171 +++++++++++++++++++++++++++++ 3 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 clean/tests/embed_dump.cpp create mode 100644 clean/tests/test_embedded_table.py diff --git a/clean/runner/build.py b/clean/runner/build.py index 272829f8..7ea08c2c 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -229,6 +229,20 @@ def scramble(name, data): return bytes(out) +MSVC_LITERAL_MAX = 60000 # MSVC caps one (concatenated) string literal near 64 KB; clang and g++ have no such cap +# bytes that may stand for themselves inside a C string literal: printable ASCII except " \ ? (the last: trigraphs) +_C_RAW = frozenset(b for b in range(0x20, 0x7f) if chr(b) not in '"\\?') +_C_ESC = [chr(b) if b in _C_RAW else "\\%03o" % b for b in range(256)] # 3-digit octal: never runs into the next byte + + +def c_string_pieces(data, piece=64): + """data as adjacent C string-literal pieces of `piece` bytes each (a line each, at most 4 piece + 2 characters).""" + if not data: + return ['""'] + esc = _C_ESC + return ['"' + "".join(esc[b] for b in data[j:j + piece]) + '"' for j in range(0, len(data), piece)] + + def write_embedded(out_dir): """embedded_files.cpp: the EMBED_FILES present in out_dir, scrambled, for the runtime library. None if none.""" files = [n for n in EMBED_FILES if os.path.isfile(os.path.join(out_dir, n))] @@ -266,12 +280,19 @@ def write_embedded(out_dir): lines = ["// Generated by clean/runner/build.py: files built into the runtime library (clean/runtime/embedded.h).", "#include ", "namespace wg_embedded {", "struct File { const char* name; const unsigned char* data; size_t size; };"] + sizes = {n: os.path.getsize(src[n]) for n in files} + if max(sizes.values()) > MSVC_LITERAL_MAX: + lines.append("#if defined(_MSC_VER) && !defined(__clang__)") + lines.append('#error "embedded_files.cpp: the built-in files are string literals; build the runtime with clang++ ' + 'or g++ (MSVC limits a string literal to 64 KB)"') + lines.append("#endif") for i, n in enumerate(files): data = scramble(n, open(src[n], "rb").read()) - lines.append(f"static const unsigned char f{i}[] = {{") - for j in range(0, len(data), 32): - lines.append(",".join(str(b) for b in data[j:j + 32]) + ",") - lines.append("};") + # one string literal per file in adjacent pieces (the compiler concatenates them; the terminating NUL is one + # byte past the file's size): far less compiler memory than a brace list of decimal bytes + lines.append(f"static const unsigned char f{i}[] =") + lines += c_string_pieces(data) + lines.append(";") lines.append("extern const File kFiles[] = {" + ", ".join( f'{{"{n}", f{i}, {os.path.getsize(src[n])}}}' for i, n in enumerate(files)) + "};") lines.append(f"extern const size_t kCount = {len(files)};") diff --git a/clean/tests/embed_dump.cpp b/clean/tests/embed_dump.cpp new file mode 100644 index 00000000..3f42091c --- /dev/null +++ b/clean/tests/embed_dump.cpp @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: MIT +// Test harness for the runtime library's built-in file table (clean/runtime/embedded.cpp + a generated +// embedded_files.cpp, clean/tests/test_embedded_table.py): walks the table and reads every file through the runtime's +// own lookup, wg_read_file, from a folder without that file (so the built-in copy is returned), and writes the bytes to +// /. Prints the file count first. +// embed_dump +#include +#include +#include +#include + +#include "embedded.h" + +namespace wg_embedded { +struct File { + const char* name; + const unsigned char* data; + size_t size; +}; +extern const File kFiles[]; +extern const size_t kCount; +} // namespace wg_embedded + +int main(int argc, char** argv) { + if (argc != 3) { + std::fprintf(stderr, "usage: embed_dump \n"); + return 2; + } + const std::string empty = argv[1], out = argv[2]; + std::printf("count %zu\n", wg_embedded::kCount); + for (size_t i = 0; i < wg_embedded::kCount; ++i) { + const std::string name = wg_embedded::kFiles[i].name; + std::vector data; + if (!wg_read_file(empty + "/" + name, &data)) { + std::fprintf(stderr, "lookup failed: %s\n", name.c_str()); + return 1; + } + std::FILE* fp = std::fopen((out + "/" + name).c_str(), "wb"); + if (!fp) return 1; + if (!data.empty() && std::fwrite(data.data(), 1, data.size(), fp) != data.size()) return 1; + std::fclose(fp); + std::printf("%s %zu\n", name.c_str(), data.size()); + } + return 0; +} diff --git a/clean/tests/test_embedded_table.py b/clean/tests/test_embedded_table.py new file mode 100644 index 00000000..33f9fc9f --- /dev/null +++ b/clean/tests/test_embedded_table.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: MIT +"""The runtime library's built-in file table (clean/runner/build.py write_embedded -> embedded_files.cpp, read by +clean/runtime/embedded.cpp): every file the runtime's own lookup (wg_read_file) returns has the SHA-256 of its source +file. A host harness (clean/tests/embed_dump.cpp) walks the compiled table and writes each file back out. + + python -m unittest clean.tests.test_embedded_table # CPU: synthetic files (every byte value, empty file, + # quotes / backslashes / "??" in text) + a mutant + python -m clean.tests.test_embedded_table --build [--model-l] [--no-model-k] [--cxx ] + # the table of a real build folder (count, bytes)""" +import argparse +import hashlib +import json +import os +import random +import re +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if ROOT not in sys.path: + sys.path.insert(0, ROOT) +from clean.runner import build # noqa: E402 + + +def source_of(out_dir, name): + """The file a table entry was made from (write_embedded's naming).""" + if name == "manifest.json": + return os.path.join(out_dir, "dx12", "manifest.json") + if name.endswith(".dxil"): + return os.path.join(out_dir, "dx12", name) + m = re.match(r"^([A-Za-z0-9]+)\.(.+\.co)$", name) + if m and not os.path.isfile(os.path.join(out_dir, name)): + p = os.path.join(out_dir, m.group(1), m.group(2)) # an extra architecture's code object + if os.path.isfile(p): + return p + return os.path.join(out_dir, name) + + +def sha(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +def dump(table_cpp, cxx=None, opt="-O1"): + """Compiles embedded.cpp + table_cpp + the harness and returns {name: bytes} of the runtime's lookup.""" + tmp = tempfile.mkdtemp(prefix="wg_embed_") + try: + exe = os.path.join(tmp, "embed_dump" + (".exe" if os.name == "nt" else "")) + cxx = cxx or os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++") + if not cxx: + raise unittest.SkipTest("no clang++ / g++ (the table is built by clang++ or g++ only)") + cmd = [cxx, "-std=c++17", opt, "-DWG_EMBEDDED_TABLE", "-I", os.path.join(ROOT, "clean", "runtime")] + cmd += ["-D_CRT_SECURE_NO_WARNINGS"] if os.name == "nt" else [] + cmd += [os.path.join(ROOT, "clean", "runtime", "embedded.cpp"), table_cpp, + os.path.join(ROOT, "clean", "tests", "embed_dump.cpp"), "-o", exe] + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode != 0: + raise AssertionError("harness build failed:\n" + r.stdout[-2000:] + r.stderr[-2000:]) + empty, out = os.path.join(tmp, "empty"), os.path.join(tmp, "out") + os.makedirs(empty) + os.makedirs(out) + r = subprocess.run([exe, empty, out], capture_output=True, text=True) + if r.returncode != 0: + raise AssertionError("harness failed:\n" + r.stdout[-2000:] + r.stderr[-2000:]) + count = int(r.stdout.splitlines()[0].split()[1]) + got = {} + for line in r.stdout.splitlines()[1:]: + name = line.rsplit(" ", 1)[0] + with open(os.path.join(out, name), "rb") as f: + got[name] = f.read() + assert count == len(got), (count, len(got)) + return got + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def check_folder(out_dir, got): + """{name: (ok, size)} against the source files; raises on a name the folder cannot account for.""" + res = {} + for name, data in got.items(): + src = source_of(out_dir, name) + res[name] = (hashlib.sha256(data).hexdigest() == sha(src), len(data)) + return res + + +def table_names(path): + with open(path) as f: + return re.findall(r'\{"([^"]+)", f\d+, \d+\}', f.read()) + + +class EmbeddedTable(unittest.TestCase): + def _folder(self): + d = tempfile.mkdtemp(prefix="wg_embed_src_") + self.addCleanup(shutil.rmtree, d, True) + rng = random.Random(20261115) + files = { + "a.co": bytes(range(256)) * 3 + b"\x00\x7f\xff\x22\x5c", # every byte value, size not a multiple of 8 + "b.co": b"", # empty + "c.json": b'{"k": "a\\\\b \\"q\\" ??= ??/ ??\' \\x41B \\101 1"}\n', # quotes, backslashes, trigraphs + "d.co": bytes(rng.getrandbits(8) for _ in range(4099)), + } + for n, data in files.items(): + with open(os.path.join(d, n), "wb") as f: + f.write(data) + os.makedirs(os.path.join(d, "dx12")) + with open(os.path.join(d, "dx12", "k.dxil"), "wb") as f: + f.write(bytes(rng.getrandbits(8) for _ in range(1000))) + with open(os.path.join(d, "dx12", "manifest.json"), "w") as f: + json.dump({"format": 1, "kernels": [{"name": "k", "file": "k.dxil"}]}, f) + return d, list(files) + + def _write(self, d, names): + with mock.patch.object(build, "EMBED_FILES", names), mock.patch.object(build, "EXTRA_ARCHS", []): + return build.write_embedded(d) + + def test_every_file_byte_identical(self): + d, names = self._folder() + path = self._write(d, names) + got = dump(path) + res = check_folder(d, got) + # the named files, the primary architecture's copies of the code objects, the manifest and its DXIL + expect = set(names) | {build.ARCH + "." + n for n in names if n.endswith(".co")} | {"manifest.json", "k.dxil"} + self.assertEqual(set(res), expect) + self.assertTrue(all(ok for ok, _ in res.values()), res) + self.assertEqual(sorted(table_names(path)), sorted(expect)) # the table line keeps its format + + def test_mutant_caught(self): + """one byte of one literal changed: the comparison must fail for exactly that file""" + d, names = self._folder() + path = self._write(d, names) + with open(path) as f: + text = f.read() + i = text.index("static const unsigned char f3[] =") # d.co (random bytes) + j = text.index('"', i) + 1 + c = text[j] + text = text[:j] + ("\\001" if c != "\\" else "A") + text[j + (4 if c == "\\" else 1):] + with open(path, "w") as f: + f.write(text) + res = check_folder(d, dump(path)) + self.assertEqual(sorted(n for n, (ok, _) in res.items() if not ok), ["d.co"]) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True, help="a build folder (its code objects, dx12/manifest.json)") + ap.add_argument("--model-l", action="store_true") + ap.add_argument("--no-model-k", action="store_true") + ap.add_argument("--cxx", default=None) + ap.add_argument("--json", default=None) + a = ap.parse_args() + build.set_model_l(a.model_l) + build.set_model_k(not a.no_model_k) + path = build.write_embedded(a.build) + got = dump(path, a.cxx, "-O2") + res = check_folder(a.build, got) + bad = sorted(n for n, (ok, _) in res.items() if not ok) + rec = {"files": len(res), "bytes": sum(s for _, s in res.values()), "mismatches": bad, + "table": {n: {"bytes": s, "sha256_match": ok} for n, (ok, s) in sorted(res.items())}} + print("files %d, bytes %d, SHA-256 mismatches %d" % (rec["files"], rec["bytes"], len(bad))) + if a.json: + with open(a.json, "w") as f: + json.dump(rec, f, indent=1) + return 1 if bad else 0 + + +if __name__ == "__main__": + sys.exit(main())