diff --git a/clean/k/runtime/model_k.cpp b/clean/k/runtime/model_k.cpp index bc879c40..83621515 100644 --- a/clean/k/runtime/model_k.cpp +++ b/clean/k/runtime/model_k.cpp @@ -126,10 +126,10 @@ bool model_from_options(const WgRtConfig& c, ModelId* model, std::string* err) { *model = ModelId::M; std::string bad; if (!last_model_option(text, model, &bad)) { - *err = "unknown model \"" + bad + "\" in the runtime options (known: M, K)"; + *err = "unknown model \"" + bad + "\" in the runtime options (known: M, K" + (model_l_built() ? ", L)" : ")"); return false; } - if (*model == ModelId::L) { // #89: recognised, not part of the runtime yet + if (*model == ModelId::L && !model_l_built()) { // #89: recognised; runs when its sources are compiled in *err = model_not_built_yet(ModelId::L); return false; } diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py new file mode 100644 index 00000000..621093c1 --- /dev/null +++ b/clean/l/runner/l_frame_check.py @@ -0,0 +1,425 @@ +# SPDX-License-Identifier: MIT +"""Model L (issue #89): the runtime's Model L frame on HIP against the CPU reference, synthetic model and frames only. +Model M's frame on the same harness is the control. + + python clean/l/runner/l_frame_check.py --build --rocm --prepare # CPU + python clean/l/runner/l_frame_check.py --build --rocm --run # GPU, inside a GPU slot + python clean/l/runner/l_frame_check.py --build --compare --json out.json # CPU + + first holds Model M's code objects (python -m clean.runner.build --out --rocm --no-runtime +--no-model-k). --prepare adds Model L's code objects (l_c32_check.compile_kernels), the runtime library with Model L +(WG_MODEL_L; build.py has no Model L row yet) and the synthetic models: Model M in raw/, Model L as L/model.wgcm with +its synthetic preset-L map (the runtime loads it through l::load_wgcm_model). --run runs the frame sequence SEQUENCE +through clean/runner/pipeline.py (backend HIP, every stage output read back) for Model L (runtime option model=L) and +Model M (post_pack=none: the base coefficient format and post kernel Model L uses), and keeps the outputs in . +Each model runs twice: with the bottleneck as six block launches (swin_bneck=none), the run the stages are checked on, +since the default bottleneck megakernel writes only b5's output and leaves b0 ... b4's buffers unwritten; and with the +runtime's default kernels, whose outputs must be byte-identical to the first run's (b0 ... b4 aside, and on the first +frame the previous frame's state buffers, which no frame has written and a reset frame does not read). +Only --run uses the GPU; the HIP runtime is loaded with the environment of clean/runner/hip.py. + +--compare checks every frame on the runtime's own stage inputs, as clean/tests/test_runtime.py does in the emulator, +but against the float64 references with the kernels' GELU (sigmoid): +- the frame state: frame counter, parity, reset and the Swin window offsets; the latent, motion history and full + history handed over from the previous frame (byte for byte); +- enc0: Model L's encoder (clean/l/reference/oracle.py encode) or Model M's (clean/reference/encoder.py), features under + Model M's per-stage rule (l_c32_check.passes), the motion surfaces at most 1 FP16 ulp and 99.5 % exact; +- the ten Swin stages (clean/reference/swin.py run_stage), each output under the per-stage rule; +- dec0: Model L's decoder (oracle decode) or Model M's (clean/reference/decoder.py) on the runtime's dec1 output: the + A, B and latent codes and the C values, each at least 99.5 % identical (l_c32_check's end-to-end bound); +- post (clean/reference/post.py reconstruct): history and display under test_post_emu.py's FP16 acceptance, the + reduced history codes within 1 and 99 % exact; the filter (when the post kernel writes a display image) as + test_post_emu.py; +- exposure: the runtime's value and the reference's, reported (not a pass condition). +Output: counts only, no timing. +""" +import argparse +import copy +import json +import os +import pickle +import shutil +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(HERE))) +sys.path.insert(0, ROOT) + +L_SOURCES = [os.path.join("..", "l", "runtime", "l_model.cpp"), os.path.join("..", "l", "runtime", "l_pack.cpp")] +SEED = 89 +MAX_RENDER = (96, 64) +OUTPUT = (120, 72) +# (render size, game reset): aligned (96 x 64), not aligned (80 x 48, 77 x 45), resets by the game, by a size change +# and on the first frame; both parities on every size +SEQUENCE = [((96, 64), False), ((96, 64), False), ((96, 64), True), ((96, 64), False), ((96, 64), False), + ((80, 48), False), ((80, 48), False), ((80, 48), False), ((77, 45), False), ((77, 45), True), + ((77, 45), False)] +PRE_EXPOSURE = 1.25 + + +def model_dir(build): + return os.path.join(build, "lframe_model") + + +def l_map(build): + return os.path.join(model_dir(build), "maps", "preset-L.v1.map.json") + + +# ---- --prepare (CPU) ------------------------------------------------------------------------------------------------ + +def prepare(build, rocm): + from clean.l.reference import oracle as L + from clean.l.runner import l_c32_check as C32 + from clean.runner import build as B + from clean.runner import synthetic_model as SM + from clean.runner.hip import clean_process_environment + from clean.runner.pipeline import logical_tensors + sys.path.insert(0, os.path.join(ROOT, "clean", "l", "runtime", "tests")) + import test_l_runtime_load as LL + + clean_process_environment(rocm) + C32.compile_kernels(build, rocm) + # the runtime library with Model L (and without Model K, as the code objects of --no-model-k) + B.set_model("K", False) + B.RUNTIME_SOURCES.extend(L_SOURCES) + plain = B.runtime_command + B.runtime_command = lambda cxx, out, embedded=None: (lambda c: c[:1] + ["-DWG_MODEL_L"] + c[1:])( + plain(cxx, out, embedded)) + B.build_runtime(B.find_host_cxx(rocm), build, embed=False) + # the HIP device loads "." (clean/runtime/device.cpp; build.py's write_embedded makes these copies of + # the files it builds in): the same for every code object here, Model L's included + for n in os.listdir(build): + if n.endswith(".co") and not n.startswith("gfx"): + shutil.copyfile(os.path.join(build, n), os.path.join(build, f"{B.ARCH}.{n}")) + + d = model_dir(build) + if os.path.isdir(d): + shutil.rmtree(d) + os.makedirs(os.path.join(d, "maps")) + SM.write_raw(d, SM.synthetic_tensors(1)) + raw_l = LL.raw_model(SEED) + synthetic_map = SM.write_wgcm(os.path.join(d, "L"), raw_l, order=list(LL.specs()), map_file=None) + shutil.copyfile(synthetic_map, l_map(build)) + # the reference's parameters: the values the file holds, in the oracle's shapes and types + p = L.random_params(np.random.default_rng(SEED)) + params_l = {} + for stage, tensors in raw_l.items(): + params_l[stage] = {} + for name, a in tensors.items(): + if stage in p and name in p[stage]: + ref = np.asarray(p[stage][name]) + params_l[stage][name] = np.asarray(a).reshape(ref.shape).astype(ref.dtype) + else: + params_l[stage][name] = np.asarray(a) + params_m = logical_tensors(SM.read_raw(d)) + with open(os.path.join(d, "params.pkl"), "wb") as f: + pickle.dump({"L": params_l, "M": params_m}, f) + print("prepared", build) + + +# ---- --run (GPU) ---------------------------------------------------------------------------------------------------- + +def frame_inputs(rng, rw, rh, reset): + from clean.reference import encoder as En + x = En.random_inputs(rng, rw=rw, rh=rh, fw=-(-rw // 32) * 32, fh=-(-rh // 32) * 32) + return {"color": x["color"], "depth": x["depth"], "motion": x["motion"], "mv_scale": x["mv_scale"], + "jitter": x["jitter"], "exposure": 1.0, "pre_exposure": PRE_EXPOSURE, "reset": reset} + + +def run(build, rocm): + from clean.runner.hip import clean_process_environment + clean_process_environment(rocm) + from clean.runner.pipeline import Pipeline + d = model_dir(build) + setups = {"L": dict(map_path=l_map(build), check_packing=False, kernels={"options": "model=L"}), + "M": dict(check_packing=True, kernels={"pack": "none"})} + for model, kw in setups.items(): + # "stages": the bottleneck as six block launches, so that b0 ... b4 are written (the default megakernel writes + # only b5's output); "default": the runtime's default kernels, which must give the same bytes + for config in ("stages", "default"): + k = dict(kw, kernels=dict(kw["kernels"], **({"bneck": "none"} if config == "stages" else {}))) + p = Pipeline(build, rocm, d, MAX_RENDER, OUTPUT, {"depth_inverted": False, "auto_exposure": True}, **k) + rng = np.random.default_rng(SEED) + frames = [] + for (rw, rh), reset in SEQUENCE: + fr = frame_inputs(rng, rw, rh, reset) + out = p.frame(fr, debug=True) + frames.append({"inputs": fr, "out": out}) + rec = {"model": model, "config": config, "kernels": p.kernels(), "notes": p.notes(), + "model_source": p.model_source, "dw": p.dw, "dh": p.dh, "ow": p.ow, "oh": p.oh, + "taps": p.output_taps(), "frames": frames} + with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "wb") as f: + pickle.dump(rec, f) + print(model, config, "frames", len(frames), "bneck", rec["kernels"].get("swin_bneck"), "post", + rec["kernels"].get("post")) + del p + + +# ---- --compare (CPU) ------------------------------------------------------------------------------------------------ + +def _f16_steps(a, b): + ia = np.asarray(a, np.float32).astype(np.float16).view(np.int16).astype(np.int64) + ib = np.asarray(b, np.float32).astype(np.float16).view(np.int16).astype(np.int64) + return np.abs(np.where(ia < 0, -(ia & 0x7FFF), ia) - np.where(ib < 0, -(ib & 0x7FFF), ib)) + + +def fp8_stage(got, ref): + from clean.l.runner import l_c32_check as C32 + m = C32.metrics(got, ref) + m["pass"] = bool(C32.passes(m)) + return m + + +def motion_surface(got, ref): + ref = np.asarray(ref, np.float32) + fin = np.isfinite(ref) + u = _f16_steps(got[fin], ref[fin]) + same = bool(np.array_equal(np.asarray(got, np.float32)[~fin], ref[~fin], equal_nan=True)) + return {"values": int(u.size), "differ": int(np.sum(u > 0)), "max_f16_steps": int(u.max()) if u.size else 0, + "pass": same and (int(u.max()) if u.size else 0) <= 1 and float(np.mean(u == 0)) >= 0.995} + + +def codes(got, ref, scale): + g, r = np.rint(np.asarray(got, np.float64) * scale), np.rint(np.asarray(ref, np.float64) * scale) + dc = np.abs(g - r) + return {"values": int(dc.size), "differ": int(np.sum(dc > 0)), "max_code_diff": int(dc.max()) if dc.size else 0} + + +def same_values(got, ref): + g, r = np.asarray(got, np.float32), np.asarray(ref, np.float32) + dif = ~((g == r) | (np.isnan(g) & np.isnan(r))) + return {"values": int(dif.size), "differ": int(np.sum(dif)), + "max_f16_steps": int(_f16_steps(g[dif], r[dif]).max()) if dif.any() else 0} + + +def post_fp16(got, ref): + finite = np.isfinite(got) + s = _f16_steps(got, ref) + d = np.abs(np.where(finite, got, 0.0).astype(np.float64) - ref) + m = {"values": int(s.size), "differ": int(np.sum(s > 0)), "max_f16_steps": int(s.max()) if s.size else 0, + "nonfinite": int(np.sum(~finite)), "frac_over_1": float(np.mean(s > 1)), "frac_diff": float(np.mean(s > 0)), + "mean_rel_rms": float(d.mean() / max(np.sqrt(np.mean(np.square(ref, dtype=np.float64))), 1e-30))} + m["pass"] = (m["nonfinite"] == 0 and m["frac_over_1"] <= 1e-4 and m["mean_rel_rms"] <= 1e-3 + and m["frac_diff"] <= 0.01) + return m + + +def compare_model(rec, params): + from clean.l.reference import oracle as L + from clean.l.reference import table as T + from clean.reference import decoder as MD + from clean.reference import encoder as ME + from clean.reference import exposure as Ex + from clean.reference import post as Po + from clean.reference import swin as S + model = rec["model"] + table = copy.deepcopy(T.TABLE) + table["consts"]["default"]["gelu"] = "sigmoid" # the kernels' GELU (clean/l/runner/l_c32_check.py CONSTS) + D = (rec["dw"], rec["dh"]) + res, prev, prev_jitter, n, f = [], None, None, 0, 0 + prev_size = None + for k, fd in enumerate(rec["frames"]): + fr, out = fd["inputs"], fd["out"] + rh, rw = fr["color"].shape[:2] + fw, fh = -(-rw // 32) * 32, -(-rh // 32) * 32 + size_change = prev_size is not None and prev_size != (rw, rh) + eff_reset = prev is None or bool(fr["reset"]) or size_change + jp = tuple(float(v) for v in (fr["jitter"] if eff_reset else prev_jitter)) + info = out["state"]["info"] + m = {"frame": k, "render": [rw, rh], "reset": eff_reset, "parity": f} + offsets_ok = all(tuple(info["offsets"][s]) == tuple(S.window_offset(s, f)) for s in T.SHARED_STAGES) + m["state"] = {"n_parity_reset": (info["n"], info["parity"], bool(info["reset"])) == (n, f, eff_reset), + "jitter_prev": bool(np.array_equal(np.float32(info["jitter_prev"]), np.float32(jp))), + "offsets": offsets_ok} + if prev is not None and not size_change: + m["state"]["latent_handed_over"] = bool(np.array_equal(out["state"]["latent_prev"], + prev["dec0"]["latent"])) + m["state"]["motion_history_handed_over"] = bool(np.array_equal(out["state"]["motion_history"], + prev["enc0"]["m_near"])) + m["state"]["history_handed_over"] = bool(np.array_equal(out["state"]["history_prev"], + prev["post"]["history"], equal_nan=True)) + m["state"]["pass"] = all(m["state"].values()) + pi = np.float32(fr["pre_exposure"]) + cnet = (fr["color"] / pi).astype(np.float32) + e_ref = Ex.exposure(fr["color"], fr["pre_exposure"], 0.18) + m["exposure"] = {"runtime": float(out["exposure"]["e"]), "reference": float(e_ref), + "identical": float(out["exposure"]["e"]) == float(e_ref)} + + # enc0 on the runtime's own inputs (the state it read) + reduced = prev["post"]["reduced"] if prev is not None else np.zeros((D[1] // 2, D[0] // 2, 3), np.float32) + x0 = {"color": cnet, "depth": fr["depth"], "motion": fr["motion"], "mv_scale": fr["mv_scale"], + "mv_bias": np.zeros(2, np.float32), "jitter": fr["jitter"], "jitter_prev": np.array(jp), + "exposure": out["exposure"]["e"], "exposure_prev": out["exposure"]["e_prev"], "reset": eff_reset, + "frame": n, "depth_inverted": False, "feature_size": (fw, fh), "recon_size": D, + "reduced_history": reduced, "latent": out["state"]["latent_prev"], + "motion_history": out["state"]["motion_history"]} + if eff_reset: + x0["latent"] = np.zeros_like(x0["latent"]) + x0["motion_history"] = np.zeros_like(x0["motion_history"]) + x0["reduced_history"] = np.zeros_like(reduced) + ref = L.encode(x0, params, f, t=table) if model == "L" else ME.encode(x0, params["enc0"]) + m["enc0"] = {"features": fp8_stage(out["enc0"]["features"], ref["features"])} + for s in ("m_near", "m_far", "m_center"): + m["enc0"][s] = motion_surface(out["enc0"][s], ref[s]) + + # the ten Swin stages + x, skips = out["enc0"]["features"], {} + for s in T.SHARED_STAGES: + off = S.window_offset(s, f) + kind = S.STAGES[s]["kind"] + skip = skips.get("enc2" if s == "dec2" else "enc1") if kind == "dec" else None + ref = S.run_stage(s, params[s], x, off, skip, gelu="sigmoid") + m[s] = {key: fp8_stage(out[s][key], ref[key]) for key in out[s]} + if kind == "enc": + skips[s] = out[s]["skip"] + x = out[s]["merged"] + else: + x = out[s]["out"] + + # dec0 on the runtime's dec1 output + xd = {"features": out["dec1"]["out"], "motion": out["enc0"]["m_near"], + "latent_prev": out["state"]["latent_prev"] if not eff_reset else np.zeros((rh, rw, 4), np.float32), + "jitter": fr["jitter"], "jitter_prev": np.array(jp), "reset": eff_reset, "feature_size": (fw, fh), + "recon_size": D} + ref = L.decode(xd, params, f, t=table) if model == "L" else MD.decode(xd, params["dec0"]) + dec = out["dec0"] + m["dec0"] = {"codes_A": codes(dec["A"], ref["A"], 255.0), "codes_B": codes(dec["B"], ref["B"], 255.0), + "codes_L": codes(dec["latent"], ref["latent"], 127.0), "C": same_values(dec["C"], ref["C"])} + m["dec0"]["pass"] = all(v["differ"] <= 0.005 * v["values"] for v in m["dec0"].values()) + + # post and filter on the runtime's own inputs + xp = {"color": cnet, "exposure": out["exposure"]["e"], "exposure_prev": out["exposure"]["e_prev"], + "A": dec["A"], "B": dec["B"], "C": dec["C"], + "history": out["state"]["history_prev"] if not eff_reset else np.zeros((D[1], D[0], 3), np.float32), + "m_near": out["enc0"]["m_near"], "m_center": out["enc0"]["m_center"], "m_far": out["enc0"]["m_far"], + "jitter": fr["jitter"], "reset": eff_reset, "recon_size": D} + ref = Po.reconstruct(xp, float(np.asarray(params["post"]["s"]).reshape(-1)[0])) + m["post"] = {"history": post_fp16(out["post"]["history"], ref["history"])} + if out["post"]["display"] is not None: + m["post"]["display"] = post_fp16(out["post"]["display"], ref["display"]) + dc = codes(out["post"]["reduced"], ref["reduced"], 1023.0) + dc["pass"] = dc["max_code_diff"] <= 1 and dc["differ"] <= 0.01 * dc["values"] + m["post"]["reduced"] = dc + if out["post"]["display"] is not None: + disp, e = out["post"]["display"], out["exposure"]["e"] + if rec["taps"] is not None: + fo = Po.output_filter_original(disp, e, (rec["ow"], rec["oh"]), rec["taps"]) + else: + fo = Po.output_filter(disp, e, (rec["ow"], rec["oh"])) + fo = (np.asarray(fo, np.float32) * pi).astype(np.float32) + close = np.isclose(out["out"], fo, rtol=1e-5, atol=2e-9) + m["filter"] = {"values": int(close.size), "outside": int(np.sum(~close)), "pass": bool(close.all())} + + stage_pass = {s: (all(v["pass"] for v in m[s].values() if isinstance(v, dict) and "pass" in v) + if isinstance(m[s], dict) else True) + for s in ["enc0"] + list(T.SHARED_STAGES) + ["post"]} + stage_pass["dec0"] = m["dec0"]["pass"] + stage_pass["state"] = m["state"]["pass"] + if "filter" in m: + stage_pass["filter"] = m["filter"]["pass"] + m["stage_pass"] = stage_pass + m["pass"] = all(stage_pass.values()) + res.append(m) + prev, prev_jitter, prev_size = out, np.asarray(fr["jitter"], np.float32), (rw, rh) + n, f = (n + 1) % 8, 1 - f + return res + + +def summary(frames): + """Per stage: frames passing, values compared and values differing (FP8 / FP16 steps or codes) over all frames.""" + out = {} + for m in frames: + for stage, part in m.items(): + if stage in ("frame", "render", "reset", "parity", "pass", "stage_pass", "exposure"): + continue + s = out.setdefault(stage, {"frames": 0, "passed": 0, "values": 0, "differ": 0}) + s["frames"] += 1 + s["passed"] += int(m["stage_pass"].get(stage, True)) + for v in part.values() if isinstance(part, dict) else []: + if isinstance(v, dict): + s["values"] += v.get("values", 0) + s["differ"] += v.get("differ", v.get("outside", 0)) + if stage == "filter": + s["values"] += part["values"] + s["differ"] += part["outside"] + out["exposure_identical"] = sum(int(m["exposure"]["identical"]) for m in frames) + return out + + +UNWRITTEN_ON_FIRST = ("state.latent_prev", "state.motion_history", "state.history_prev") + + +def default_identical(a, b, prefix=""): + """Outputs of the default kernels (b) that differ from the six-launch bottleneck's (a), byte for byte; b0 ... b4 + are not compared (the megakernel does not write them).""" + bad = [] + for k in a: + name = prefix + k + if name in ("b0", "b1", "b2", "b3", "b4"): + continue + x, y = a[k], b[k] + if isinstance(x, dict): + bad += default_identical(x, y, name + ".") + elif isinstance(x, np.ndarray): + if x.shape != y.shape or x.tobytes() != y.tobytes(): + bad.append(name) + elif x != y: + bad.append(name) + return bad + + +def compare(build, json_path): + with open(os.path.join(model_dir(build), "params.pkl"), "rb") as f: + params = pickle.load(f) + res = {} + for model in ("L", "M"): + recs = {} + for config in ("stages", "default"): + with open(os.path.join(build, f"lframe_{model}_{config}.pkl"), "rb") as f: + recs[config] = pickle.load(f) + frames = compare_model(recs["stages"], params[model]) + # on the first frame the previous frame's state buffers hold what no frame wrote (and a reset frame does not + # read them): not compared there + same = [[n for n in default_identical(a["out"], b["out"]) if not (i == 0 and n in UNWRITTEN_ON_FIRST)] + for i, (a, b) in enumerate(zip(recs["stages"]["frames"], recs["default"]["frames"]))] + res[model] = {"kernels": recs["default"]["kernels"], "kernels_stages": recs["stages"]["kernels"], + "model_source": recs["default"]["model_source"], "notes": recs["default"]["notes"], + "frames": frames, "summary": summary(frames), + "default_identical": {"frames": len(same), "identical": sum(not d for d in same), + "differing_outputs": sorted({k for d in same for k in d})}, + "pass": all(m["pass"] for m in frames) and not any(same)} + res["pass"] = res["L"]["pass"] and res["M"]["pass"] + with open(json_path, "w") as f: + json.dump(res, f, indent=1, default=lambda o: o.item() if hasattr(o, "item") else str(o)) + for model in ("L", "M"): + print(model, "pass" if res[model]["pass"] else "FAIL", json.dumps(res[model]["summary"])) + print(" default kernels against the six-launch bottleneck:", json.dumps(res[model]["default_identical"])) + for m in res[model]["frames"]: + if not m["pass"]: + print(" frame", m["frame"], "failing:", [s for s, ok in m["stage_pass"].items() if not ok]) + return 0 if res["pass"] else 1 + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--build", required=True) + ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--prepare", action="store_true") + g.add_argument("--run", action="store_true") + g.add_argument("--compare", action="store_true") + ap.add_argument("--json") + a = ap.parse_args(argv) + if a.prepare: + prepare(a.build, a.rocm) + return 0 + if a.run: + run(a.build, a.rocm) + return 0 + return compare(a.build, a.json or os.path.join(a.build, "lframe_check.json")) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/clean/l/runtime/tests/test_l_runtime_abi.py b/clean/l/runtime/tests/test_l_runtime_abi.py new file mode 100644 index 00000000..730f8c38 --- /dev/null +++ b/clean/l/runtime/tests/test_l_runtime_abi.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: MIT +"""Model L's code objects against what the runtime launches (#89), without running them: l_enc_values.co, l_ops.co +and l_swin.co are compiled (clean/l/runner/l_c32_check.py compile_kernels; HIP SDK clang, compile only), and each +kernel the runtime frame launches (clean/runtime/runtime.cpp, clean/l/FORMATS.md) must be exported with the arguments +the runtime passes: their number and sizes (a device address 8 bytes, an int 4, the encoder parameters WgEncParams +116, the decoder parameters WgDecParams 56). The kernels' own metadata (the AMDGPU metadata note of each code object, +amdhsa.kernels) is read here; nothing is loaded or executed. The argument kinds are the frame trace's table +(clean/tests/runtime_frame_trace.cpp), so the recorder that checks the runtime's arguments and the kernels agree. + +Skipped without the HIP SDK (ROCM_PATH or C:\\Program Files\\AMD\\ROCm\\7.2). + +Run (repository root): + python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_abi.py +""" +import os +import re +import shutil +import struct +import sys +import tempfile +import unittest + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) +SIZES = {"p": 8, "i": 4, "E": 116, "D": 56} +# the kernels the Model L frame launches, by code object (the lev tag of l_enc_values.co is the runtime's) +LAUNCHED = {"l_enc_values.co": ["input_encoder"], + "l_ops.co": ["l_linear_32_32_relu", "l_linear_32_32_f16", "l_project_gather", "l_dec_project", "l_coef"], + "l_swin.co": ["l_swin_block_c32_enc", "l_swin_block_c32_enc_nc", "l_swin_block_c32_dec", + "l_swin_block_c32_dec_nc"]} +_BUILD = None + + +def rocm(): + for d in (os.environ.get("ROCM_PATH"), r"C:\Program Files\AMD\ROCm\7.2"): + if d and os.path.isdir(d): + return d + return None + + +def build(): + global _BUILD + if _BUILD is None: + r = rocm() + if not r: + raise unittest.SkipTest("no HIP SDK (ROCM_PATH)") + sys.path.insert(0, ROOT) + from clean.l.runner import l_c32_check as C32 + d = tempfile.mkdtemp(prefix="wg_labi_") + C32.compile_kernels(d, r) + _BUILD = d + return _BUILD + + +def kinds_table(): + """{kernel name or prefix*: kinds} from the frame trace program's table.""" + with open(os.path.join(ROOT, "clean", "tests", "runtime_frame_trace.cpp"), encoding="utf-8") as f: + text = f.read() + return dict(re.findall(r'\{"([A-Za-z0-9_*]+)", "([piED]+)"\}', text)) + + +def kinds_of(table, kernel): + for name, kinds in table.items(): + if name.endswith("*") and kernel.startswith(name[:-1]) or name == kernel: + return kinds + return None + + +# ---- the AMDGPU metadata note: ELF64 notes, NT_AMDGPU_METADATA (32, owner "AMDGPU"), MessagePack ---- +def msgpack(b, i=0): + """(value, next index) of one MessagePack object (the types the metadata uses).""" + t = b[i] + if t <= 0x7F: + return t, i + 1 + if 0x80 <= t <= 0x8F or t in (0xDE, 0xDF): + n, i = (t & 0x0F, i + 1) if t <= 0x8F else (struct.unpack_from(">H" if t == 0xDE else ">I", b, i + 1)[0], + i + (3 if t == 0xDE else 5)) + out = {} + for _ in range(n): + k, i = msgpack(b, i) + v, i = msgpack(b, i) + out[k] = v + return out, i + if 0x90 <= t <= 0x9F or t in (0xDC, 0xDD): + n, i = (t & 0x0F, i + 1) if t <= 0x9F else (struct.unpack_from(">H" if t == 0xDC else ">I", b, i + 1)[0], + i + (3 if t == 0xDC else 5)) + out = [] + for _ in range(n): + v, i = msgpack(b, i) + out.append(v) + return out, i + if 0xA0 <= t <= 0xBF or t in (0xD9, 0xDA, 0xDB): + if t <= 0xBF: + n, i = t & 0x1F, i + 1 + else: + w = {0xD9: 1, 0xDA: 2, 0xDB: 4}[t] + n, i = int.from_bytes(b[i + 1:i + 1 + w], "big"), i + 1 + w + return b[i:i + n].decode(), i + n + fixed = {0xC0: (None, 1), 0xC2: (False, 1), 0xC3: (True, 1)} + if t in fixed: + return fixed[t][0], i + fixed[t][1] + ints = {0xCC: ">B", 0xCD: ">H", 0xCE: ">I", 0xCF: ">Q", 0xD0: ">b", 0xD1: ">h", 0xD2: ">i", 0xD3: ">q"} + if t in ints: + return struct.unpack_from(ints[t], b, i + 1)[0], i + 1 + struct.calcsize(ints[t]) + if t >= 0xE0: + return t - 0x100, i + 1 + raise ValueError("MessagePack type 0x%02x not handled" % t) + + +def kernels_of(path): + """{kernel name: [argument sizes]} from a code object's AMDGPU metadata.""" + with open(path, "rb") as f: + e = f.read() + assert e[:4] == b"\x7fELF" and e[4] == 2, path + shoff, shentsize, shnum = struct.unpack_from(" x1 -> x2 -> x3 -> enc0's + features -> enc1; dec1's output -> y -> y2 -> l_coef), no launch writes what it reads, every buffer holds its map + (allocation sizes), the extents (npix, tw, th, H, W), the window offsets of both parities, the render size, padded + grid and reset in the encoder's and decoder's parameters, and the history (the latent l_coef writes is what the next + frame's encoder reads). Sizes aligned, non-aligned and changing from frame to frame, and a game reset. +- A failing launch stops its frame; the next frame runs whole. +- Model L's code objects are loaded (l_enc_values.co under the tag lev, l_ops.co, l_swin.co), Model M's encoder and + decoder not. Model M runs unchanged in the build with Model L. +- Model data: only the converter's model.wgcm, checked against the preset-L map (a synthetic file and map here). A raw + folder is refused, so is a file whose shared stages are Model M's (their digests are not the map's), and a missing + file. The paths that keep dec1's map on chip, need Model M's encoder or the pack12 decoder are refused. + +Built with the host C++ compiler (clean/tests/host_cxx.py; skipped without one). + +Run (repository root): + python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_frame.py +""" +import copy +import hashlib +import os +import shutil +import subprocess +import tempfile +import unittest + +import numpy as np + +import test_l_runtime_load as LL +from clean.l.reference import oracle as L +from clean.l.reference import table as T +from clean.l.runner import l_pack as LP +from clean.runner import synthetic_model as SM +from clean.tests import host_cxx +from clean.tests import test_runtime_frame_trace as FT + +L_SOURCES = [os.path.join("clean", "l", "runtime", "l_pack.cpp"), os.path.join("clean", "l", "runtime", "l_model.cpp")] +SEED = 6 +RW, RH = 80, 48 # the maximum render size of most runs: not a multiple of 32 (F = 96 x 64) +SHARED = ("enc1", "enc2", "b0", "b1", "b2", "b3", "b4", "b5", "dec2", "dec1") +_EXE = None +_MODEL = None + + +def trace_exe(): + global _EXE + if _EXE is None: + _EXE = host_cxx.build("runtime_frame_trace_l", FT.SOURCES + L_SOURCES, ["WG_NO_MODEL_K", "WG_MODEL_L"]) + return _EXE + + +def write_l(model_dir, tensors, map_file=None): + """/L/model.wgcm in the preset-L map's stage order; with map_file, the file records that map (whose + digests it then must match), else a synthetic map of its own is written. Returns the map's path.""" + return SM.write_wgcm(os.path.join(model_dir, "L"), tensors, order=list(LL.specs()), map_file=map_file) + + +def model_dir(): + """A synthetic Model M in raw/ and a synthetic Model L as L/model.wgcm; its map under the preset-L map's file + name in maps/ (the runtime finds Model L's map by that name next to map_path).""" + global _MODEL + if _MODEL is None: + d = tempfile.mkdtemp(prefix="wg_lf_") + SM.write_raw(d, SM.synthetic_tensors(1)) + synthetic_map = write_l(d, LL.raw_model(SEED)) + os.makedirs(os.path.join(d, "maps")) + shutil.copyfile(synthetic_map, os.path.join(d, "maps", "preset-L.v1.map.json")) + _MODEL = d + return _MODEL + + +def map_path(): + return os.path.join(model_dir(), "maps", "preset-L.v1.map.json") + + +def run(options="", frames=4, build_files=(), env=None, model=None, exe=None, flags=(), rw=RW, rh=RH): + """The trace program's output lines (WG_* removed from the environment, then env applied).""" + build = tempfile.mkdtemp(prefix="wg_lfb_") + for n in build_files: + open(os.path.join(build, n), "wb").close() + e = {k: v for k, v in os.environ.items() if not k.upper().startswith("WG_")} + e.update(env or {}) + try: + r = subprocess.run([exe or trace_exe(), build, model or model_dir(), str(rw), str(rh), "120", "72", str(frames), + options, "--format=0", "--map=" + map_path(), *flags], capture_output=True, text=True, + env=e, timeout=600) + finally: + shutil.rmtree(build, ignore_errors=True) + if r.returncode != 0: + raise AssertionError("runtime_frame_trace failed (exit %d)\n%s%s" % (r.returncode, r.stdout, r.stderr)) + return r.stdout.splitlines() + + +def blob_digests(): + """{blob name: SHA-256 hex} of Model L's blobs packed in Python (clean/l/runner/l_pack.py) from the test model's + parameters: the independent names of the weight buffers (the recorder logs what create uploads, with its hash).""" + blobs = LP.pack_model_l(L.random_params(np.random.default_rng(SEED))) + return {n: hashlib.sha256(b).hexdigest() for n, b in blobs.items()} + + +L_KERNELS = ("lev/input_encoder", "l_linear_32_32_relu", "l_linear_32_32_f16", "l_swin_block_c32_enc", + "l_project_gather", "l_dec_project", "l_swin_block_c32_dec", "l_coef") + + +def expected_args(t, f, digests): + """Every Model L launch's arguments from the buffers' names (the runtime's wg_rt_buffer names after the frame) and + the weight blobs' addresses (the upload holding each blob's bytes), independently of the launches themselves: + {kernel (without _nc): [argument]}, None where the parameters E / D stand (checked field by field elsewhere).""" + n = f["names"] + w = {} + for blob, digest in digests.items(): + w[blob] = t.uploaded(digest) + if w[blob] is None: + raise AssertionError("no single upload holds blob %s" % blob) + info = f["info"] + fw, fh = cdiv(info["rw"], 32) * 32, cdiv(info["rh"], 32) * 32 + npix, tw, th = str((fh + 4) * (fw + 4)), str(fw // 2), str(fh // 2) + eo = [str(v) for v in T.block_offset(T.TABLE, "first", info["parity"])] + do = [str(v) for v in T.block_offset(T.TABLE, "second", info["parity"])] + return { + # the input encoder's values: Model M's encoder inputs (the colour as the network reads it, here the game's, + # pre-exposure 1), the state it reads, q into the wide map, the motion surfaces, the exposure pair + "lev/input_encoder": [n["input.color"], n["input.depth"], n["input.motion"], n["post.reduced"], + n["state.latent_prev"], n["state.motion_history"], n["l.wide"], n["enc0.m_near"], + n["enc0.m_far"], n["enc0.m_center"], n["exposure.pair"], w["enc0.values"], None], + "l_linear_32_32_relu": [n["l.wide"], n["l.narrow"], w["enc0.Win1"], npix], # q -> x1 + "l_linear_32_32_f16": [n["l.narrow"], n["l.wide"], w["enc0.Win2"], npix], # x1 -> x2 + "l_swin_block_c32_enc": [n["l.wide"], "0", n["l.x3"], w["enc0_block"], str(fh), str(fw)] + eo, # x2 -> x3 + "l_project_gather": [n["l.x3"], n["enc0.features"], w["enc0.project"], tw, th], # x3 -> enc1's input + "l_dec_project": [n["dec1.out"], n["l.narrow"], w["dec0.project"], tw, th], # dec1's output -> y + "l_swin_block_c32_dec": [n["l.narrow"], "0", n["l.wide"], w["dec0_block"], str(fh), str(fw)] + do, # -> y2 + "l_coef": [n["l.wide"], n["dec0.A"], n["dec0.B"], n["dec0.C"], n["dec0.latent"], w["dec0.coef"], None], + } + + +def check_roles(t, f, digests): + """Raises AssertionError naming every Model L launch argument that is not the expected buffer or blob.""" + exp = expected_args(t, f, digests) + seen, bad = set(), [] + for ln in f["launches"]: + k = ln.kernel.replace("_nc", "") + if k not in exp: + continue + seen.add(k) + want = exp[k] + if len(ln.args) != len(want): + bad.append("%s: %d arguments, expected %d" % (k, len(ln.args), len(want))) + continue + bad += ["%s argument %d: %s, expected %s" % (k, i, a, e) for i, (a, e) in enumerate(zip(ln.args, want)) + if e is not None and a != e] + missing = [k for k in L_KERNELS if k not in seen] + if bad or missing: + raise AssertionError("; ".join(bad + ["not launched: " + k for k in missing])) + + +def cdiv(a, b): + return -(-a // b) + + +def nc_flags(): + blobs = LP.pack_model_l(L.random_params(np.random.default_rng(SEED))) + return [bool(int.from_bytes(blobs[n][:4], "little") & 1) for n in ("enc0_block", "dec0_block")] + + +def expected_l(m_frame, parity, fh, fw): + """Model M's launches of one frame with Model L's steps 2 and 4 (clean/l/FORMATS.md, "Kernels" and "Launch order + of one frame"): kernel, grid, block.""" + nc = nc_flags() + npix, tokens = (fh + 4) * (fw + 4), (fh // 2) * (fw // 2) + + def block(kind, offsets, i): + ox, oy = T.block_offset(T.TABLE, offsets, parity) + nwin = cdiv(fw + ox, 4) * cdiv(fh + oy, 4) + return ("l_swin_block_c32_%s%s" % (kind, "_nc" if nc[i] else ""), cdiv(nwin, 8), 256) + + out = [] + for k, g, b in m_frame: + if k == "input_encoder": + out += [("lev/input_encoder", cdiv(((fh + 4) // 2) * ((fw + 4) // 2), 128), 256), + ("l_linear_32_32_relu", cdiv(npix, 128), 256), ("l_linear_32_32_f16", cdiv(npix, 128), 256), + block("enc", "first", 0), ("l_project_gather", cdiv(tokens, 128), 256)] + elif k == "coef_decoder": + out += [("l_dec_project", cdiv(tokens, 128), 256), block("dec", "second", 1), + ("l_coef", cdiv(tokens, 128), 256)] + else: + out.append((k, g, b)) + return out + + +class Frame(unittest.TestCase): + def check(self, build_files=(), m_env=None): + fh, fw = cdiv(RH, 32) * 32, cdiv(RW, 32) * 32 + cm, lm, fm, km = FT.parse(run(build_files=build_files, env=m_env)) + cl, ll, fl, kl = FT.parse(run("model=L", build_files=build_files)) + self.assertEqual(cm, "create ok model M") + self.assertEqual(cl, "create ok model L") + self.assertEqual(len(fl), 4) + for i, (m, l_) in enumerate(zip(fm, fl)): + names = [k for k, _, _ in m] + self.assertIn("input_encoder", names) + self.assertIn("coef_decoder", names) # Model M's base decoder: the one Model L's l_coef stands in for + self.assertEqual(l_, expected_l(m, i & 1, fh, fw), "frame %d" % i) + l_mods = ["l_enc_values.co as lev", "l_ops.co", "l_swin.co"] + self.assertEqual([x for x in ll if x not in l_mods], [x for x in lm if x not in (km["enc0"], km["dec0"])]) + for want in l_mods: + self.assertIn(want, ll) + self.assertEqual(kl.get("model"), "L") + self.assertEqual(kl.get("post_pack"), "none") + self.assertEqual(kl.get("model_map"), "same") # read from model.wgcm, checked against its map + return fl + + def test_empty_build(self): + self.check() + + def test_shipping_build(self): + # l_coef writes Model M's base coefficient format: Model L runs without pack12 (and so without the fused pack12 + # dec1 + dec0), compared with Model M without it (WG_POST_PACK=none) + fl = self.check(FT.SHIPPING, {"WG_POST_PACK": "none"}) + names = [k for k, _, _ in fl[0]] + self.assertTrue(all(not n.startswith(("d0p12/", "d1p12/")) and "_p12" not in n for n in names), names) + self.assertTrue(any(n.startswith("w64/") for n in names)) # the wave64 post (not a pack12 kernel) is kept + + def test_model_m_unchanged(self): + """Model M in the build with Model L: the same trace as in the build without it (clean/tests).""" + for files in ((), FT.SHIPPING): + self.assertEqual(run(frames=2, build_files=files), run(frames=2, build_files=files, exe=FT.trace_exe())) + + +class Arguments(unittest.TestCase): + """The recorder's view of every Model L launch (clean/l/FORMATS.md, "Maps" and "Kernels").""" + + def check_frame(self, t, f, roles=True): + info, ls = f["info"], f["launches"] + rw, rh = info["rw"], info["rh"] + fw, fh = cdiv(rw, 32) * 32, cdiv(rh, 32) * 32 + npix, tw, th = (fh + 4) * (fw + 4), fw // 2, fh // 2 + k = {ln.kernel.replace("_nc", ""): (i, ln) for i, ln in enumerate(ls)} + _, lev = k["lev/input_encoder"] + _, relu = k["l_linear_32_32_relu"] + _, f16 = k["l_linear_32_32_f16"] + _, enc = k["l_swin_block_c32_enc"] + gi, gather = k["l_project_gather"] + di, dproj = k["l_dec_project"] + _, dec = k["l_swin_block_c32_dec"] + _, coef = k["l_coef"] + q, x1, x2, x3, feat = lev.args[6], relu.args[1], f16.args[1], enc.args[2], gather.args[1] + # the chain of maps (no launch reads what it writes) + self.assertEqual(relu.args[0], q) + self.assertEqual(f16.args[0], x1) + self.assertEqual(enc.args[0], x2) + self.assertEqual(enc.args[1], "0") # xadd: none + self.assertEqual(gather.args[0], x3) + self.assertTrue(ls[gi + 1].kernel.startswith("swin_block_c64")) + self.assertEqual(ls[gi + 1].args[0], feat) # enc1 reads Model L's features + dec1 = [ln for ln in ls[:di] if ln.kernel.startswith("swin_block_c64")][-1] + y, y2 = dproj.args[1], dec.args[2] + self.assertEqual(dproj.args[0], dec1.args[2]) # the decoder reads dec1's output + self.assertEqual(dec.args[0], y) + self.assertEqual(dec.args[1], "0") + self.assertEqual(coef.args[0], y2) + for a, b in ((q, x1), (x1, x2), (x2, x3), (x3, feat), (dec1.args[2], y), (y, y2)): + self.assertNotEqual(a, b) + # every buffer holds its map (FORMATS.md "Maps") + maps = ((q, 32 * npix), (x1, 32 * npix), (x2, 64 * npix), (x3, 32 * fh * fw), (feat, 64 * tw * th), + (y, 32 * fh * fw), (y2, 64 * fh * fw)) + coefs = ((coef.args[1], 4 * rw * rh), (coef.args[2], 4 * rw * rh), (coef.args[3], 8 * rw * rh), + (coef.args[4], 4 * rw * rh)) # A, B, C, the new latent: Model M's formats + for buf, need in maps + coefs: + self.assertGreaterEqual(t.extent(buf), need, buf) + # extents, offsets, parameters + self.assertEqual(relu.args[3], str(npix)) + self.assertEqual(f16.args[3], str(npix)) + self.assertEqual(gather.args[3:5], [str(tw), str(th)]) + self.assertEqual(dproj.args[3:5], [str(tw), str(th)]) + for blk, kind in ((enc, "first"), (dec, "second")): + ox, oy = T.block_offset(T.TABLE, kind, info["parity"]) + self.assertEqual(blk.args[4:8], [str(fh), str(fw), str(ox), str(oy)]) + e, d = lev.params(12), coef.params(6) + self.assertEqual(e[:4], [rw, rh, fw, fh]) + self.assertEqual(e[5], info["reset"]) + self.assertEqual(d, [rw, rh, fw, fh, info["reset"]]) + self.assertTrue(lev.args[11].startswith("b")) # the values blob: an allocation + if roles: # every argument against the named buffers and blobs + check_roles(t, f, blob_digests()) + return lev, coef + + def test_sizes_reset_history(self): + sizes = "--sizes=96x96,96x96,80x48,80x48,80x48,64x64" # aligned, non-aligned, a smaller aligned size + t = FT.Trace(run("model=L", frames=6, rw=96, rh=96, flags=(sizes, "--reset=4"))) + self.assertEqual(t.create, "create ok model L") + # first frame, a size change, the game's reset, a size change + self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 1, 0, 1, 1]) + prev = None + for f in t.frames: + self.assertIsNone(f["error"]) + lev, coef = self.check_frame(t, f) + if prev: # the latent l_coef wrote and the near motion the encoder wrote: what this frame's encoder reads + self.assertEqual(lev.args[4], prev[1].args[4]) + self.assertEqual(lev.args[5], prev[0].args[7]) + prev = (lev, coef) + + def test_steady_frames(self): + t = FT.Trace(run("model=L", frames=4)) + self.assertEqual([f["info"]["reset"] for f in t.frames], [1, 0, 0, 0]) + self.assertEqual(len({f["info"]["parity"] for f in t.frames}), 2) + prev = None + for f in t.frames: + lev, coef = self.check_frame(t, f) + if prev: + self.assertEqual(lev.args[4], prev[1].args[4]) + self.assertNotEqual(lev.args[4], prev[0].args[4]) # the two latent buffers swap + prev = (lev, coef) + + def test_failed_launch_stops_the_frame(self): + t = FT.Trace(run("model=L", frames=3, flags=("--fail=l_coef@1",))) + ok0, bad, ok2 = t.frames + self.assertEqual(bad["failed"], "l_coef") + self.assertIn("injected launch failure: l_coef", bad["error"]) + names = [ln.kernel for ln in bad["launches"]] + self.assertEqual(names, [ln.kernel for ln in ok0["launches"]][:len(names)]) + self.assertTrue(names[-1].startswith("l_swin_block_c32_dec")) # stopped right before l_coef + self.assertIsNone(ok2["error"]) + self.check_frame(t, ok2) + + +class Roles(unittest.TestCase): + """Buffer identities: each Model L launch's weight blob, inputs and outputs are the named ones, and a mix-up of two + of them fails even where the map chain, sizes and argument kinds stay as they were.""" + + MIXUPS = ( # (kernel, argument, kernel, argument) swapped in a recorded frame + ("l_linear_32_32_relu", 2, "l_linear_32_32_f16", 2), # enc0.Win1 <-> enc0.Win2 + ("l_swin_block_c32_enc", 3, "l_swin_block_c32_dec", 3), # enc0_block <-> dec0_block + ("l_project_gather", 2, "l_dec_project", 2), # enc0.project <-> dec0.project + ("lev/input_encoder", 11, "l_coef", 5), # enc0.values <-> dec0.coef + ("l_coef", 1, "l_coef", 2), # A <-> B + ("l_coef", 3, "l_coef", 4), # C <-> the new latent + ("lev/input_encoder", 0, "lev/input_encoder", 1), # colour <-> depth + ("lev/input_encoder", 4, "lev/input_encoder", 5), # previous latent <-> motion history + ("lev/input_encoder", 7, "lev/input_encoder", 8), # m_near <-> m_far + ("l_linear_32_32_relu", 0, "l_linear_32_32_relu", 1), # q's map <-> x1's map + ) + + @classmethod + def setUpClass(cls): + cls.trace = FT.Trace(run("model=L", frames=3, rw=96, rh=64, flags=("--sizes=96x64,80x48,80x48",))) + + def test_every_argument_named(self): + t, digests = self.trace, blob_digests() + addrs = [t.uploaded(d) for d in digests.values()] + self.assertNotIn(None, addrs) + self.assertEqual(len(set(addrs)), len(addrs)) # eight distinct blobs, each in its own allocation + for f in t.frames: + self.assertEqual(len(f["names"]) >= 30, True, sorted(f["names"])) + check_roles(t, f, digests) + + def test_mixups_fail(self): + digests = blob_digests() + for ka, ia, kb, ib in self.MIXUPS: + with self.subTest(mixup=(ka, ia, kb, ib)): + t = copy.deepcopy(self.trace) + f = t.frames[1] + ls = {ln.kernel.replace("_nc", ""): ln for ln in f["launches"]} + a, b = ls[ka], ls[kb] + a.args[ia], b.args[ib] = b.args[ib], a.args[ia] + with self.assertRaises(AssertionError): + check_roles(t, f, digests) + + def test_swapped_weights_in_the_runtime(self): + """The review's example in the runtime itself: enc0.Win1 and enc0.Win2 swapped in its source. The map chain, + sizes, grids and argument kinds stay as they were (the frame checks without the roles pass), and the role + check fails on exactly the two linears' weights.""" + with open(os.path.join(host_cxx.ROOT, "clean", "runtime", "runtime.cpp"), encoding="utf-8") as fh: + src = fh.read() + right = 'w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"]' + self.assertEqual(src.count(right), 1) + tmp = tempfile.mkdtemp(prefix="wg_lmut_") + mutant = os.path.join(tmp, "runtime.cpp") + with open(mutant, "w", encoding="utf-8") as fh: + fh.write(src.replace(right, 'w1 = wdev["enc0.Win2"], w2 = wdev["enc0.Win1"]')) + sources = [mutant if s.replace("\\", "/").endswith("clean/runtime/runtime.cpp") else s + for s in FT.SOURCES] + L_SOURCES + exe = host_cxx.build("runtime_frame_trace_lmut", sources, ["WG_NO_MODEL_K", "WG_MODEL_L"], + include_dirs=[os.path.join(host_cxx.ROOT, "clean", "runtime")]) + t = FT.Trace(run("model=L", frames=2, exe=exe)) + digests = blob_digests() + for f in t.frames: + Arguments.check_frame(self, t, f, roles=False) + with self.assertRaises(AssertionError) as cm: + check_roles(t, f, digests) + msg = str(cm.exception) + self.assertIn("l_linear_32_32_relu argument 2", msg) + self.assertIn("l_linear_32_32_f16 argument 2", msg) + self.assertEqual(msg.count("argument"), 2, msg) + + +class Refusals(unittest.TestCase): + def test_paths(self): + self.assertTrue(run("model=L", env={"WG_DEC1_DEC0": "fused"})[0].startswith( + "ERR create: Model L: WG_DEC1_DEC0=fused")) + self.assertTrue(run("model=L", env={"WG_RECON_SIZE": "output"})[0].startswith( + "ERR create: Model L: recon=output")) + self.assertTrue(run("model=L;motion_outres=original", build_files=("encoder_mo.co",))[0] + .startswith("ERR create: Model L: motion_outres=original")) + self.assertTrue(run("model=L", build_files=FT.SHIPPING, env={"WG_POST_PACK": "p12"})[0].startswith( + "ERR create: Model L: post_pack=p12")) + + def test_model_data(self): + d = tempfile.mkdtemp(prefix="wg_lf_") + try: + SM.write_raw(d, SM.synthetic_tensors(1)) + # no Model L file + self.assertEqual(run("model=L", model=d), ["ERR create: model data missing: " + d + "/L/model.wgcm"]) + # raw tensors, even complete and of the right shapes: refused (nothing shows they are Model L's) + LL.write_raw(os.path.join(d, "L"), LL.raw_model(SEED)) + self.assertTrue(run("model=L", model=d, flags=("--format=1",))[0].startswith( + "ERR create: Model L reads only the converter's model.wgcm")) + # Model L's own stages with Model M's shared stages (same names and shapes): their digests are not the + # map's, refused + mixed = LL.raw_model(SEED) + m = SM.synthetic_tensors(1) + for st in SHARED: + mixed[st] = m[st] + write_l(d, mixed, map_file=map_path()) + got = run("model=L", model=d) + self.assertEqual(len(got), 1) + self.assertTrue(got[0].startswith("ERR create: model data damaged: "), got[0]) + self.assertIn("does not match the map", got[0]) + # the same file with Model L's shared stages loads + write_l(d, LL.raw_model(SEED), map_file=map_path()) + self.assertEqual(run("model=L", model=d, frames=1)[0], "create ok model L") + finally: + shutil.rmtree(d, ignore_errors=True) + + +if __name__ == "__main__": + unittest.main() diff --git a/clean/runtime/model_select.h b/clean/runtime/model_select.h index fcbe7389..96395c10 100644 --- a/clean/runtime/model_select.h +++ b/clean/runtime/model_select.h @@ -5,8 +5,8 @@ // surroundings, so the callers do not compare against one model by hand. Header only: every build that compiles // clean/k/runtime/model_k.cpp or clean/runtime/no_model_k.cpp gets it without a new source file. // -// Model L has a row so that model=L is recognised and refused by name ("not part of this build yet") until its -// network exists; the runtime never selects it today. +// Model L has a row so that model=L is recognised. The runtime runs it when its sources are compiled in (define +// WG_MODEL_L, model_l_built()); otherwise model=L is refused by name ("not part of this build yet"). #pragma once #include @@ -67,7 +67,16 @@ inline bool last_model_option(const std::string& text, ModelId* model, std::stri return true; } -// The refusal of model=L until its network is part of the runtime (one place for both builds). +// Model L is part of this runtime library: its sources (clean/l/runtime) compiled with the define WG_MODEL_L. +inline bool model_l_built() { +#ifdef WG_MODEL_L + return true; +#else + return false; +#endif +} + +// The refusal of model=L when it is not part of the runtime library (one place for both builds). inline std::string model_not_built_yet(ModelId id) { return std::string("Model ") + model_info(id).name + " is not part of this build yet (runtime option model=" + model_info(id).name + "); remove the option to use Model M"; diff --git a/clean/runtime/no_model_k.cpp b/clean/runtime/no_model_k.cpp index ded375a9..86e3feb5 100644 --- a/clean/runtime/no_model_k.cpp +++ b/clean/runtime/no_model_k.cpp @@ -22,17 +22,18 @@ bool model_from_options(const WgRtConfig& c, ModelId* model, std::string* err) { ModelId last = ModelId::M; std::string bad; if (!last_model_option(text, &last, &bad)) { - *err = "unknown model \"" + bad + "\" in the runtime options (known: M)"; + *err = "unknown model \"" + bad + "\" in the runtime options (known: M" + (model_l_built() ? ", L)" : ")"); return false; } if (last == ModelId::K) { *err = "Model K is not part of this build (runtime option model=K); remove the option to use Model M"; return false; } - if (last == ModelId::L) { // #89: recognised, not part of the runtime yet + if (last == ModelId::L && !model_l_built()) { // #89: recognised; runs when its sources are compiled in *err = model_not_built_yet(ModelId::L); return false; } + *model = last; return true; } diff --git a/clean/runtime/runtime.cpp b/clean/runtime/runtime.cpp index 6e3f007d..d8b7d8ea 100644 --- a/clean/runtime/runtime.cpp +++ b/clean/runtime/runtime.cpp @@ -16,6 +16,10 @@ #ifndef WG_NO_MODEL_K // build.py --no-model-k: a build without model K (model=K fails at create, no_model_k.cpp) #include "../k/runtime/runtime_k.h" #endif +#ifdef WG_MODEL_L // Model L (#89; model_select.h model_l_built): its model data and weight packer (clean/l/runtime) +#include "../l/runtime/l_model.h" +#include "../l/runtime/l_pack.h" +#endif #include "device.h" #include "embedded.h" #include "fpconv.h" @@ -230,6 +234,18 @@ struct Runtime { std::unique_ptr dev; #ifndef WG_NO_MODEL_K std::unique_ptr k; // model K (runtime option model=K): the whole frame runs there (clean/k/runtime) +#endif + // Model L (runtime option model=L, builds with WG_MODEL_L): Model M's frame with Model L's encoder in step 2 and its + // decoder in step 4 (clean/l/FORMATS.md); the ten Swin stages, exposure and post are Model M's. + bool model_l = false; +#ifdef WG_MODEL_L + // The maps of clean/l/FORMATS.md in three buffers (no kernel works in place): + // l_wide [fh + 4][fw + 4] x 64 bytes: q (FP8 halo, 32 bytes a pixel), then x2 (FP16 halo; q is dead after the + // first linear), then the decoder's y2 (FP16, [fh][fw]) + // l_narrow [fh + 4][fw + 4] x 32 bytes: x1 (FP8 halo), then the decoder's y (FP8, [fh][fw]) + // l_x3 [fh][fw] x 32 bytes: the encoder block's output + DevPtr l_wide = 0, l_narrow = 0, l_x3 = 0; + bool l_nc[2] = {false, false}; // enc0_block, dec0_block: clamp-free block kernel (bit 0 of the blob's first word) #endif DRow drow; int block_threads[5] = {256, 256, 256, 256, 256}, gemm_threads = 256, nt = 2, wpw[3] = {1, 1, 1}; // = variants.DEFAULT_GEOMETRY @@ -494,8 +510,11 @@ struct Runtime { #ifndef WG_NO_MODEL_K if (model_id == ModelId::K) return create_k(c, std::move(given)); #else - if (model_id != ModelId::M) return fail("Model K is not part of this build"); // (model_from_options says so first) + if (model_id == ModelId::K) return fail("Model K is not part of this build"); // (model_from_options says so first) #endif + model_l = model_id == ModelId::L; // (model_from_options accepts it only in builds with WG_MODEL_L) + if (model_l && (c.backend == 1 || c.backend == 2)) + return fail("Model L runs on the HIP backend (backend 0) only for now"); if (c.max_rw <= 0 || c.max_rh <= 0 || c.ow <= 0 || c.oh <= 0) return fail("invalid render or output size"); // post_core's fallback colour read uses a 32-bit byte offset (mul24, kernels/post.hip): the render colour image // must stay within 4 GB (16 rw rh <= 2^32, 16 rw < 2^24; any D3D12 texture, at most 16384 x 16384, does) @@ -521,6 +540,13 @@ struct Runtime { if (c.wpw96) wpw[1] = c.wpw96; if (c.wpw128) wpw[2] = c.wpw128; if (!choose_kernels(c)) return false; + if (model_l && recon_output) return fail("Model L: recon=output (D = O) is Model M's experimental mode"); + if (model_l && enc0_co != "encoder.co") + return fail("Model L: enc0 variants are Model M's encoder; Model L's runs from l_enc_values.co"); + // Model L's coefficient kernel (l_coef) writes Model M's coef_decoder formats, not pack12: the _p12 post kernels + // are not for it (choose_kernels does not choose pack12 by default for Model L; asked for, it is refused here) + if (model_l && p12) return fail("Model L: post_pack=p12 (the pack12 decoder) is Model M's; Model L's l_coef " + "writes the base coefficient format"); bool any = false; for (int i = 0; i < 16; ++i) any |= c.d_row[i] != 0; drow = drow_gfx12(); @@ -535,12 +561,23 @@ struct Runtime { if (!c.model_dir) return fail("no model directory"); const std::string wgcm = std::string(c.model_dir) + "/model.wgcm"; bool use_wgcm = c.model_format == 2; - if (c.model_format == 0) { + if (c.model_format == 0 && !model_l) { std::FILE* fp = std::fopen(wgcm.c_str(), "rb"); use_wgcm = fp != nullptr; if (fp) std::fclose(fp); } - if (use_wgcm) { + if (model_l) { +#ifdef WG_MODEL_L + // Model L: the converter's model.wgcm in its folder under the model directory (model_select.h), checked against + // the preset-L map (whole-file hash, every stage's keyed digest, names, types, shapes, values). Raw tensors are + // refused: nothing in them shows that they are Model L's (a folder with Model M's shared stages would load). + if (c.model_format == 1) + return fail("Model L reads only the converter's model.wgcm (raw tensors cannot be verified as Model L's)"); + const std::string dir = std::string(c.model_dir) + "/" + model_info(ModelId::L).model_subdir; + if (!l::load_wgcm_model(dir + "/model.wgcm", l_map_path(c), &model, &e, &model_other_map)) return fail(e); + use_wgcm = true; +#endif + } else if (use_wgcm) { if (!c.map_path) return fail("model.wgcm needs the converter map (map_path)"); if (!load_wgcm_model(wgcm, c.map_path, &model, &e, &model_other_map)) return fail(e); } else if (!load_raw_model(c.model_dir, &model, &e)) { @@ -550,9 +587,21 @@ struct Runtime { if (c.swin_force_clamp) force_clamp = true; if (c.swin_fuse) fuse = true; for (const std::string& s : model_stages()) { - if (s == "post") continue; + if (s == "post" || (model_l && (s == "enc0" || s == "dec0"))) continue; // Model L's: below if (!pack_stage_blob(s, model[s], drow, &blobs[s], &e)) return fail(e); } +#ifdef WG_MODEL_L + if (model_l) { // enc0.values, enc0.Win1, enc0.Win2, enc0_block, enc0.project, dec0.project, dec0_block, dec0.coef + std::map> lb; + if (!l::pack_model(model, drow, &lb, &e)) return fail(e); + for (auto& kv : lb) blobs[kv.first] = std::move(kv.second); + for (int i = 0; i < 2; ++i) { + uint32_t flags = 0; + std::memcpy(&flags, blobs[i ? "dec0_block" : "enc0_block"].data(), 4); + l_nc[i] = (flags & 1u) != 0; + } + } +#endif // kernels/swin.hip: the first word of a Swin stage's packed weights says whether the clamp-free block kernel // (`_nc`, identical results) may be used. for (const char* s : kSwinOrder) { @@ -590,19 +639,32 @@ struct Runtime { } if (!dev) return fail(e); for (const std::string& co : {swin_co, enc0_co, dec0_co, exposure_co, std::string("post.co"), - std::string("frame.co")}) + std::string("frame.co")}) { + if (model_l && (co == enc0_co || co == dec0_co)) continue; // Model L's encoder and decoder: below if (!dev->load_module(build_dir + "/" + co)) return fail(dev->err() + (co.find("__") != std::string::npos ? " (variant code objects: clean/runner/build.py --variants / --perf-variants)" : "")); + } + // Model L's code objects (clean/l/FORMATS.md): the encoder's values under the tag lev (lev/input_encoder, next to + // Model M's input_encoder), the linears / projection / channel copy and the width-32 block untagged + if (model_l && (!dev->load_module_tagged(build_dir + "/l_enc_values.co", "lev") || + !dev->load_module(build_dir + "/l_ops.co") || !dev->load_module(build_dir + "/l_swin.co"))) + return fail(dev->err() + " (Model L's code objects l_enc_values.co, l_ops.co, l_swin.co)"); if (!stage_variants(c)) return false; if (!runtime_options(c)) return false; + if (model_l) { + // Model L reads dec1's output map (dec1.out) in its decoder front: not the paths that keep it on chip + if (motion_outres == 1) return fail("Model L: motion_outres=original needs Model M's encoder (encoder_mo.co)"); + d1d0_p12_opt = false; + } if (!mid_setup(c)) return false; if (!bneck_setup(c)) return false; // D = O (fast mode, recon=output) runs without pack12, so dec1 + dec0 fused is the default there (bit-identical; // measured -0.027 ms per 1440p-Q frame); the D = 1.5 x path keeps pack12 and the separate kernels. if (choose(c.dec1_dec0, "WG_DEC1_DEC0", recon_output && !p12 ? "fused" : "separate") == "fused") { + if (model_l) return fail("Model L: WG_DEC1_DEC0=fused keeps dec1's map on chip; its decoder front reads it"); if (stage_geo.count("dec1")) return fail("WG_DEC1_DEC0=fused together with a per-stage variant of dec1"); if (block_threads[1] != 256 || wpw[0] != 1 || (block_wpg[1] > 0 && block_wpg[1] != 8)) return fail("WG_DEC1_DEC0=fused needs dec1 at 256 lanes with one window per wave (the base geometry)"); @@ -756,6 +818,17 @@ struct Runtime { } #endif + // Model L's converter map: map_path itself when it names that map's file, else that file next to map_path (or in + // the build folder without one): the maps are shipped side by side next to the runtime library + // (clean/runner/build.py). + std::string l_map_path(const WgRtConfig& c) const { + const std::string file = model_info(ModelId::L).map_file; + const std::string given = c.map_path ? c.map_path : ""; + const size_t slash = given.find_last_of("\\/"); + if (!given.empty() && given.substr(slash == std::string::npos ? 0 : slash + 1) == file) return given; + return (slash == std::string::npos ? build_dir : given.substr(0, slash)) + "/" + file; + } + // Native route selection uses manifest identities, never HIP .co file presence. The fused pack12 decoder supplies // the selected path's pack12 capability; its standalone decoder fallback is still rejected at launch if reached. bool module_available(const std::string& path) const { @@ -799,7 +872,7 @@ struct Runtime { { const bool post_given = (c.post_kernel && *c.post_kernel) || choose(nullptr, "WG_POST_KERNEL", "") != ""; const bool dec0_given = (c.dec0_variant && *c.dec0_variant) || choose(nullptr, "WG_DEC0_VARIANT", "") != ""; - p12_default = module_available(p12_path) && !post_given && !dec0_given && !recon_output && + p12_default = !model_l && module_available(p12_path) && !post_given && !dec0_given && !recon_output && choose(c.dec1_dec0, "WG_DEC1_DEC0", "separate") != "fused"; } const std::string pack = choose(c.post_pack, "WG_POST_PACK", p12_default ? "p12" : "none"); @@ -1258,6 +1331,8 @@ struct Runtime { kernels_text += std::string(" output_taps=") + (has_taps ? "model" : "none"); kernels_text += std::string(" model_map=") + (model_source == "raw" ? "none" : model_other_map ? "other" : "same"); if (native_direct_io) kernels_text += " native_io=direct_textures"; + // Model L: its encoder and decoder (enc0= / dec0= above name Model M's, not loaded for Model L) + if (model_l) kernels_text += " model=L l_kernels=l_enc_values.co,l_ops.co,l_swin.co"; return kernels_text.c_str(); } @@ -1286,6 +1361,12 @@ struct Runtime { ok = alloc(&xd_sync, 4 * nxd) && (dev->memset(xd_sync, 0, 4 * nxd) || dfail()); } if (ok && uses_prep(post_kernel)) ok = alloc(&prep, R * 32); +#ifdef WG_MODEL_L + if (ok && model_l) { + const size_t halo = (FH + 4) * (FW + 4); + ok = alloc(&l_wide, halo * 64) && alloc(&l_narrow, halo * 32) && alloc(&l_x3, FH * FW * 32); + } +#endif return ok; } @@ -1370,6 +1451,54 @@ struct Runtime { return true; } +#ifdef WG_MODEL_L + // ---- Model L (clean/l/FORMATS.md: maps, blobs, kernels, launch order) ---- + // Window offsets (ox, oy) of Model L's two width-32 blocks for frame parity f: the default table's window_offsets + // (clean/l/reference/table.py), "first" for the encoder's block, "second" for the decoder's. + static void l_offsets(bool decoder, int f, int* ox, int* oy) { + static const int kOff[2][2][2] = {{{2, 2}, {0, 2}}, {{0, 0}, {2, 0}}}; + *ox = kOff[decoder][f & 1][0]; + *oy = kOff[decoder][f & 1][1]; + } + + // One width-32 Swin block on the fh x fw grid (l_swin.co): "_enc" reads the FP16 halo map x2 and writes FP8, "_dec" + // reads FP8 and writes FP16. + bool l_block(const char* kind, DevPtr x, DevPtr out, DevPtr w, bool nc, int H, int W, bool decoder, int f) { + DevPtr none = 0; + int ox = 0, oy = 0; + l_offsets(decoder, f, &ox, &oy); + const int nwin = ((W + ox + 3) / 4) * ((H + oy + 3) / 4); + const std::string k = std::string("l_swin_block_c32_") + kind + (nc ? "_nc" : ""); + return launch(k.c_str(), cdiv(nwin, 8), 256, {&x, &none, &out, &w, &H, &W, &ox, &oy}); + } + + // Step 2 for Model L: the 20 values and their dithered FP8 for every pixel of the halo map (Model M's input encoder + // arguments, q in place of the features; the motion surfaces as Model M), the two per-pixel linears, the encoder + // block, and the gather of each token's 2 x 2 pixels into the 64 features enc1 reads (feat0). + bool l_encoder(EncParams& p, DevPtr cnet, DevPtr din, DevPtr min_, int cur, int nxt, int fh, int fw, int f) { + DevPtr wv = wdev["enc0.values"], w1 = wdev["enc0.Win1"], w2 = wdev["enc0.Win2"], wp = wdev["enc0.project"]; + int npix = (fh + 4) * (fw + 4), tw = fw / 2, th = fh / 2; + return launch("lev/input_encoder", cdiv(((fh + 4) / 2) * ((fw + 4) / 2), 128), 256, + {&cnet, &din, &min_, &reduced, &latent[cur], &mnear[cur], &l_wide, &mnear[nxt], &mfar, &mcenter, + &expo, &wv, &p}) && + launch("l_linear_32_32_relu", cdiv(npix, 128), 256, {&l_wide, &l_narrow, &w1, &npix}) && + launch("l_linear_32_32_f16", cdiv(npix, 128), 256, {&l_narrow, &l_wide, &w2, &npix}) && + l_block("enc", l_wide, l_x3, wdev["enc0_block"], l_nc[0], fh, fw, false, f) && + launch("l_project_gather", cdiv(tw * th, 128), 256, {&l_x3, &feat0, &wp, &tw, &th}); + } + + // Step 4 for Model L: the projection of dec1's output to 32 values per pixel (y), the decoder block (y2, FP16), and + // Model L's per-pixel functions into A, B, C and the new latent (Model M's buffers and formats; no motion, previous + // latent, jitter or reset is read). + bool l_decoder(DecParams& p, int nxt, int fh, int fw, int f) { + DevPtr wd = wdev["dec0.project"], wc = wdev["dec0.coef"]; + int tw = fw / 2, th = fh / 2; + return launch("l_dec_project", cdiv(tw * th, 128), 256, {&out1, &l_narrow, &wd, &tw, &th}) && + l_block("dec", l_narrow, l_wide, wdev["dec0_block"], l_nc[1], fh, fw, true, f) && + launch("l_coef", cdiv(tw * th, 128), 256, {&l_wide, &A, &B, &C, &latent[nxt], &wc, &p}); + } +#endif + // The arguments of a kernel that runs the 5-tap output filter (post_filter*, post_tf*, post_fused*): with // output_filter=original (post_w64__orig.co, kernels/post.hip WG_LZ_KARGS) the five weights follow as u32 FP16 bit // patterns; the other builds take no such arguments. @@ -1766,7 +1895,11 @@ struct Runtime { p.irh = (float)(1.0 / rh); DevPtr w = wdev["enc0"]; const int ntok = (fh / 2) * (fw / 2); - if (mo) { // kernels/encoder_mo.hip: one motion texel per render pixel, no depth neighbourhood + if (model_l) { +#ifdef WG_MODEL_L + if (!l_encoder(p, cnet, din, min_, cur, nxt, fh, fw, f)) return false; +#endif + } else if (mo) { // kernels/encoder_mo.hip: one motion texel per render pixel, no depth neighbourhood p.sx = mop.sx; p.sy = mop.sy; p.bx = mop.bx; @@ -1891,7 +2024,11 @@ struct Runtime { p.pjy = jpy; DevPtr w = wdev["dec0"]; p12_frame = p12 && recon2_supported(rw, rh, dw, dh); - if (p12_frame && fused_p12) { // dec1's block with dec0 (pack12) as its epilogue + if (model_l) { // (no pack12 for Model L: refused at create) +#ifdef WG_MODEL_L + if (!l_decoder(p, nxt, fh, fw, f)) return false; +#endif + } else if (p12_frame && fused_p12) { // dec1's block with dec0 (pack12) as its epilogue DevPtr w1 = wdev["dec1"]; int ox = off[9][0], oy = off[9][1], h2 = H2, w2 = W2; const int nwin = ((W2 + ox + 3) / 4) * ((H2 + oy + 3) / 4); @@ -2197,6 +2334,14 @@ struct Runtime { {"input.depth", {depth, R * 4}}, {"input.motion", {motion, R * 8}}, }; for (int i = 0; i < 6; ++i) m["b" + std::to_string(i) + ".out"] = {bout[i], t8 * 128}; +#ifdef WG_MODEL_L + if (model_l) { // Model L's maps (FORMATS.md "Maps"; the frame reuses them: q / x2 / y2, x1 / y, x3) + const size_t halo = (size_t)(fw + 4) * (fh + 4); + m["l.wide"] = {l_wide, halo * 64}; + m["l.narrow"] = {l_narrow, halo * 32}; + m["l.x3"] = {l_x3, (size_t)fw * fh * 32}; + } +#endif auto it = m.find(nm); if (it == m.end()) return fail("unknown buffer name " + nm); *p = it->second.p; @@ -2239,9 +2384,11 @@ const char* const* runtime_timing_labels(Runtime* rt) { #endif } #ifdef WG_NO_MODEL_K -const char* runtime_model(Runtime*) { return model_info(ModelId::M).name; } +const char* runtime_model(Runtime* rt) { return model_info(rt->model_l ? ModelId::L : ModelId::M).name; } #else -const char* runtime_model(Runtime* rt) { return model_info(rt->k ? ModelId::K : ModelId::M).name; } +const char* runtime_model(Runtime* rt) { + return model_info(rt->k ? ModelId::K : rt->model_l ? ModelId::L : ModelId::M).name; +} #endif void runtime_recon_size(Runtime* rt, int* dw, int* dh) { *dw = rt->dw; diff --git a/clean/tests/host_cxx.py b/clean/tests/host_cxx.py index 55e70204..227bfa03 100644 --- a/clean/tests/host_cxx.py +++ b/clean/tests/host_cxx.py @@ -21,22 +21,24 @@ def _vcvars64(): return vcvars if r.returncode == 0 and r.stdout.strip() and os.path.exists(vcvars) else None -def build(name, sources, defines=()): - """sources: paths relative to the repository root; defines: NAME or NAME=VALUE. Returns the program's path.""" +def build(name, sources, defines=(), include_dirs=()): + """sources: paths relative to the repository root (or absolute); defines: NAME or NAME=VALUE; include_dirs: extra + include folders (absolute). Returns the program's path.""" tmp = tempfile.mkdtemp(prefix="wg_%s_" % name) exe = os.path.join(tmp, name + (".exe" if os.name == "nt" else "")) srcs = [os.path.join(ROOT, s) for s in sources] cxx = os.environ.get("CXX") or shutil.which("clang++") or shutil.which("g++") if cxx: - cmd = [cxx, "-std=c++17", "-O1"] + ["-D" + d for d in defines] + cmd = [cxx, "-std=c++17", "-O1"] + ["-D" + d for d in defines] + ["-I" + d for d in include_dirs] cmd += (["-D_CRT_SECURE_NO_WARNINGS"] if os.name == "nt" else []) + srcs + ["-o", exe] r = subprocess.run(cmd + ([] if os.name == "nt" else ["-ldl", "-pthread"]), capture_output=True, text=True) elif os.name == "nt" and (shutil.which("cl") or _vcvars64()): vcvars = None if shutil.which("cl") else _vcvars64() objdir = os.path.join(tmp, "obj") os.makedirs(objdir) - cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s %s /Fo"%s\\\\" /Fe"%s"' - % (" ".join("/D" + d for d in defines), " ".join('"%s"' % s for s in srcs), objdir, exe)) + cl = ('cl /nologo /std:c++17 /O1 /EHsc /utf-8 /D_CRT_SECURE_NO_WARNINGS %s %s %s /Fo"%s\\\\" /Fe"%s"' + % (" ".join("/D" + d for d in defines), " ".join('/I"%s"' % d for d in include_dirs), + " ".join('"%s"' % s for s in srcs), objdir, exe)) bat = os.path.join(tmp, "build.bat") with open(bat, "w") as f: f.write("@echo off\r\n" + ('call "%s" >nul || exit /b 1\r\n' % vcvars if vcvars else "") + diff --git a/clean/tests/runtime_frame_trace.cpp b/clean/tests/runtime_frame_trace.cpp index 1e64b16c..8f299a55 100644 --- a/clean/tests/runtime_frame_trace.cpp +++ b/clean/tests/runtime_frame_trace.cpp @@ -12,12 +12,16 @@ // --reset=f,... frames the game asks for a reset (WgRtFrame.reset = 1) // --fail=@ that kernel's launch fails in that frame (the device returns false) // -// Prints the create result and the allocations ("alloc b "), the code objects loaded ("load [ as -// ]"), then per frame "frame index parity

reset rw rh " (wg_rt_info) and one line per +// Prints the create result and the allocations ("alloc b "), what create uploads ("upload b+ +// ": the weight blobs, so a test can tell which blob an address holds), the code objects +// loaded ("load [ as ]"), then per frame "frame index parity

reset rw rh " +// (wg_rt_info) and one line per // launch: "launch ", followed for the kernels of the table below by " | " and the decoded // arguments: a device address as b+ (allocation k) or 0, an int as its value, the encoder and decoder // parameters as E{rw,rh,fw,fh,frame,reset} / D{rw,rh,fw,fh,reset}. A failed frame prints what was launched up to the -// failure and "ERR frame : "; the next frame runs. "ERR " at create. The code object files are not +// failure and "ERR frame : "; the next frame runs. After each frame, "name b+ " +// for every buffer name the runtime knows (wg_rt_buffer; kNames below), so the launches' addresses can be checked +// against the buffers' names. "ERR " at create. The code object files are not // read: names in the build folder (empty files) select the runtime's optional paths as real code objects would. Inputs // are zeros (the recording device does not compute): the sequence and the arguments are what is checked. #include @@ -29,6 +33,7 @@ #include #include +#include "../converter/src/sha256.h" #include "../runtime/device.h" #include "../runtime/runtime_core.h" #include "../runtime/wg_runtime.h" @@ -121,6 +126,9 @@ class RecordingDevice : public Device { bool upload(DevPtr dst, const void* src, size_t n) override { uint8_t* d = at(dst, n); if (d && n) std::memcpy(d, src, n); + if (frame < 0 && d) // create: the weight blobs (and the runtime's tables) + log->push_back("upload " + addr(dst) + " " + std::to_string(n) + " " + + wgc::to_hex(wgc::Sha256::hash(src, n))); return d != nullptr; } bool download(void* dst, DevPtr src, size_t n) override { @@ -201,6 +209,15 @@ std::vector split(const std::string& s, char sep) { } // namespace +// The buffer names of wg_rt_buffer (clean/runtime/runtime.cpp Runtime::buffer, FORMATS.md "Per-frame runtime"); a +// name the runtime does not know in a build or for a model is skipped. +const char* const kNames[] = { + "exposure.pair", "input.color", "input.depth", "input.motion", "enc0.features", "enc0.m_near", "enc0.m_far", + "enc0.m_center", "enc1.skip", "enc1.merged", "enc2.skip", "enc2.merged", "b0.out", "b1.out", "b2.out", "b3.out", + "b4.out", "b5.out", "dec2.out", "dec1.out", "dec0.A", "dec0.B", "dec0.C", "dec0.latent", "post.history", + "post.reduced", "post.display", "filter.out", "state.latent_prev", "state.motion_history", "state.history_prev", + "l.wide", "l.narrow", "l.x3"}; + // The device backends are not linked (runtime_create gets the recording device); the runtime still names them. namespace wgrt { #if defined(_WIN32) @@ -291,6 +308,12 @@ int main(int argc, char** argv) { for (const std::string& l : log) std::printf("%s\n", l.c_str()); log.clear(); if (!ok) std::printf("ERR frame %d: %s\n", f, runtime_error(rt).c_str()); + for (const char* nm : kNames) { + uintptr_t p = 0; + size_t bytes = 0; + if (wg_rt_buffer(rt, nm, &p, &bytes) == 0) + std::printf("name %s %s %zu\n", nm, rec->addr((DevPtr)p).c_str(), bytes); + } } std::printf("kernels %s\n", runtime_kernels(rt)); runtime_destroy(rt); diff --git a/clean/tests/test_runtime_frame_trace.py b/clean/tests/test_runtime_frame_trace.py index f275e51a..062004fa 100644 --- a/clean/tests/test_runtime_frame_trace.py +++ b/clean/tests/test_runtime_frame_trace.py @@ -75,21 +75,28 @@ def run_trace(rw=64, rh=64, ow=96, oh=96, frames=3, options="", build_files=(), class Trace: - """A parsed run: create (the first line), allocs {k: bytes}, loads, frames [{"info": {...}, "launches": [Launch], - "failed": kernel or None, "error": message or None}], kernels {key: value}.""" + """A parsed run: create (the first line), allocs {k: bytes}, uploads {address: (bytes, SHA-256 hex)} (create's), + loads, frames [{"info": {...}, "launches": [Launch], "failed": kernel or None, "error": message or None, "names": + {buffer name: address}}], kernels {key: value}.""" def __init__(self, lines): - self.create, self.allocs, self.loads, self.frames, self.kernels = lines[0], {}, [], [], {} + self.create, self.allocs, self.uploads, self.loads, self.frames, self.kernels = lines[0], {}, {}, [], [], {} for ln in lines[1:]: if ln.startswith("alloc b"): k, n = ln[7:].split(" ") self.allocs[int(k)] = int(n) + elif ln.startswith("upload "): + addr, n, digest = ln[7:].split(" ") + self.uploads[addr] = (int(n), digest) + elif ln.startswith("name "): + nm, addr, _ = ln[5:].split(" ") + self.frames[-1]["names"][nm] = addr elif ln.startswith("load "): self.loads.append(ln[5:]) elif ln.startswith("frame "): f = ln.split(" ") info = {f[i]: int(f[i + 1]) for i in range(2, len(f) - 1, 2)} - self.frames.append({"info": info, "launches": [], "failed": None, "error": None}) + self.frames.append({"info": info, "launches": [], "failed": None, "error": None, "names": {}}) elif ln.startswith("launch "): self.frames[-1]["launches"].append(Launch(ln[7:])) elif ln.startswith("failed "): @@ -105,6 +112,11 @@ def unknown(self): """Decoded arguments that are no allocation's address (a wrong entry in the recorder's table shows here).""" return [(ln.kernel, a) for f in self.frames for ln in f["launches"] for a in ln.args if a.startswith("?")] + def uploaded(self, digest): + """The address create uploaded these bytes to (SHA-256 hex), the whole allocation; None if none.""" + hits = [a for a, (_, d) in self.uploads.items() if d == digest and a.endswith("+0")] + return hits[0] if len(hits) == 1 else None + def extent(self, addr): """Bytes from a decoded address b+ to the end of its allocation.""" k, off = addr[1:].split("+")