From ff3fc5df91ad5a711ee70f12d75fbf26bccc39d5 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:25:16 -0500 Subject: [PATCH 1/4] Builds: the official HIP SDK only, never one from the environment - clean/runner/sdk.py: the SDK of every build is AMD's official HIP SDK folder; --rocm may name only that folder (another one needs --force-sdk); a banner names the SDK, compiler and device-library folders in every build log - build.py / k_build.py: --rocm defaults to the official SDK instead of ROCM_PATH; find_clang never falls back to PATH, find_host_cxx not on Windows; children get --force-sdk when the parent had it; a release runtime build still reuses pinned Model L code objects unless --rocm is given - every other script and test that defaulted to ROCM_PATH (or WG_K_TEST_ROCM) uses the official SDK; hip.py's default comes from sdk.py and its LLVM_PATH warning no longer prints the folder - clean/tests/test_sdk.py: the default ignores ROCM_PATH / HIP_PATH / LLVM_PATH / HIP_DEVICE_LIB_PATH, another --rocm is refused without --force-sdk, clang is never taken from PATH, the banner, build.py's device compiles use the official device libraries with ROCM_PATH set elsewhere, and no Python file of clean/ reads ROCM_PATH as a default - BUILDING.md: the SDK rule Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- BUILDING.md | 9 +- clean/dx12/f16lut/run.py | 2 +- clean/dx12/f16lut/run_in.py | 2 +- clean/dx12/gelu_lut/run.py | 2 +- clean/dx12/tests/native_helpers_check.py | 2 +- clean/dx12/tests/run_device_test.py | 2 +- clean/k/runner/k_bench.py | 3 +- clean/k/runner/k_build.py | 10 ++- clean/k/tests/test_k_hidden_guard.py | 5 +- clean/k/tests/test_k_kernels_emu.py | 5 +- clean/k/tests/test_k_norm_r16.py | 5 +- clean/k/tests/test_k_output_arithmetic.py | 5 +- clean/k/tests/test_k_output_selector.py | 5 +- clean/k/tests/test_k_runtime_gpu.py | 5 +- clean/k/tests/test_k_tiled_input.py | 3 +- clean/l/runtime/tests/test_l_runtime_abi.py | 6 +- clean/runner/bench_bneck.py | 3 +- clean/runner/build.py | 54 +++++++----- clean/runner/hip.py | 4 +- clean/runner/mlp_probe.py | 4 +- clean/runner/overlap_probe.py | 4 +- clean/runner/perf_bench.py | 3 +- clean/runner/post_ablate.py | 3 +- clean/runner/post_bench.py | 3 +- clean/runner/post_exact.py | 3 +- clean/runner/post_fused_bench.py | 3 +- clean/runner/post_sizes.py | 3 +- clean/runner/post_tf_runtime.py | 3 +- clean/runner/post_traffic.py | 3 +- clean/runner/post_variants.py | 3 +- clean/runner/profile_stages.py | 3 +- clean/runner/profile_v2.py | 3 +- clean/runner/rates.py | 3 +- clean/runner/sdk.py | 49 +++++++++++ clean/runner/selftest.py | 3 +- clean/runner/sweep.py | 3 +- clean/tests/test_build_motion_define.py | 7 +- clean/tests/test_sdk.py | 95 +++++++++++++++++++++ 38 files changed, 262 insertions(+), 71 deletions(-) create mode 100644 clean/runner/sdk.py create mode 100644 clean/tests/test_sdk.py diff --git a/BUILDING.md b/BUILDING.md index cb9aad0e..c9867a69 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -33,7 +33,7 @@ these defaults: | `WG_TOOLCHAINS` | folder that holds `dxc-v1.10.2605.24\` (the DXC release zip, unpacked) and `rga-2.15\` (the RGA zip, unpacked) | `\toolchains` (ignored by git) | | `WG_DXC_DIR` | DXC's `bin\x64` folder (overrides `WG_TOOLCHAINS`) | `%WG_TOOLCHAINS%\dxc-v1.10.2605.24\bin\x64` | | `WG_RGA` | `rga.exe` (overrides `WG_TOOLCHAINS`) | `%WG_TOOLCHAINS%\rga-2.15\rga.exe` | -| `ROCM_PATH` | HIP SDK folder. Scripts with a `--rocm` argument default to it (the argument wins); the others read it | `C:\Program Files\AMD\ROCm\7.2` | +| `ROCM_PATH` | Not read by the build scripts: they use AMD's official HIP SDK folder `C:\Program Files\AMD\ROCm\7.2` (`clean/runner/sdk.py`); `--rocm` may name only that folder, another one needs `--force-sdk`. Every build prints the SDK, compiler and device-library folders it uses | - | | `VSDEVCMD` | Visual Studio's `VsDevCmd.bat` (the kernel scripts build two small helper tools); for another edition e.g. `C:\Program Files\Microsoft Visual Studio\18\Professional\Common7\Tools\VsDevCmd.bat` (or `...\18\Enterprise\...`) | `C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat` | | `CXX`, `WG_K_RT_CXX`, `WG_K_RT_BUILD`, `WG_CONVERTER_EXE` | Model K table tests (`clean\k\tests\test_k_runtime_tables`, `test_k_runtime_pack`): the compiler is `%CXX%`, else clang++ / g++ on PATH, else the HIP SDK's clang++, else MSVC (`WG_K_RT_CXX=msvc` picks MSVC first); their build folder; an already built converter. With no compiler found the tests skip themselves with that reason | temporary folder; converter built from source | | `WG_BUILD_ROOT` | default output root of the scripts that have one | `\build` (ignored by git) | @@ -41,9 +41,10 @@ these defaults: | `WG_GPU_LOCK` | optional: a script run as `python --timeout -- ` around GPU jobs | unset: GPU jobs run directly | | `WRONGGPU_HIP_RUNTIME` | the HIP runtime the launcher's probe loads (tests: the stand-in of step 5.1) | the system's | -Set `ROCM_PATH` to the HIP SDK folder (the commands below also pass it as `--rocm`). Remove `HIP_PATH`, `LLVM_PATH`, -`HIP_DEVICE_LIB_PATH` and `HIP_CLANG_PATH` from the build's environment unless you set them on purpose; the scripts give -the HIP compiler the folder explicitly. +The build scripts use AMD's official HIP SDK in `C:\Program Files\AMD\ROCm\7.2` and never take an SDK, a +clang or a device-library folder from the environment (`ROCM_PATH`, `HIP_PATH`, `LLVM_PATH`, `HIP_DEVICE_LIB_PATH`, +`HIP_CLANG_PATH`) or from `PATH`. Remove those variables from the build's environment anyway: the vendor runtime +reads some of them. A build with another SDK needs `--rocm --force-sdk`, and its log names that folder. Open a "x64 Native Tools Command Prompt for VS 2026" (or call `vcvars64.bat`) for every step below, at the repository root. The commands write into `out\`; any folder outside the repository works the same. diff --git a/clean/dx12/f16lut/run.py b/clean/dx12/f16lut/run.py index 00dd87f8..6e070b90 100644 --- a/clean/dx12/f16lut/run.py +++ b/clean/dx12/f16lut/run.py @@ -28,7 +28,7 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = os.environ.get("ROCM_PATH", r"C:\Program Files\AMD\ROCm\7.2") +ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") diff --git a/clean/dx12/f16lut/run_in.py b/clean/dx12/f16lut/run_in.py index 8bf7bc9c..ef6ac054 100644 --- a/clean/dx12/f16lut/run_in.py +++ b/clean/dx12/f16lut/run_in.py @@ -33,7 +33,7 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = os.environ.get("ROCM_PATH", r"C:\Program Files\AMD\ROCm\7.2") +ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") diff --git a/clean/dx12/gelu_lut/run.py b/clean/dx12/gelu_lut/run.py index 5e72d8d6..c6ccb49b 100644 --- a/clean/dx12/gelu_lut/run.py +++ b/clean/dx12/gelu_lut/run.py @@ -32,7 +32,7 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = os.environ.get("ROCM_PATH", r"C:\Program Files\AMD\ROCm\7.2") +ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") diff --git a/clean/dx12/tests/native_helpers_check.py b/clean/dx12/tests/native_helpers_check.py index 46290484..c53c14a6 100644 --- a/clean/dx12/tests/native_helpers_check.py +++ b/clean/dx12/tests/native_helpers_check.py @@ -25,7 +25,7 @@ sys.path.insert(0, str(DX12)) import build_dx12 # noqa: E402 -ROCM = os.environ.get("ROCM_PATH", r"C:\Program Files\AMD\ROCm\7.2") +ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH GUARD = 16 POISON = np.uint32(0xa5c31f7b) diff --git a/clean/dx12/tests/run_device_test.py b/clean/dx12/tests/run_device_test.py index 3c8f515b..ab40a3bd 100644 --- a/clean/dx12/tests/run_device_test.py +++ b/clean/dx12/tests/run_device_test.py @@ -15,7 +15,7 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = os.environ.get("ROCM_PATH", r"C:\Program Files\AMD\ROCm\7.2") +ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH def main(): diff --git a/clean/k/runner/k_bench.py b/clean/k/runner/k_bench.py index 3c8dc6ce..f868a1a3 100644 --- a/clean/k/runner/k_bench.py +++ b/clean/k/runner/k_bench.py @@ -29,6 +29,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.k.reference import network as N # noqa: E402 from clean.k.reference.layout import (DEFAULT_S, FUSED_IO, STAGES, base_size, level_size, stage_offsets, # noqa: E402 threshold_windows, window_flags_to_map, window_grid) @@ -291,7 +292,7 @@ def print_bench(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=("hip", "emu")) ap.add_argument("--sizes", nargs="*", default=["1080p", "1440p", "4k"]) ap.add_argument("--s", type=float, default=DEFAULT_S, diff --git a/clean/k/runner/k_build.py b/clean/k/runner/k_build.py index d3a18e1d..6440bdce 100644 --- a/clean/k/runner/k_build.py +++ b/clean/k/runner/k_build.py @@ -62,7 +62,8 @@ def emu_command(cxx, out): def main(argv=None): from clean.runner.build import find_clang, run ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + from clean.runner import sdk + sdk.add_arguments(ap) ap.add_argument("--out", required=True) ap.add_argument("--clang") ap.add_argument("--no-device-libs", action="store_true") @@ -72,11 +73,12 @@ def main(argv=None): ap.add_argument("--emu", action="store_true", help="also the host emulator library " + EMU_LIBRARY) ap.add_argument("--cxx", default=os.environ.get("CXX", "clang++")) a = ap.parse_args(argv) + a.rocm = sdk.resolve(a.rocm, a.force_sdk) os.makedirs(a.out, exist_ok=True) if not a.emu_only: - if not a.rocm and not a.no_device_libs: - raise SystemExit("--rocm is required (or --no-device-libs for a compile check)") - build_device(a.clang or find_clang(a.rocm), a.rocm, a.out, not a.no_device_libs, a.report, io_wave64=not a.io_wave32) + clang = a.clang or find_clang(a.rocm) + print(sdk.banner(a.rocm, clang=clang, device_libs=not a.no_device_libs), flush=True) + build_device(clang, a.rocm, a.out, not a.no_device_libs, a.report, io_wave64=not a.io_wave32) if a.emu or a.emu_only: run(emu_command(a.cxx, os.path.join(a.out, EMU_LIBRARY))) diff --git a/clean/k/tests/test_k_hidden_guard.py b/clean/k/tests/test_k_hidden_guard.py index 063550cf..ee94bd4e 100644 --- a/clean/k/tests/test_k_hidden_guard.py +++ b/clean/k/tests/test_k_hidden_guard.py @@ -8,6 +8,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.build import device_command, find_clang from clean.runner.hip import Hip @@ -19,7 +20,7 @@ def setUpClass(cls): if not build: raise unittest.SkipTest("GPU only: set WG_K_GPU_BUILD and use shared GPU lock") root = Path(__file__).resolve().parents[3] - sdk = os.environ.get("WG_K_TEST_ROCM") or os.environ.get("ROCM_PATH") or r"C:\Program Files\AMD\ROCm\7.2" + sdk = OFFICIAL_ROCM obj = Path(build).resolve() / "k_hidden_guard_probe.co" cmd = device_command(find_clang(sdk), sdk, str(Path(__file__).with_name("k_hidden_guard_probe.hip")), str(obj), @@ -28,7 +29,7 @@ def setUpClass(cls): result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode: raise RuntimeError(result.stdout + result.stderr) - cls.dev = Hip(os.environ.get("ROCM_PATH", "")) + cls.dev = Hip(OFFICIAL_ROCM) cls.mod = cls.dev.load_module(obj.read_bytes()) def launch(self, name, args): diff --git a/clean/k/tests/test_k_kernels_emu.py b/clean/k/tests/test_k_kernels_emu.py index d3aa2222..dda0ea54 100644 --- a/clean/k/tests/test_k_kernels_emu.py +++ b/clean/k/tests/test_k_kernels_emu.py @@ -18,6 +18,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.k.reference import linear as L from clean.k.reference import network as N from clean.k.reference.layout import STAGES, cdiv, window_partition @@ -49,7 +50,7 @@ def runner(): from clean.runner.stage_runner import Runner gpu = os.environ.get("WG_K_GPU_BUILD") if gpu: - return Runner(gpu, os.environ.get("ROCM_PATH", ""), backend="hip", model="K") + return Runner(gpu, OFFICIAL_ROCM, backend="hip", model="K") return Runner(build_dir(), "", backend="emu", model="K") @@ -58,7 +59,7 @@ def device(): from clean.k.runner.k_bench import KFuncs, open_device gpu = os.environ.get("WG_K_GPU_BUILD") backend, build = ("hip", gpu) if gpu else ("emu", build_dir()) - dev = open_device(backend, build, os.environ.get("ROCM_PATH", "")) + dev = open_device(backend, build, OFFICIAL_ROCM) return dev, KFuncs(dev, build, backend) diff --git a/clean/k/tests/test_k_norm_r16.py b/clean/k/tests/test_k_norm_r16.py index 04dbc3ae..a57a7ee5 100644 --- a/clean/k/tests/test_k_norm_r16.py +++ b/clean/k/tests/test_k_norm_r16.py @@ -12,6 +12,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.build import device_command, find_clang from clean.runner.hip import Hip @@ -22,7 +23,7 @@ def test_all_half_inputs(self): if not build: self.skipTest("GPU only: set WG_K_GPU_BUILD and use shared GPU lock") root = Path(__file__).resolve().parents[3] - sdk = os.environ.get("WG_K_TEST_ROCM") or os.environ.get("ROCM_PATH") or r"C:\Program Files\AMD\ROCm\7.2" + sdk = OFFICIAL_ROCM obj = Path(build).resolve() / "k_norm_r16_probe.co" cmd = device_command(find_clang(sdk), sdk, str(Path(__file__).with_name("k_norm_r16_probe.hip")), str(obj), @@ -30,7 +31,7 @@ def test_all_half_inputs(self): result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode: raise RuntimeError(result.stdout + result.stderr) - dev = Hip(os.environ.get("ROCM_PATH", "")) + dev = Hip(OFFICIAL_ROCM) mod = dev.load_module(obj.read_bytes()) original = dev.get_function(mod, "k_norm_r16_original") for name in ("k_norm_r16_candidate", "k_norm_r16_candidate_select"): diff --git a/clean/k/tests/test_k_output_arithmetic.py b/clean/k/tests/test_k_output_arithmetic.py index 99936d69..a6165ec2 100644 --- a/clean/k/tests/test_k_output_arithmetic.py +++ b/clean/k/tests/test_k_output_arithmetic.py @@ -13,6 +13,7 @@ import subprocess import unittest +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.build import device_command, find_clang from clean.runner.hip import Hip @@ -24,14 +25,14 @@ def setUpClass(cls): if not build: raise unittest.SkipTest("GPU only: set WG_K_GPU_BUILD and use shared GPU lock") root = Path(__file__).resolve().parents[3] - sdk = os.environ.get("WG_K_TEST_ROCM") or os.environ.get("ROCM_PATH") or r"C:\Program Files\AMD\ROCm\7.2" + sdk = OFFICIAL_ROCM obj = Path(build).resolve() / "k_output_arithmetic_probe.co" cmd = device_command(find_clang(sdk), sdk, str(Path(__file__).with_name("k_output_arithmetic_probe.hip")), str(obj), extra=["-I", str(root/"clean/kernels"), "-mwavefrontsize64"]) proc = subprocess.run(cmd, capture_output=True, text=True) if proc.returncode: raise RuntimeError(proc.stdout + proc.stderr) - cls.dev = Hip(os.environ.get("ROCM_PATH", "")) + cls.dev = Hip(OFFICIAL_ROCM) mod = cls.dev.load_module(obj.read_bytes()) cls.fn = {name: cls.dev.get_function(mod, "k_probe_"+name) for name in ("native_rcp", "candidate_rcp", "native_axis", "candidate_axis", "candidate_bounded", "compare")} diff --git a/clean/k/tests/test_k_output_selector.py b/clean/k/tests/test_k_output_selector.py index e1b48e2c..94d8577b 100644 --- a/clean/k/tests/test_k_output_selector.py +++ b/clean/k/tests/test_k_output_selector.py @@ -8,6 +8,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.build import device_command, find_clang from clean.runner.hip import Hip @@ -18,7 +19,7 @@ def test_all_half_inputs(self): if not build: self.skipTest("GPU only: set WG_K_GPU_BUILD and use shared GPU lock") root = Path(__file__).resolve().parents[3] - sdk = os.environ.get("WG_K_TEST_ROCM") or os.environ.get("ROCM_PATH") or r"C:\Program Files\AMD\ROCm\7.2" + sdk = OFFICIAL_ROCM obj = Path(build).resolve() / "k_output_selector_probe.co" cmd = device_command(find_clang(sdk), sdk, str(Path(__file__).with_name("k_output_selector_probe.hip")), str(obj), @@ -26,7 +27,7 @@ def test_all_half_inputs(self): result = subprocess.run(cmd, capture_output=True, text=True) if result.returncode: raise RuntimeError(result.stdout + result.stderr) - dev = Hip(os.environ.get("ROCM_PATH", "")) + dev = Hip(OFFICIAL_ROCM) mod = dev.load_module(obj.read_bytes()) native = dev.get_function(mod, "k_selector_native") candidate = dev.get_function(mod, "k_selector_fast") diff --git a/clean/k/tests/test_k_runtime_gpu.py b/clean/k/tests/test_k_runtime_gpu.py index 7201e04c..03f1020f 100644 --- a/clean/k/tests/test_k_runtime_gpu.py +++ b/clean/k/tests/test_k_runtime_gpu.py @@ -30,6 +30,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.k.reference.layout import STAGE_NAMES from clean.k.reference.network import random_network_params from clean.k.reference.params import normalize @@ -135,7 +136,7 @@ def __init__(self, build, model_dir, map_path, out_size, max_render, hdr, depth_ from clean.runner.build import runtime_library_name from clean.runner.pipeline import WgRtConfig, WgRtFrame self.WgRtFrame = WgRtFrame - rocm = os.environ.get("ROCM_PATH", "") + rocm = OFFICIAL_ROCM if sys.platform == "win32" and rocm and hasattr(os, "add_dll_directory"): os.add_dll_directory(os.path.join(rocm, "bin")) lib = self.lib = ctypes.CDLL(os.path.abspath(os.path.join(build, runtime_library_name()))) @@ -601,7 +602,7 @@ def test_missing_model_message(self): lib = ctypes.CDLL(os.path.abspath(os.path.join(self.build, runtime_library_name()))) lib.wg_rt_create_external.argtypes = [P(WgRtConfig), P(vp), ctypes.c_char_p, ctypes.c_int] cfg = WgRtConfig() - keep = [os.fsencode(self.build), os.fsencode(os.environ.get("ROCM_PATH", "")), + keep = [os.fsencode(self.build), os.fsencode(OFFICIAL_ROCM), os.fsencode(os.path.join(self.mdir, "nothing")), b"model=K"] cfg.build_dir, cfg.rocm_dir, cfg.model_dir, cfg.runtime_options = keep cfg.max_rw, cfg.max_rh, cfg.ow, cfg.oh = 64, 64, 128, 128 diff --git a/clean/k/tests/test_k_tiled_input.py b/clean/k/tests/test_k_tiled_input.py index a8bbe61e..313f9de4 100644 --- a/clean/k/tests/test_k_tiled_input.py +++ b/clean/k/tests/test_k_tiled_input.py @@ -9,6 +9,7 @@ import numpy as np +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.k.runner import k_io from clean.k.reference.instage import input_stage from clean.k.tests.test_k_io_emu import frame, same_bits @@ -74,7 +75,7 @@ def setUpClass(cls): if not build: raise unittest.SkipTest("GPU only: set WG_K_GPU_BUILD and use shared GPU lock") from clean.runner.stage_runner import Runner - cls.runner = Runner(build, os.environ.get("ROCM_PATH", ""), backend="hip", model="K") + cls.runner = Runner(build, OFFICIAL_ROCM, backend="hip", model="K") def run_input(self, x, exact, tiled, device_scalars=False): stage = "input@exact" if exact else "input" diff --git a/clean/l/runtime/tests/test_l_runtime_abi.py b/clean/l/runtime/tests/test_l_runtime_abi.py index f74f4b34..d4628393 100644 --- a/clean/l/runtime/tests/test_l_runtime_abi.py +++ b/clean/l/runtime/tests/test_l_runtime_abi.py @@ -8,7 +8,7 @@ executed. The argument kinds are the frame trace's table (clean/tests/runtime_frame_trace.cpp), so the recorder that checks the runtime's arguments and the kernels agree. -Skipped without the HIP SDK (ROCM_PATH or C:\\Program Files\\AMD\\ROCm\\7.2). +Skipped without the HIP SDK (C:\\Program Files\\AMD\\ROCm\\7.2). Run (repository root): python -m unittest discover -s clean/l/runtime/tests -t clean/l/runtime/tests -p test_l_runtime_abi.py @@ -33,7 +33,7 @@ def rocm(): - for d in (os.environ.get("ROCM_PATH"), r"C:\Program Files\AMD\ROCm\7.2"): + for d in (r"C:\Program Files\AMD\ROCm\7.2",): if d and os.path.isdir(d): return d return None @@ -44,7 +44,7 @@ def build(): if _BUILD is None: r = rocm() if not r: - raise unittest.SkipTest("no HIP SDK (ROCM_PATH)") + raise unittest.SkipTest(r"no HIP SDK (C:\Program Files\AMD\ROCm\7.2)") sys.path.insert(0, ROOT) from clean.l.runner import l_c32_check as C32 d = tempfile.mkdtemp(prefix="wg_labi_") diff --git a/clean/runner/bench_bneck.py b/clean/runner/bench_bneck.py index 60683358..f8167a62 100644 --- a/clean/runner/bench_bneck.py +++ b/clean/runner/bench_bneck.py @@ -23,6 +23,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner import variants as V # noqa: E402 from clean.runner.stage_runner import Runner # noqa: E402 from clean.runner.sweep import GAME_SIZES, stage_inputs, stage_params # noqa: E402 @@ -107,7 +108,7 @@ def format_report(rep): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True, help="output directory of build.py") - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variants", help="comma-separated bmega* builds (default: every one built)") ap.add_argument("--groups", help="comma-separated workgroup counts for the megakernel (default: the variant's)") diff --git a/clean/runner/build.py b/clean/runner/build.py index 7ea08c2c..b6b12dd5 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -26,6 +26,7 @@ HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, os.path.dirname(os.path.dirname(HERE))) from clean.runner import perf_variants as PV # noqa: E402 +from clean.runner import sdk # noqa: E402 from clean.runner import variants as V # noqa: E402 from clean.k.runner import k_build # noqa: E402 (model K kernels, clean/k) KERNEL_DIR = os.path.join(os.path.dirname(HERE), "kernels") @@ -152,17 +153,16 @@ def set_model_k(on): EXTRA_ARCHS = [] -def find_clang(rocm): +def find_clang(rocm=None): + """The HIP SDK's clang (rocm: the SDK folder, default the official one; clean/runner/sdk.py). Never from PATH or an + environment variable: a clang elsewhere needs --clang.""" + rocm = rocm or sdk.OFFICIAL_ROCM exe = "clang.exe" if sys.platform == "win32" else "clang" - if rocm: - for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): - p = os.path.join(rocm, sub, exe) - if os.path.exists(p): - return p - p = shutil.which(exe) - if p: - return p - raise SystemExit("clang not found; pass --clang") + for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): + p = os.path.join(rocm, sub, exe) + if os.path.exists(p): + return p + raise SystemExit(f"clang not found in the HIP SDK {rocm}; pass --clang") def device_command(clang, rocm, src, out, device_libs=True, report=False, extra=(), soft_fp8=False, defines=None): @@ -199,13 +199,17 @@ def runtime_library_name(): return "wronggpu_runtime.dll" if sys.platform == "win32" else "libwg_runtime.so" -def find_host_cxx(rocm): +def find_host_cxx(rocm=None): + """Windows: the HIP SDK's clang++ (rocm: the SDK folder, default the official one), never one from PATH; other + systems: clang++ or g++ from PATH.""" exe = "clang++.exe" if sys.platform == "win32" else "clang++" - if sys.platform == "win32" and rocm: + if sys.platform == "win32": + rocm = rocm or sdk.OFFICIAL_ROCM for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): p = os.path.join(rocm, sub, exe) if os.path.exists(p): return p + raise SystemExit(f"clang++ not found in the HIP SDK {rocm}; pass --runtime-cxx") p = shutil.which(exe) or shutil.which("g++") if p: return p @@ -440,7 +444,7 @@ def run_post_w64(cmd, out_dir, gate=True, report_name=POST_W64_REPORT, gates=Non def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", ""), help="ROCm / HIP SDK directory") + sdk.add_arguments(ap) ap.add_argument("--out", required=True, help="output directory for the code objects") ap.add_argument("--clang", help="clang executable (default: from --rocm, then PATH)") ap.add_argument("--no-device-libs", action="store_true", help="add -nogpulib instead of the ROCm device libs") @@ -488,6 +492,8 @@ def main(argv=None): "code objects and its map built in (with --runtime-only and --rocm its code objects are built " "first, for every target); its D3D12 kernels come from clean/dx12/build_dx12.py --model-l") args = ap.parse_args(argv) + rocm_given = bool(args.rocm) # release runtime builds reuse pinned Model L code objects unless --rocm is given + args.rocm = sdk.resolve(args.rocm, args.force_sdk) for name in OPTIONAL_MODELS: set_model(name, not getattr(args, "no_model_" + name.lower())) set_model_l(args.model_l) @@ -502,22 +508,28 @@ def main(argv=None): if any(a not in ("gfx1200", "gfx1201") for a in [ARCH] + EXTRA_ARCHS): raise SystemExit("supported release targets are gfx1200 and gfx1201 only") if args.runtime_only: - if MODEL_L and args.rocm: # Model L's code objects next to the given ones (release builds reuse pinned ones) + if MODEL_L and rocm_given: # Model L's code objects next to the given ones (release builds reuse pinned ones) clang = args.clang or find_clang(args.rocm) + print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) build_device_l(clang, args.rocm, args.out, device_libs=not args.no_device_libs) for arch in EXTRA_ARCHS: build_device_l(clang, args.rocm, os.path.join(args.out, arch), device_libs=not args.no_device_libs, arch=arch) - build_runtime(args.runtime_cxx or find_host_cxx(args.rocm), args.out, embed=not args.no_embed) + cxx = args.runtime_cxx or find_host_cxx(args.rocm) + print(sdk.banner(args.rocm, cxx=cxx, device_libs=False), flush=True) + build_runtime(cxx, args.out, embed=not args.no_embed) return if args.motion_outres_only: if not args.rocm and not args.no_device_libs: raise SystemExit("--rocm is required (or --no-device-libs for a compile check)") - build_motion_outres(args.clang or find_clang(args.rocm), args) + clang = args.clang or find_clang(args.rocm) + print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) + build_motion_outres(clang, args) for arch in EXTRA_ARCHS: sub = [sys.executable, os.path.abspath(__file__), "--out", os.path.join(args.out, arch), "--motion-outres-only"] - sub += (["--rocm", args.rocm] if args.rocm else []) + (["--clang", args.clang] if args.clang else []) + sub += ["--rocm", args.rocm] + (["--force-sdk"] if args.force_sdk else []) + sub += ["--clang", args.clang] if args.clang else [] sub += (["--no-device-libs"] if args.no_device_libs else []) + (["--soft-fp8"] if args.soft_fp8 else []) sub += ["--no-resource-gate"] if args.no_resource_gate else [] # as the ordinary branch's child: the defines of an experiment build (without them the child builds the @@ -533,6 +545,7 @@ def main(argv=None): if not args.rocm and not args.no_device_libs: raise SystemExit("--rocm is required (or --no-device-libs for a compile check)") clang = args.clang or find_clang(args.rocm) + print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) for src in SOURCES: out = os.path.join(args.out, os.path.splitext(src)[0] + ".co") run(device_command(clang, args.rocm, os.path.join(KERNEL_DIR, src), out, @@ -637,7 +650,8 @@ def main(argv=None): if EXTRA_ARCHS and not args.emu_only: # the same default code objects for the further targets for arch in EXTRA_ARCHS: sub = [sys.executable, os.path.abspath(__file__), "--out", os.path.join(args.out, arch), "--no-runtime"] - sub += (["--rocm", args.rocm] if args.rocm else []) + (["--clang", args.clang] if args.clang else []) + sub += ["--rocm", args.rocm] + (["--force-sdk"] if args.force_sdk else []) + sub += ["--clang", args.clang] if args.clang else [] sub += (["--no-device-libs"] if args.no_device_libs else []) + (["--soft-fp8"] if args.soft_fp8 else []) sub += ["--no-model-" + n.lower() for n in OPTIONAL_MODELS if getattr(args, "no_model_" + n.lower())] sub += ["--model-l"] if MODEL_L else [] @@ -661,7 +675,9 @@ def main(argv=None): "profile": V.is_profile(n), "source": V.source(n)} for n in ["base"] + names}, f, indent=1) if not args.emu_only and not args.no_runtime: - build_runtime(args.runtime_cxx or find_host_cxx(args.rocm), args.out, embed=not args.no_embed) + cxx = args.runtime_cxx or find_host_cxx(args.rocm) + print(sdk.banner(args.rocm, cxx=cxx, device_libs=False), flush=True) + build_runtime(cxx, args.out, embed=not args.no_embed) if __name__ == "__main__": diff --git a/clean/runner/hip.py b/clean/runner/hip.py index 5fe18ee7..bfc4613f 100644 --- a/clean/runner/hip.py +++ b/clean/runner/hip.py @@ -22,7 +22,7 @@ class HipError(RuntimeError): # HSA is removed, except HIP_PLATFORM and HIP_PATH_72, and ROCM_PATH is set to the SDK. A stray inherited variable # can point the vendor runtime at an unrelated folder (an LLVM_PATH outside the SDK made hipModuleLoadData crash). # Per process only: the user's and the machine's environment are not changed. -DEFAULT_ROCM = r"C:\Program Files\AMD\ROCm\7.2" +from clean.runner.sdk import OFFICIAL_ROCM as DEFAULT_ROCM # noqa: E402 (the official HIP SDK) HIP_ENV_KEEP = ("HIP_PLATFORM", "HIP_PATH_72") HIP_ENV_PARTS = ("HIP", "LLVM", "ROCM", "HSA") @@ -63,7 +63,7 @@ def _check_environment(rocm_dir): root = os.path.normcase(os.path.abspath(rocm_dir)) if rocm_dir else "" path = os.path.normcase(os.path.abspath(llvm)) if not root or os.path.commonpath([root, path]) != root: - warnings.warn(f"LLVM_PATH is set to {llvm!r}, outside the HIP SDK ({rocm_dir!r}); an unrelated LLVM there " + warnings.warn(f"LLVM_PATH is set to a folder outside the HIP SDK ({rocm_dir!r}); an unrelated LLVM there " "can crash hipModuleLoadData. Unset it if module loading fails.", RuntimeWarning, stacklevel=3) diff --git a/clean/runner/mlp_probe.py b/clean/runner/mlp_probe.py index dc24dc75..de160770 100644 --- a/clean/runner/mlp_probe.py +++ b/clean/runner/mlp_probe.py @@ -27,6 +27,8 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) + BLOCK = 256 SIMDS = 128 PARTS = ["111", "111p", "011", "101", "110", "100", "001", "010", "000"] # W1 GELU W2 present (1) / removed (0) @@ -96,7 +98,7 @@ def format_table(res): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--json") args = ap.parse_args(argv) res = run(args.build, args.rocm) diff --git a/clean/runner/overlap_probe.py b/clean/runner/overlap_probe.py index 85577ec1..6fbb7c1e 100644 --- a/clean/runner/overlap_probe.py +++ b/clean/runner/overlap_probe.py @@ -29,6 +29,8 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) + BLOCK = 256 WAVES_PER_BLOCK = BLOCK // 32 WMMA_FLOP = 2 * 16 * 16 * 16 # one 16x16x16 WMMA @@ -127,7 +129,7 @@ def format_report(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--json") ap.add_argument("--repeats", type=int, default=9) args = ap.parse_args(argv) diff --git a/clean/runner/perf_bench.py b/clean/runner/perf_bench.py index 91c85f5c..10c55c8b 100644 --- a/clean/runner/perf_bench.py +++ b/clean/runner/perf_bench.py @@ -28,6 +28,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference import decoder as Dc # noqa: E402 from clean.reference import encoder as E # noqa: E402 from clean.reference import exposure as X # noqa: E402 @@ -176,7 +177,7 @@ def bench_fusion(r, report, repeats, check=True, built=("fused",)): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--repeats", type=int, default=100) ap.add_argument("--stages", default="enc0,dec0,exposure,dec1dec0", help="comma-separated: enc0, dec0, exposure, dec1dec0 (dec1 + dec0 separate vs fused)") diff --git a/clean/runner/post_ablate.py b/clean/runner/post_ablate.py index b71f53b5..c744a256 100644 --- a/clean/runner/post_ablate.py +++ b/clean/runner/post_ablate.py @@ -24,6 +24,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner import build as B # noqa: E402 from clean.runner import post as PS # noqa: E402 from clean.runner.post_bench import big_inputs # noqa: E402 @@ -103,7 +104,7 @@ def time_build(r, co, w, x_norm, x_reset, disp, repeats, keys=None): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--clang", help="clang (default: from --rocm)") ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=1) diff --git a/clean/runner/post_bench.py b/clean/runner/post_bench.py index ea929872..45b7081c 100644 --- a/clean/runner/post_bench.py +++ b/clean/runner/post_bench.py @@ -18,6 +18,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference import post as P # noqa: E402 from clean.runner.stage_runner import Runner # noqa: E402 from clean.runner import post as PS # noqa: E402 @@ -89,7 +90,7 @@ def check_small(r, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--json", help="also write the results to this file") ap.add_argument("--no-check", action="store_true", help="timing only") diff --git a/clean/runner/post_exact.py b/clean/runner/post_exact.py index 3fa1c64d..132e83db 100644 --- a/clean/runner/post_exact.py +++ b/clean/runner/post_exact.py @@ -28,6 +28,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.post_sizes import configs # noqa: E402 # odd sizes: dynamic-resolution steps, odd D, an output width that is not a multiple of 32 (copy path: post_filter_h) @@ -50,7 +51,7 @@ def dump(python, repo, build, rocm, render, output, frames, paths, motion, seed, def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--a-repo", required=True) ap.add_argument("--a-build", required=True) ap.add_argument("--b-repo", required=True) diff --git a/clean/runner/post_fused_bench.py b/clean/runner/post_fused_bench.py index 7ccac841..c0dfffd7 100644 --- a/clean/runner/post_fused_bench.py +++ b/clean/runner/post_fused_bench.py @@ -25,6 +25,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference import post as P # noqa: E402 from clean.runner import post as PS # noqa: E402 from clean.runner.post_bench import big_inputs, f16_steps # noqa: E402 @@ -91,7 +92,7 @@ def check_small(r, code_objects, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--clang", help="clang for building post_w64.co (default: from --rocm)") ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--json", help="also write the results to this file") diff --git a/clean/runner/post_sizes.py b/clean/runner/post_sizes.py index d5e7f343..7306934b 100644 --- a/clean/runner/post_sizes.py +++ b/clean/runner/post_sizes.py @@ -59,6 +59,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner import post as PS # noqa: E402 W64 = "post_w64.co" @@ -629,7 +630,7 @@ def main(argv=None): for name in ("time", "dump"): p = sub.add_parser(name) p.add_argument("--build", required=True) - p.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + p.add_argument("--rocm", default=OFFICIAL_ROCM) p.add_argument("--code-object", default=W64, help="code object of the post kernels (default post_w64.co)") p.add_argument("--render", help="render size WxH (with --output: one configuration)") p.add_argument("--output", help="output size WxH") diff --git a/clean/runner/post_tf_runtime.py b/clean/runner/post_tf_runtime.py index 29a3c8df..856b31c7 100644 --- a/clean/runner/post_tf_runtime.py +++ b/clean/runner/post_tf_runtime.py @@ -28,6 +28,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference import encoder as En # noqa: E402 from clean.runner.pipeline import Pipeline # noqa: E402 from clean.runner.synthetic_model import synthetic_tensors, write_raw # noqa: E402 @@ -118,7 +119,7 @@ def frame_inputs(rng, rw, rh, reset, pi): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--model", required=True) ap.add_argument("--output", default="2560x1440") ap.add_argument("--frames", type=int, default=14) diff --git a/clean/runner/post_traffic.py b/clean/runner/post_traffic.py index c58c7b47..5bf89831 100644 --- a/clean/runner/post_traffic.py +++ b/clean/runner/post_traffic.py @@ -39,6 +39,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner import post as PS # noqa: E402 from clean.runner.post_bench import IN_GAME, big_inputs # noqa: E402 from clean.runner.stage_runner import Runner # noqa: E402 @@ -215,7 +216,7 @@ def run(r, sizes, repeats=50, rounds=3, reference=True, report=None, flush_bytes def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=3) ap.add_argument("--no-reference", action="store_true", help="only the traffic layouts, not the real kernels") diff --git a/clean/runner/post_variants.py b/clean/runner/post_variants.py index 051db937..ae9a92cc 100644 --- a/clean/runner/post_variants.py +++ b/clean/runner/post_variants.py @@ -23,6 +23,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference import post as P # noqa: E402 from clean.runner import post as PS # noqa: E402 from clean.runner.post_bench import big_inputs, f16_steps # noqa: E402 @@ -103,7 +104,7 @@ def check(r, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=3) ap.add_argument("--json", help="also write the results to this file") diff --git a/clean/runner/profile_stages.py b/clean/runner/profile_stages.py index bdb0142c..cbf42fc1 100644 --- a/clean/runner/profile_stages.py +++ b/clean/runner/profile_stages.py @@ -36,6 +36,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference.swin import STAGES, random_inputs, random_params, window_offset # noqa: E402 from clean.runner import variants as V # noqa: E402 from clean.runner.packing import block_layout # noqa: E402 @@ -145,7 +146,7 @@ def block_ms(times): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variant", default="prof") ap.add_argument("--stages", help="comma-separated (default: enc1, enc2, b0, dec2, dec1)") diff --git a/clean/runner/profile_v2.py b/clean/runner/profile_v2.py index 551f49e2..33ab6a45 100644 --- a/clean/runner/profile_v2.py +++ b/clean/runner/profile_v2.py @@ -24,6 +24,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference.swin import random_inputs, random_params, window_offset # noqa: E402 from clean.runner.stage_runner import Runner # noqa: E402 from clean.runner.sweep import GAME_SIZES, stage_inputs, stage_offset, stage_params # noqa: E402 @@ -130,7 +131,7 @@ def format_report(rep): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variant", default="v2prof") ap.add_argument("--stages", help="comma-separated (default: all of sweep.GAME_SIZES)") diff --git a/clean/runner/rates.py b/clean/runner/rates.py index 7ebe9a79..5e8d611f 100644 --- a/clean/runner/rates.py +++ b/clean/runner/rates.py @@ -22,6 +22,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.runner.gen_rates import MIX_K, OPS # noqa: E402 BLOCK = 256 @@ -102,7 +103,7 @@ def format_table(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--json") args = ap.parse_args(argv) r = run(args.build, args.rocm) diff --git a/clean/runner/sdk.py b/clean/runner/sdk.py new file mode 100644 index 00000000..e7ae63d2 --- /dev/null +++ b/clean/runner/sdk.py @@ -0,0 +1,49 @@ +# SPDX-License-Identifier: MIT +"""The HIP SDK of every build and HIP process of this project: AMD's official HIP SDK in its standard install folder, +never one taken from the environment (ROCM_PATH, HIP_PATH, LLVM_PATH, HIP_DEVICE_LIB_PATH or PATH). A script's --rocm +may name only that folder; another one needs --force-sdk. Every build prints the SDK, compiler and device-library +folders it uses (banner), so a build log shows where its code objects came from.""" +import os +import sys + +OFFICIAL_ROCM = r"C:\Program Files\AMD\ROCm\7.2" if sys.platform == "win32" else "/opt/rocm" + + +def _norm(path): + return os.path.normcase(os.path.normpath(os.path.abspath(path))) + + +def is_official(path): + return bool(path) and _norm(path) == _norm(OFFICIAL_ROCM) + + +def resolve(rocm=None, force=False): + """The SDK folder for a --rocm value: empty or None = the official SDK; another folder only with force.""" + if not rocm: + return OFFICIAL_ROCM + if is_official(rocm) or force: + return rocm + raise SystemExit(f"--rocm {rocm!r} is not the official HIP SDK ({OFFICIAL_ROCM}); pass --force-sdk to build with " + "it anyway") + + +def device_lib_dir(rocm): + return os.path.join(rocm, "amdgcn", "bitcode") + + +def add_arguments(ap, help_suffix=""): + """--rocm (default: the official SDK, never an environment variable) and --force-sdk.""" + ap.add_argument("--rocm", default="", help=f"HIP SDK folder (default and only accepted value without --force-sdk: " + f"{OFFICIAL_ROCM}){help_suffix}") + ap.add_argument("--force-sdk", action="store_true", help="accept a --rocm other than the official HIP SDK") + + +def banner(rocm, clang=None, cxx=None, device_libs=True): + """One line naming the SDK, the compilers and the device-library folder of a build.""" + parts = [f"HIP SDK: {rocm} ({'official' if is_official(rocm) else 'NOT the official SDK: forced'})"] + if clang: + parts.append(f"clang: {clang}") + if cxx: + parts.append(f"host C++: {cxx}") + parts.append(f"device libs: {device_lib_dir(rocm) if device_libs else 'none (-nogpulib)'}") + return "; ".join(parts) diff --git a/clean/runner/selftest.py b/clean/runner/selftest.py index ad9952be..f4747190 100644 --- a/clean/runner/selftest.py +++ b/clean/runner/selftest.py @@ -34,6 +34,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference.fp8 import DECODE, fp8_encode, q8 # noqa: E402 from clean.runner.packing import D_ROW_LAYOUTS # noqa: E402 @@ -311,7 +312,7 @@ def selftest(build_dir, rocm_dir="", backend="hip"): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) args = ap.parse_args(argv) report = selftest(args.build, args.rocm, args.backend) diff --git a/clean/runner/sweep.py b/clean/runner/sweep.py index 808ede3e..baa14e2b 100644 --- a/clean/runner/sweep.py +++ b/clean/runner/sweep.py @@ -19,6 +19,7 @@ import numpy as np # noqa: E402 +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) from clean.reference.swin import STAGES, random_inputs, random_params, window_offset # noqa: E402 from clean.runner import variants as V # noqa: E402 from clean.runner.stage_runner import Runner # noqa: E402 @@ -156,7 +157,7 @@ def format_report(report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=os.environ.get("ROCM_PATH", "")) + ap.add_argument("--rocm", default=OFFICIAL_ROCM) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--stages", help="comma-separated (default: all)") ap.add_argument("--variants", default="all", help="comma-separated names, or 'all' (everything in variants.json, " diff --git a/clean/tests/test_build_motion_define.py b/clean/tests/test_build_motion_define.py index af6fd35f..05576c88 100644 --- a/clean/tests/test_build_motion_define.py +++ b/clean/tests/test_build_motion_define.py @@ -23,8 +23,9 @@ def fake_run(cmd, *args, **kwargs): with tempfile.TemporaryDirectory() as out, patch.object(build, "EXTRA_ARCHS", []), \ patch.object(build, "build_motion_outres") as parent, patch.object(build.subprocess, "run", fake_run), \ contextlib.redirect_stdout(io.StringIO()): - # (--rocm is given: the default would be the ROCM_PATH of the environment this runs in) - build.main(["--out", out, "--motion-outres-only", "--rocm", "dummy-rocm", "--no-device-libs", "--clang", "dummy-clang", + # (a dummy --rocm, accepted with --force-sdk: no compiler runs) + build.main(["--out", out, "--motion-outres-only", "--rocm", "dummy-rocm", "--force-sdk", "--no-device-libs", + "--clang", "dummy-clang", "--extra-archs", "gfx1200,gfx1201"] + argv) parent.assert_called_once() return started @@ -55,7 +56,7 @@ def test_without_them_the_child_has_neither(self): def test_the_options_that_were_forwarded_before_still_are(self): cmd = child_commands(["--soft-fp8", "--no-resource-gate", "--define", "X=1"])[0][0] - for flag in ("--no-device-libs", "--soft-fp8", "--no-resource-gate"): + for flag in ("--no-device-libs", "--soft-fp8", "--no-resource-gate", "--force-sdk"): self.assertIn(flag, cmd) self.assertEqual(values_of(cmd, "--clang"), ["dummy-clang"]) diff --git a/clean/tests/test_sdk.py b/clean/tests/test_sdk.py new file mode 100644 index 00000000..f8b49c70 --- /dev/null +++ b/clean/tests/test_sdk.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: MIT +"""The HIP SDK of the builds (clean/runner/sdk.py): AMD's official SDK folder, never one from the environment. No +compiler and no GPU: argument parsing, the refusal of another --rocm, clang lookup without PATH, the build log banner, +and a scan that no Python file of clean/ takes an SDK from ROCM_PATH.""" +import argparse +import os +import re +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(os.path.dirname(HERE)) +sys.path.insert(0, ROOT) + +from clean.runner import build, sdk # noqa: E402 + + +class SdkTest(unittest.TestCase): + def parse(self, argv): + ap = argparse.ArgumentParser() + sdk.add_arguments(ap) + return ap.parse_args(argv) + + def test_default_is_the_official_sdk_whatever_the_environment(self): + with patch.dict(os.environ, {"ROCM_PATH": "C:\\elsewhere", "HIP_PATH": "C:\\elsewhere", + "LLVM_PATH": "C:\\elsewhere", "HIP_DEVICE_LIB_PATH": "C:\\elsewhere"}): + a = self.parse([]) + self.assertEqual(sdk.resolve(a.rocm, a.force_sdk), sdk.OFFICIAL_ROCM) + + def test_another_sdk_needs_force(self): + a = self.parse(["--rocm", "C:\\elsewhere"]) + with self.assertRaises(SystemExit): + sdk.resolve(a.rocm, a.force_sdk) + a = self.parse(["--rocm", "C:\\elsewhere", "--force-sdk"]) + self.assertEqual(sdk.resolve(a.rocm, a.force_sdk), "C:\\elsewhere") + self.assertEqual(sdk.resolve(sdk.OFFICIAL_ROCM.upper() if sys.platform == "win32" else sdk.OFFICIAL_ROCM), + sdk.OFFICIAL_ROCM.upper() if sys.platform == "win32" else sdk.OFFICIAL_ROCM) + + def test_clang_never_from_path(self): + with tempfile.TemporaryDirectory() as empty, patch.object(build.shutil, "which", lambda *a: "C:\\elsewhere\\clang"): + with self.assertRaises(SystemExit): + build.find_clang(empty) + if sys.platform == "win32": + with self.assertRaises(SystemExit): + build.find_host_cxx(empty) + + def test_clang_from_the_sdk(self): + with tempfile.TemporaryDirectory() as root: + exe = "clang.exe" if sys.platform == "win32" else "clang" + os.makedirs(os.path.join(root, "bin")) + open(os.path.join(root, "bin", exe), "w").close() + self.assertEqual(build.find_clang(root), os.path.join(root, "bin", exe)) + + def test_banner(self): + line = sdk.banner(sdk.OFFICIAL_ROCM, clang="c", device_libs=True) + self.assertIn(f"HIP SDK: {sdk.OFFICIAL_ROCM} (official)", line) + self.assertIn("device libs: " + os.path.join(sdk.OFFICIAL_ROCM, "amdgcn", "bitcode"), line) + self.assertIn("NOT the official SDK", sdk.banner("C:\\elsewhere")) + + def test_build_py_ignores_rocm_path(self): + """build.py with no --rocm compiles with the official SDK's device libraries, ROCM_PATH set elsewhere.""" + cmds = [] + with tempfile.TemporaryDirectory() as out, patch.dict(os.environ, {"ROCM_PATH": "C:\\elsewhere"}), \ + patch.object(build, "run", lambda cmd, *a, **k: cmds.append(cmd)), \ + patch.object(build, "find_clang", lambda rocm=None: "clang"), \ + patch.object(build, "build_runtime", lambda *a, **k: None), \ + patch.object(build, "run_post_w64", lambda cmd, *a, **k: cmds.append(cmd)), \ + patch.object(build, "find_host_cxx", lambda rocm=None: "cxx"): + try: + build.main(["--out", out, "--no-model-k", "--no-resource-gate"]) + except (SystemExit, Exception): + pass + dev = [c for c in cmds if any(str(a).startswith("--rocm-device-lib-path=") for a in c)] + self.assertTrue(dev, "no device compile recorded") + for c in dev: + self.assertIn("--rocm-device-lib-path=" + sdk.device_lib_dir(sdk.OFFICIAL_ROCM), c) + self.assertFalse(any("elsewhere" in str(a) for a in c)) + + def test_no_rocm_path_defaults_left(self): + files = subprocess.run(["git", "ls-files", "clean/*.py", "clean/**/*.py"], cwd=ROOT, capture_output=True, + text=True).stdout.split() + pat = re.compile(r"os\.environ\.get\(\s*[\"']ROCM_PATH[\"']") + hits = [] + for f in files: + with open(os.path.join(ROOT, f), encoding="utf-8") as fh: + if pat.search(fh.read()): + hits.append(f) + self.assertEqual(hits, []) + + +if __name__ == "__main__": + unittest.main() From 68c192d9fd476fa1410d3f1e476a7c81165d43f2 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 03:37:24 -0500 Subject: [PATCH 2/4] Builds: one SDK policy boundary for every HIP compile and runtime load (review fixes) - sdk.py: official installs discovered by version (C:\Program Files\AMD\ROCm\, /opt/rocm-, /opt/rocm), the newest compatible 7.x by default; the compiler canonicalised to the SDK's own (bare and absolute names), anything else approved only with --force-sdk; check() is the boundary; runtime_library() the only HIP runtime source; custom folders logged by id only (banner, refusal, command echo via redact) - build.py: device_command calls sdk.check (every entry point: post_ablate, l_c32_check, k_port_check, fg_d3d12_check, ... pass it); find_clang / find_host_cxx through sdk.compiler; --clang validated; run() redacts - k_build.py: the compiler through sdk.compiler - hip.py: the runtime loaded only from the checked SDK by full path (no bare-name fallback); Hip(rocm_dir) under the same policy; fails closed - test_sdk.py: discovery (newest by numeric version, 7.2 missing, none), compilers, the boundary and the four entry points, build.py / k_build.py refusals, the runtime loader failing closed, redaction, every build.py compile with the SDK's clang and flags (build completes), the SDK-variable scan with negative controls (indexed, getenv, aliases, other variables) and a failing git call failing the test Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- BUILDING.md | 13 +- clean/k/runner/k_build.py | 5 +- clean/runner/build.py | 31 ++-- clean/runner/hip.py | 42 ++--- clean/runner/sdk.py | 212 ++++++++++++++++++++++--- clean/tests/test_sdk.py | 323 ++++++++++++++++++++++++++++++-------- 6 files changed, 489 insertions(+), 137 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index c9867a69..1d77cf82 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -33,7 +33,7 @@ these defaults: | `WG_TOOLCHAINS` | folder that holds `dxc-v1.10.2605.24\` (the DXC release zip, unpacked) and `rga-2.15\` (the RGA zip, unpacked) | `\toolchains` (ignored by git) | | `WG_DXC_DIR` | DXC's `bin\x64` folder (overrides `WG_TOOLCHAINS`) | `%WG_TOOLCHAINS%\dxc-v1.10.2605.24\bin\x64` | | `WG_RGA` | `rga.exe` (overrides `WG_TOOLCHAINS`) | `%WG_TOOLCHAINS%\rga-2.15\rga.exe` | -| `ROCM_PATH` | Not read by the build scripts: they use AMD's official HIP SDK folder `C:\Program Files\AMD\ROCm\7.2` (`clean/runner/sdk.py`); `--rocm` may name only that folder, another one needs `--force-sdk`. Every build prints the SDK, compiler and device-library folders it uses | - | +| `ROCM_PATH` | Not read by the build scripts: they use the newest official AMD HIP SDK 7.x install (`C:\Program Files\AMD\ROCm\`, `/opt/rocm-`; `clean/runner/sdk.py`); `--rocm` or `--clang` outside it needs `--force-sdk` | - | | `VSDEVCMD` | Visual Studio's `VsDevCmd.bat` (the kernel scripts build two small helper tools); for another edition e.g. `C:\Program Files\Microsoft Visual Studio\18\Professional\Common7\Tools\VsDevCmd.bat` (or `...\18\Enterprise\...`) | `C:\Program Files\Microsoft Visual Studio\18\Community\Common7\Tools\VsDevCmd.bat` | | `CXX`, `WG_K_RT_CXX`, `WG_K_RT_BUILD`, `WG_CONVERTER_EXE` | Model K table tests (`clean\k\tests\test_k_runtime_tables`, `test_k_runtime_pack`): the compiler is `%CXX%`, else clang++ / g++ on PATH, else the HIP SDK's clang++, else MSVC (`WG_K_RT_CXX=msvc` picks MSVC first); their build folder; an already built converter. With no compiler found the tests skip themselves with that reason | temporary folder; converter built from source | | `WG_BUILD_ROOT` | default output root of the scripts that have one | `\build` (ignored by git) | @@ -41,10 +41,13 @@ these defaults: | `WG_GPU_LOCK` | optional: a script run as `python --timeout -- ` around GPU jobs | unset: GPU jobs run directly | | `WRONGGPU_HIP_RUNTIME` | the HIP runtime the launcher's probe loads (tests: the stand-in of step 5.1) | the system's | -The build scripts use AMD's official HIP SDK in `C:\Program Files\AMD\ROCm\7.2` and never take an SDK, a -clang or a device-library folder from the environment (`ROCM_PATH`, `HIP_PATH`, `LLVM_PATH`, `HIP_DEVICE_LIB_PATH`, -`HIP_CLANG_PATH`) or from `PATH`. Remove those variables from the build's environment anyway: the vendor runtime -reads some of them. A build with another SDK needs `--rocm --force-sdk`, and its log names that folder. +The build scripts use the newest official AMD HIP SDK 7.x install (`C:\Program Files\AMD\ROCm\`; on +other systems `/opt/rocm-` or `/opt/rocm`), its own clang and device libraries, and load the HIP runtime +library from it by full path. They never take an SDK, a compiler or a library from the environment (`ROCM_PATH`, +`HIP_PATH`, `LLVM_PATH`, `HIP_DEVICE_LIB_PATH`, `HIP_CLANG_PATH`) or from `PATH`; every HIP compile checks this at +one place (`device_command`). Remove those variables from the build's environment anyway: the vendor runtime reads +some of them. Another SDK or compiler needs `--rocm ` / `--clang ` with `--force-sdk` (build.py, +k_build.py); every build log names the SDK, compiler and device-library folders, a custom one only by an id. Open a "x64 Native Tools Command Prompt for VS 2026" (or call `vcvars64.bat`) for every step below, at the repository root. The commands write into `out\`; any folder outside the repository works the same. diff --git a/clean/k/runner/k_build.py b/clean/k/runner/k_build.py index 6440bdce..45888464 100644 --- a/clean/k/runner/k_build.py +++ b/clean/k/runner/k_build.py @@ -75,9 +75,8 @@ def main(argv=None): a = ap.parse_args(argv) a.rocm = sdk.resolve(a.rocm, a.force_sdk) os.makedirs(a.out, exist_ok=True) - if not a.emu_only: - clang = a.clang or find_clang(a.rocm) - print(sdk.banner(a.rocm, clang=clang, device_libs=not a.no_device_libs), flush=True) + if not a.emu_only: # the SDK's own compiler; another one only with --force-sdk (device_command checks it again) + clang = sdk.compiler(a.rocm, a.clang, a.force_sdk) build_device(clang, a.rocm, a.out, not a.no_device_libs, a.report, io_wave64=not a.io_wave32) if a.emu or a.emu_only: run(emu_command(a.cxx, os.path.join(a.out, EMU_LIBRARY))) diff --git a/clean/runner/build.py b/clean/runner/build.py index b6b12dd5..824c2083 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -154,21 +154,17 @@ def set_model_k(on): def find_clang(rocm=None): - """The HIP SDK's clang (rocm: the SDK folder, default the official one; clean/runner/sdk.py). Never from PATH or an - environment variable: a clang elsewhere needs --clang.""" - rocm = rocm or sdk.OFFICIAL_ROCM - exe = "clang.exe" if sys.platform == "win32" else "clang" - for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): - p = os.path.join(rocm, sub, exe) - if os.path.exists(p): - return p - raise SystemExit(f"clang not found in the HIP SDK {rocm}; pass --clang") + """The HIP SDK's clang (rocm: the SDK folder, default the newest official install; clean/runner/sdk.py checks it). + Never from PATH or an environment variable.""" + return sdk.compiler(rocm or sdk.resolve(), None) def device_command(clang, rocm, src, out, device_libs=True, report=False, extra=(), soft_fp8=False, defines=None): # a fixed compilation-unit id from the output file's name, so the __hip_cuid_... symbol of a code object is the same # in every build folder (without -cuid, clang derives the id itself) cuid = "wg_" + re.sub(r"[^A-Za-z0-9_]", "_", os.path.splitext(os.path.basename(out))[0]) + # the SDK policy's boundary (clean/runner/sdk.py): every HIP compile of every entry point passes here + sdk.check(rocm or sdk.resolve(), clang) cmd = [clang, "-x", "hip", f"--offload-arch={ARCH}", "--cuda-device-only", "-nogpuinc", "-O3", "--no-gpu-bundle-output", f"-cuid={cuid}"] if device_libs: @@ -204,12 +200,7 @@ def find_host_cxx(rocm=None): systems: clang++ or g++ from PATH.""" exe = "clang++.exe" if sys.platform == "win32" else "clang++" if sys.platform == "win32": - rocm = rocm or sdk.OFFICIAL_ROCM - for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): - p = os.path.join(rocm, sub, exe) - if os.path.exists(p): - return p - raise SystemExit(f"clang++ not found in the HIP SDK {rocm}; pass --runtime-cxx") + return sdk.compiler(rocm or sdk.resolve(), None, default="clang++") p = shutil.which(exe) or shutil.which("g++") if p: return p @@ -358,7 +349,7 @@ def build_runtime(cxx, out_dir, embed=True): def run(cmd): - print(" ".join(f'"{c}"' if " " in c else c for c in cmd), flush=True) + print(sdk.redact(" ".join(f'"{c}"' if " " in c else c for c in cmd)), flush=True) # custom SDK paths by id only r = subprocess.run(cmd) if r.returncode != 0: raise SystemExit(f"build failed ({r.returncode})") @@ -494,6 +485,8 @@ def main(argv=None): args = ap.parse_args(argv) rocm_given = bool(args.rocm) # release runtime builds reuse pinned Model L code objects unless --rocm is given args.rocm = sdk.resolve(args.rocm, args.force_sdk) + if args.clang: # the SDK's own compiler (a bare name means the SDK's file); another one only with --force-sdk + args.clang = sdk.compiler(args.rocm, args.clang, args.force_sdk) for name in OPTIONAL_MODELS: set_model(name, not getattr(args, "no_model_" + name.lower())) set_model_l(args.model_l) @@ -510,7 +503,6 @@ def main(argv=None): if args.runtime_only: if MODEL_L and rocm_given: # Model L's code objects next to the given ones (release builds reuse pinned ones) clang = args.clang or find_clang(args.rocm) - print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) build_device_l(clang, args.rocm, args.out, device_libs=not args.no_device_libs) for arch in EXTRA_ARCHS: build_device_l(clang, args.rocm, os.path.join(args.out, arch), device_libs=not args.no_device_libs, @@ -522,9 +514,7 @@ def main(argv=None): if args.motion_outres_only: if not args.rocm and not args.no_device_libs: raise SystemExit("--rocm is required (or --no-device-libs for a compile check)") - clang = args.clang or find_clang(args.rocm) - print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) - build_motion_outres(clang, args) + build_motion_outres(args.clang or find_clang(args.rocm), args) for arch in EXTRA_ARCHS: sub = [sys.executable, os.path.abspath(__file__), "--out", os.path.join(args.out, arch), "--motion-outres-only"] @@ -545,7 +535,6 @@ def main(argv=None): if not args.rocm and not args.no_device_libs: raise SystemExit("--rocm is required (or --no-device-libs for a compile check)") clang = args.clang or find_clang(args.rocm) - print(sdk.banner(args.rocm, clang=clang, device_libs=not args.no_device_libs), flush=True) for src in SOURCES: out = os.path.join(args.out, os.path.splitext(src)[0] + ".co") run(device_command(clang, args.rocm, os.path.join(KERNEL_DIR, src), out, diff --git a/clean/runner/hip.py b/clean/runner/hip.py index bfc4613f..9721c7c9 100644 --- a/clean/runner/hip.py +++ b/clean/runner/hip.py @@ -4,7 +4,6 @@ Only the calls the runner needs: device memory, module loading, kernel launch, events. """ import ctypes -import glob import os import sys import warnings @@ -22,6 +21,7 @@ class HipError(RuntimeError): # HSA is removed, except HIP_PLATFORM and HIP_PATH_72, and ROCM_PATH is set to the SDK. A stray inherited variable # can point the vendor runtime at an unrelated folder (an LLVM_PATH outside the SDK made hipModuleLoadData crash). # Per process only: the user's and the machine's environment are not changed. +from clean.runner import sdk # noqa: E402 from clean.runner.sdk import OFFICIAL_ROCM as DEFAULT_ROCM # noqa: E402 (the official HIP SDK) HIP_ENV_KEEP = ("HIP_PLATFORM", "HIP_PATH_72") HIP_ENV_PARTS = ("HIP", "LLVM", "ROCM", "HSA") @@ -68,29 +68,29 @@ def _check_environment(rocm_dir): def _find_library(rocm_dir): - if sys.platform == "win32": - bindir = os.path.join(rocm_dir, "bin") - if os.path.isdir(bindir) and hasattr(os, "add_dll_directory"): - os.add_dll_directory(bindir) - names = ["amdhip64_7.dll", "amdhip64_6.dll", "amdhip64.dll"] - candidates = [os.path.join(bindir, n) for n in names] + names - else: - candidates = [os.path.join(rocm_dir, "lib", "libamdhip64.so")] - candidates += sorted(glob.glob(os.path.join(rocm_dir, "lib", "libamdhip64.so.*"))) - candidates += ["libamdhip64.so"] - errors = [] - for c in candidates: - if os.path.isabs(c) and not os.path.exists(c): - continue - try: - return ctypes.CDLL(c) - except OSError as e: - errors.append(f"{c}: {e}") - raise HipError("cannot load the HIP runtime library; tried:\n " + "\n ".join(errors or candidates)) + """The HIP runtime of the checked SDK (clean/runner/sdk.py runtime_library), loaded by its full path; never a bare + name searched by the loader. Fails closed.""" + try: + path = sdk.runtime_library(rocm_dir) + except SystemExit as e: + raise HipError(str(e)) from None + bindir = os.path.dirname(path) + if sys.platform == "win32" and hasattr(os, "add_dll_directory"): + os.add_dll_directory(bindir) + try: + return ctypes.CDLL(path) + except OSError as e: + raise HipError(f"cannot load the HIP runtime of the HIP SDK {sdk.label(rocm_dir)}: {e}") from None class Hip: - def __init__(self, rocm_dir, device=0): + def __init__(self, rocm_dir=None, device=0): + # rocm_dir: an official HIP SDK install (default the newest), or one approved with --force-sdk (sdk.approve) + rocm_dir = rocm_dir or sdk.resolve() + try: + sdk.check_rocm(rocm_dir) + except SystemExit as e: + raise HipError(str(e)) from None clean_process_environment(rocm_dir) _check_environment(rocm_dir) lib = self.lib = _find_library(rocm_dir) diff --git a/clean/runner/sdk.py b/clean/runner/sdk.py index e7ae63d2..e10e5310 100644 --- a/clean/runner/sdk.py +++ b/clean/runner/sdk.py @@ -1,30 +1,190 @@ # SPDX-License-Identifier: MIT -"""The HIP SDK of every build and HIP process of this project: AMD's official HIP SDK in its standard install folder, -never one taken from the environment (ROCM_PATH, HIP_PATH, LLVM_PATH, HIP_DEVICE_LIB_PATH or PATH). A script's --rocm -may name only that folder; another one needs --force-sdk. Every build prints the SDK, compiler and device-library -folders it uses (banner), so a build log shows where its code objects came from.""" +"""The HIP SDK policy of every build and HIP process of this project, enforced at one boundary: + +- the SDK is an official AMD HIP SDK install: C:\\Program Files\\AMD\\ROCm\\ on Windows, /opt/rocm- or + /opt/rocm on other systems, with a compatible version (major SUPPORTED_MAJOR); without --rocm the newest one + (numeric version order). Never an SDK from the environment (ROCM_PATH, HIP_PATH, ...) or from PATH; +- the compiler is the selected SDK's own (a bare name like "clang" means the SDK's bin\\clang.exe; an absolute path must + lie inside the SDK); +- anything else needs a deliberate override (--force-sdk), which approves exactly that SDK folder or compiler for this + process (approve); +- the HIP runtime library is loaded from the selected SDK only (runtime_library), never by a bare name the loader would + search for; +- logs name official folders as they are and custom ones only as "custom SDK (forced, id )" (label, redact). + +device_command / find_clang (clean/runner/build.py) and the HIP runtime loader (clean/runner/hip.py) call check(): +every entry point that compiles HIP code or loads the HIP runtime passes it, whichever script started it.""" +import hashlib import os +import re import sys -OFFICIAL_ROCM = r"C:\Program Files\AMD\ROCm\7.2" if sys.platform == "win32" else "/opt/rocm" +SUPPORTED_MAJOR = 7 +WINDOWS_ROOT = r"C:\Program Files\AMD\ROCm" +LINUX_ROOT = "/opt" +FALLBACK = os.path.join(WINDOWS_ROOT, "7.2") if sys.platform == "win32" else "/opt/rocm" +_VERSION = re.compile(r"^(\d+)\.(\d+)(?:\.(\d+))?$") + +_approved = set() # normalized folders / compilers approved with --force-sdk in this process +_bannered = set() def _norm(path): - return os.path.normcase(os.path.normpath(os.path.abspath(path))) + return os.path.normcase(os.path.normpath(os.path.realpath(os.path.abspath(path)))) + + +def _exe(name): + return name + ".exe" if sys.platform == "win32" and not name.lower().endswith(".exe") else name + + +def _valid_sdk(path): + return (os.path.isfile(os.path.join(path, "bin", _exe("clang"))) and + os.path.isdir(os.path.join(path, "amdgcn", "bitcode"))) + + +def official_installs(windows_root=None, linux_root=None, windows=None): + """[(version tuple, folder)] of the official installs, newest first (numeric order); only folders with the SDK's + clang and device libraries and a compatible version.""" + windows = sys.platform == "win32" if windows is None else windows + out = [] + if windows: + root = windows_root or WINDOWS_ROOT + names = [(n, os.path.join(root, n)) for n in (os.listdir(root) if os.path.isdir(root) else [])] + else: + root = linux_root or LINUX_ROOT + names = [(n[len("rocm-"):], os.path.join(root, n)) for n in (os.listdir(root) if os.path.isdir(root) else []) + if n.startswith("rocm-")] + for ver, path in names: + m = _VERSION.match(ver) + if m and int(m.group(1)) == SUPPORTED_MAJOR and _valid_sdk(path): + out.append((tuple(int(g or 0) for g in m.groups()), path)) + out.sort(reverse=True) + if not windows: # /opt/rocm: official when it is (or links to) one of them + link = os.path.join(root, "rocm") + if os.path.isdir(link) and any(_norm(link) == _norm(p) for _, p in out): + out.insert(0, (out[0][0], link)) + return out + + +def default_rocm(): + """The newest official compatible install, or None.""" + found = official_installs() + return found[0][1] if found else None + + +OFFICIAL_ROCM = default_rocm() or FALLBACK # for scripts that need one folder name (the default of their --rocm) + + +def path_id(path): + return hashlib.sha256(_norm(path).encode("utf-8")).hexdigest()[:10] def is_official(path): - return bool(path) and _norm(path) == _norm(OFFICIAL_ROCM) + return bool(path) and any(_norm(path) == _norm(p) for _, p in official_installs()) + + +def label(path): + """How a log names an SDK or compiler folder: official ones as they are, others by an id only.""" + if not path: + return "none" + if is_official(path) or any(_inside(path, p) for _, p in official_installs()): + return path + return f"custom (forced, id {path_id(path)})" + + +def approve(path): + """--force-sdk: accept this exact SDK folder or compiler in this process.""" + _approved.add(_norm(path)) + + +def _inside(path, root): + try: + a, r = _norm(path), _norm(root) + return os.path.commonpath([a, r]) == r + except ValueError: + return False def resolve(rocm=None, force=False): - """The SDK folder for a --rocm value: empty or None = the official SDK; another folder only with force.""" + """The SDK folder for a --rocm value: empty or None = the newest official install; another folder only with + force (then approved for this process).""" if not rocm: - return OFFICIAL_ROCM - if is_official(rocm) or force: + d = default_rocm() + if not d: + raise SystemExit(f"no official HIP SDK {SUPPORTED_MAJOR}.x found ({WINDOWS_ROOT}\\ or " + f"/opt/rocm-)") + return d + if is_official(rocm): + return rocm + if force: + approve(rocm) return rocm - raise SystemExit(f"--rocm {rocm!r} is not the official HIP SDK ({OFFICIAL_ROCM}); pass --force-sdk to build with " - "it anyway") + raise SystemExit(f"--rocm names a folder that is not an official HIP SDK {SUPPORTED_MAJOR}.x install (id " + f"{path_id(rocm)}); pass --force-sdk to build with it anyway") + + +def check_rocm(rocm): + """The boundary for an SDK folder: official, or approved with --force-sdk.""" + if not rocm or not (is_official(rocm) or _norm(rocm) in _approved): + raise SystemExit(f"HIP SDK folder {label(rocm) if rocm else '(none)'} is neither an official install nor " + "approved with --force-sdk") + return rocm + + +def compiler(rocm, name_or_path=None, force=False, default="clang"): + """The canonical compiler for an SDK: None = the SDK's own `default`; a bare name = that file in the SDK's bin + folder; a path = itself, when inside the SDK. Anything else only with force (then approved).""" + check_rocm(rocm) + if not name_or_path: + name_or_path = default + bare = os.path.basename(name_or_path) == name_or_path + if bare: + for sub in ("bin", os.path.join("lib", "llvm", "bin"), os.path.join("llvm", "bin")): + p = os.path.join(rocm, sub, _exe(name_or_path)) + if os.path.isfile(p): + return p + if force: # the deliberate override: whatever that name is, approved as such + import shutil + p = shutil.which(name_or_path) or name_or_path + approve(p) + return p + raise SystemExit(f"compiler {name_or_path} not found in the HIP SDK {label(rocm)} (another one needs " + "--force-sdk and a path)") + if _inside(name_or_path, rocm) and os.path.isfile(name_or_path): + return name_or_path + if force: + approve(name_or_path) + return name_or_path + raise SystemExit(f"compiler {label(name_or_path)} is not inside the HIP SDK {label(rocm)}; pass --force-sdk to " + "use it anyway") + + +def check(rocm, clang): + """The build boundary (device_command): the SDK folder and the compiler pass the policy; prints the banner once per + toolchain.""" + check_rocm(rocm) + if not (_inside(clang, rocm) or _norm(clang) in _approved): + raise SystemExit(f"compiler {label(clang)} is not inside the HIP SDK {label(rocm)} and not approved with " + "--force-sdk") + key = (_norm(rocm), _norm(clang)) + if key not in _bannered: + _bannered.add(key) + print(banner(rocm, clang=clang), flush=True) + + +def runtime_library(rocm): + """The HIP runtime library of the checked SDK (never a bare name for the loader to search); raises when missing.""" + check_rocm(rocm) + if sys.platform == "win32": + cands = [os.path.join(rocm, "bin", n) for n in ("amdhip64_7.dll", "amdhip64_6.dll", "amdhip64.dll")] + else: + lib = os.path.join(rocm, "lib") + cands = [os.path.join(lib, "libamdhip64.so")] + sorted( + os.path.join(lib, n) for n in (os.listdir(lib) if os.path.isdir(lib) else []) if n.startswith("libamdhip64.so.")) + for c in cands: + if os.path.isfile(c): + return c + raise SystemExit(f"no HIP runtime library in the HIP SDK {label(rocm)}") def device_lib_dir(rocm): @@ -32,18 +192,28 @@ def device_lib_dir(rocm): def add_arguments(ap, help_suffix=""): - """--rocm (default: the official SDK, never an environment variable) and --force-sdk.""" - ap.add_argument("--rocm", default="", help=f"HIP SDK folder (default and only accepted value without --force-sdk: " - f"{OFFICIAL_ROCM}){help_suffix}") - ap.add_argument("--force-sdk", action="store_true", help="accept a --rocm other than the official HIP SDK") + """--rocm (default: the newest official install, never an environment variable) and --force-sdk.""" + ap.add_argument("--rocm", default="", help="HIP SDK folder (default: the newest official install; another " + f"folder needs --force-sdk){help_suffix}") + ap.add_argument("--force-sdk", action="store_true", + help="accept a --rocm or compiler outside the official HIP SDK installs") def banner(rocm, clang=None, cxx=None, device_libs=True): - """One line naming the SDK, the compilers and the device-library folder of a build.""" - parts = [f"HIP SDK: {rocm} ({'official' if is_official(rocm) else 'NOT the official SDK: forced'})"] + """One line naming the SDK, the compilers and the device-library folder of a build (custom ones by id only).""" + parts = [f"HIP SDK: {label(rocm)}" + (" (official)" if is_official(rocm) else "")] if clang: - parts.append(f"clang: {clang}") + parts.append(f"clang: {label(clang)}") if cxx: - parts.append(f"host C++: {cxx}") - parts.append(f"device libs: {device_lib_dir(rocm) if device_libs else 'none (-nogpulib)'}") + parts.append(f"host C++: {label(cxx) if _inside(cxx, rocm) or _norm(cxx) in _approved else cxx}") + parts.append(f"device libs: {label(device_lib_dir(rocm)) if device_libs else 'none (-nogpulib)'}") return "; ".join(parts) + + +def redact(text): + """text with every approved custom folder or compiler replaced by its label (both slash forms).""" + for p in sorted(_approved, key=len, reverse=True): + lab = f"custom (forced, id {hashlib.sha256(p.encode('utf-8')).hexdigest()[:10]})" + for form in {p, p.replace("\\", "/"), p.replace("/", "\\")}: + text = re.sub(re.escape(form), lab, text, flags=re.IGNORECASE) + return text diff --git a/clean/tests/test_sdk.py b/clean/tests/test_sdk.py index f8b49c70..2c0e9ec5 100644 --- a/clean/tests/test_sdk.py +++ b/clean/tests/test_sdk.py @@ -1,8 +1,20 @@ # SPDX-License-Identifier: MIT -"""The HIP SDK of the builds (clean/runner/sdk.py): AMD's official SDK folder, never one from the environment. No -compiler and no GPU: argument parsing, the refusal of another --rocm, clang lookup without PATH, the build log banner, -and a scan that no Python file of clean/ takes an SDK from ROCM_PATH.""" -import argparse +"""The HIP SDK policy (clean/runner/sdk.py) and its boundary. No compiler and no GPU: fake SDK folders, stubbed +subprocesses. + +- discovery: the newest compatible official install by numeric version, 7.2 missing, several versions, none; +- the compiler: bare and absolute names canonicalised to the SDK's own, anything else only with --force-sdk; +- the boundary: device_command / find_clang refuse an unapproved SDK or compiler, so the entry points post_ablate, + l_c32_check, k_port_check and fg_d3d12_check (and build.py, k_build.py) cannot compile with one; +- the HIP runtime: only the checked SDK's library, by full path, never a bare name; fails closed; +- logs: custom folders appear by id only (banner, refusal, command echo); +- build.py: with ROCM_PATH and friends pointing elsewhere, every compile it starts uses the SDK's clang and its device + libraries, and the build completes; +- a scan of clean/ and tools/: no Python file reads an SDK variable (ROCM_PATH, HIP_PATH, LLVM_PATH, + HIP_DEVICE_LIB_PATH, HIP_CLANG_PATH, HIPFIRE_ROCM_PATH, ROCM_HOME) by os.environ[...], os.getenv, .get() or an + alias; the scanner's own negative controls; a git failure fails the test.""" +import contextlib +import io import os import re import subprocess @@ -15,79 +27,258 @@ ROOT = os.path.dirname(os.path.dirname(HERE)) sys.path.insert(0, ROOT) -from clean.runner import build, sdk # noqa: E402 +from clean.runner import build, hip, sdk # noqa: E402 + +EXE = ".exe" if sys.platform == "win32" else "" + + +def fake_sdk(path, runtime=True): + """A folder that looks like a HIP SDK install: bin/clang, bin/clang++, device libraries, the runtime library.""" + os.makedirs(os.path.join(path, "bin"), exist_ok=True) + os.makedirs(os.path.join(path, "amdgcn", "bitcode"), exist_ok=True) + for n in ("clang", "clang++"): + open(os.path.join(path, "bin", n + EXE), "w").close() + if runtime: + if sys.platform == "win32": + open(os.path.join(path, "bin", "amdhip64_7.dll"), "w").close() + else: + os.makedirs(os.path.join(path, "lib"), exist_ok=True) + open(os.path.join(path, "lib", "libamdhip64.so"), "w").close() + return path + +class Env(unittest.TestCase): + """Each test: a fake official root with versions, the policy's process state reset.""" -class SdkTest(unittest.TestCase): - def parse(self, argv): - ap = argparse.ArgumentParser() - sdk.add_arguments(ap) - return ap.parse_args(argv) + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.root = os.path.join(self.tmp.name, "AMD", "ROCm") + self.custom = fake_sdk(os.path.join(self.tmp.name, "custom-sdk")) + self.patches = [patch.object(sdk, "WINDOWS_ROOT", self.root), patch.object(sdk, "LINUX_ROOT", self.root), + patch.object(sdk, "_approved", set()), patch.object(sdk, "_bannered", set())] + for p in self.patches: + p.start() - def test_default_is_the_official_sdk_whatever_the_environment(self): - with patch.dict(os.environ, {"ROCM_PATH": "C:\\elsewhere", "HIP_PATH": "C:\\elsewhere", - "LLVM_PATH": "C:\\elsewhere", "HIP_DEVICE_LIB_PATH": "C:\\elsewhere"}): - a = self.parse([]) - self.assertEqual(sdk.resolve(a.rocm, a.force_sdk), sdk.OFFICIAL_ROCM) + def tearDown(self): + for p in self.patches: + p.stop() + self.tmp.cleanup() - def test_another_sdk_needs_force(self): - a = self.parse(["--rocm", "C:\\elsewhere"]) + def install(self, *versions): + out = {} + for v in versions: + name = v if sys.platform == "win32" else "rocm-" + v + out[v] = fake_sdk(os.path.join(self.root, name)) + return out + + +class Discovery(Env): + def test_newest_compatible_by_numeric_version(self): + got = self.install("7.2", "7.10", "7.0.1", "6.4", "8.0") + os.makedirs(os.path.join(self.root, "7.9" if sys.platform == "win32" else "rocm-7.9")) # incomplete: no clang + self.assertEqual(sdk.default_rocm(), got["7.10"]) + self.assertEqual([v for v, _ in sdk.official_installs()], [(7, 10, 0), (7, 2, 0), (7, 0, 1)]) + self.assertTrue(sdk.is_official(got["7.2"])) + self.assertFalse(sdk.is_official(got["6.4"]) or sdk.is_official(got["8.0"])) + + def test_7_2_missing(self): + got = self.install("7.1") + self.assertEqual(sdk.resolve(""), got["7.1"]) + self.assertEqual(sdk.resolve(got["7.1"]), got["7.1"]) # an official one needs no force + + def test_none_installed(self): + self.assertIsNone(sdk.default_rocm()) with self.assertRaises(SystemExit): - sdk.resolve(a.rocm, a.force_sdk) - a = self.parse(["--rocm", "C:\\elsewhere", "--force-sdk"]) - self.assertEqual(sdk.resolve(a.rocm, a.force_sdk), "C:\\elsewhere") - self.assertEqual(sdk.resolve(sdk.OFFICIAL_ROCM.upper() if sys.platform == "win32" else sdk.OFFICIAL_ROCM), - sdk.OFFICIAL_ROCM.upper() if sys.platform == "win32" else sdk.OFFICIAL_ROCM) - - def test_clang_never_from_path(self): - with tempfile.TemporaryDirectory() as empty, patch.object(build.shutil, "which", lambda *a: "C:\\elsewhere\\clang"): + sdk.resolve("") + + def test_environment_is_ignored(self): + got = self.install("7.2") + with patch.dict(os.environ, {k: self.custom for k in ("ROCM_PATH", "HIP_PATH", "LLVM_PATH", + "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH")}): + self.assertEqual(sdk.resolve(""), got["7.2"]) + + +class Compiler(Env): + def test_bare_and_absolute_names(self): + s = self.install("7.2")["7.2"] + own = os.path.join(s, "bin", "clang" + EXE) + self.assertEqual(sdk.compiler(s, "clang"), own) + self.assertEqual(sdk.compiler(s, "clang" + EXE), own) + self.assertEqual(sdk.compiler(s, own), own) + self.assertEqual(sdk.compiler(s, None), own) + + def test_other_compilers_need_force(self): + s = self.install("7.2")["7.2"] + other = os.path.join(self.custom, "bin", "clang" + EXE) + with patch("shutil.which", lambda *a: other): + with self.assertRaises(SystemExit): + sdk.compiler(s, other) + with self.assertRaises(SystemExit): + sdk.compiler(s, "gcc") # bare, not in the SDK (PATH would have one) + self.assertEqual(sdk.compiler(s, other, force=True), other) + sdk.check(s, other) # approved now + + def test_custom_sdk_needs_force(self): + with self.assertRaises(SystemExit): + sdk.resolve(self.custom) + self.assertEqual(sdk.resolve(self.custom, force=True), self.custom) + sdk.check_rocm(self.custom) + + +class Boundary(Env): + def no_subprocess(self): + def fail(*a, **k): + raise AssertionError("a compiler was started") + return contextlib.ExitStack(), fail + + def assert_refused(self, fn): + with patch.object(build.subprocess, "run", side_effect=AssertionError("compiler started")), \ + patch.object(subprocess, "run", side_effect=AssertionError("compiler started")), \ + patch.dict(os.environ, {}), contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit): + fn() + + def test_device_command_and_find_clang(self): + s = self.install("7.2")["7.2"] + own = os.path.join(s, "bin", "clang" + EXE) + with contextlib.redirect_stdout(io.StringIO()) as log: + cmd = build.device_command(own, s, "k.hip", "k.co") + self.assertIn("--rocm-device-lib-path=" + sdk.device_lib_dir(s), cmd) + self.assertIn("HIP SDK: " + s, log.getvalue()) + for bad in (lambda: build.device_command(own, self.custom, "k.hip", "k.co"), + lambda: build.device_command(os.path.join(self.custom, "bin", "clang" + EXE), s, "k.hip", "k.co"), + lambda: build.find_clang(self.custom)): with self.assertRaises(SystemExit): - build.find_clang(empty) - if sys.platform == "win32": - with self.assertRaises(SystemExit): - build.find_host_cxx(empty) - - def test_clang_from_the_sdk(self): - with tempfile.TemporaryDirectory() as root: - exe = "clang.exe" if sys.platform == "win32" else "clang" - os.makedirs(os.path.join(root, "bin")) - open(os.path.join(root, "bin", exe), "w").close() - self.assertEqual(build.find_clang(root), os.path.join(root, "bin", exe)) - - def test_banner(self): - line = sdk.banner(sdk.OFFICIAL_ROCM, clang="c", device_libs=True) - self.assertIn(f"HIP SDK: {sdk.OFFICIAL_ROCM} (official)", line) - self.assertIn("device libs: " + os.path.join(sdk.OFFICIAL_ROCM, "amdgcn", "bitcode"), line) - self.assertIn("NOT the official SDK", sdk.banner("C:\\elsewhere")) - - def test_build_py_ignores_rocm_path(self): - """build.py with no --rocm compiles with the official SDK's device libraries, ROCM_PATH set elsewhere.""" + bad() + + def test_entry_points_refuse_a_custom_sdk(self): + self.install("7.2") + out = os.path.join(self.tmp.name, "out") + from clean.runner import post_ablate + from clean.l.runner import l_c32_check + from clean.k.dx12 import k_port_check + from clean.fg.runner import fg_d3d12_check + clang = os.path.join(self.custom, "bin", "clang" + EXE) + self.assert_refused(lambda: post_ablate.build_all(out, self.custom, None, ["base"])) + self.assert_refused(lambda: post_ablate.build_all(out, self.custom, clang, ["base"])) + self.assert_refused(lambda: l_c32_check.compile_kernels(out, self.custom)) + self.assert_refused(lambda: l_c32_check.compile_kernels(out, self.custom, clang=clang)) + self.assert_refused(lambda: k_port_check.prepare(out, self.custom, list(k_port_check.FAMILY_KERNELS)[:1])) + self.assert_refused(lambda: fg_d3d12_check.prepare(out, self.custom, [])) + + def test_build_and_k_build_refuse_without_force(self): + self.install("7.2") + from clean.k.runner import k_build + out = os.path.join(self.tmp.name, "out") + self.assert_refused(lambda: build.main(["--out", out, "--rocm", self.custom])) + self.assert_refused(lambda: build.main(["--out", out, "--clang", os.path.join(self.custom, "bin", "clang" + EXE)])) + self.assert_refused(lambda: k_build.main(["--out", out, "--rocm", self.custom])) + self.assert_refused(lambda: k_build.main(["--out", out, "--clang", os.path.join(self.custom, "bin", "clang" + EXE)])) + + +class Runtime(Env): + def test_only_the_sdk_library(self): + s = self.install("7.2")["7.2"] + loaded = [] + with patch.object(hip.ctypes, "CDLL", lambda p: loaded.append(p) or object()), \ + patch.object(hip.os, "add_dll_directory", lambda p: None, create=True): + hip._find_library(s) + self.assertEqual(len(loaded), 1) + self.assertTrue(os.path.isabs(loaded[0]) and sdk._inside(loaded[0], s)) + + def test_fails_closed(self): + s = fake_sdk(os.path.join(self.root, "7.3" if sys.platform == "win32" else "rocm-7.3"), runtime=False) + loaded = [] + with patch.object(hip.ctypes, "CDLL", lambda p: loaded.append(p) or object()): + with self.assertRaises(hip.HipError): + hip._find_library(s) # official, but no runtime library: no bare-name fallback + with self.assertRaises(hip.HipError): + hip._find_library(self.custom) # not approved + with self.assertRaises(hip.HipError): + hip.Hip(self.custom) + self.assertEqual(loaded, []) + + +class Logs(Env): + def test_custom_paths_by_id_only(self): + self.install("7.2") + with self.assertRaises(SystemExit) as e: + sdk.resolve(self.custom) + self.assertNotIn(self.custom, str(e.exception)) + sdk.resolve(self.custom, force=True) + clang = os.path.join(self.custom, "bin", "clang" + EXE) + line = sdk.banner(self.custom, clang=clang) + self.assertNotIn(os.path.normcase(self.custom), os.path.normcase(line)) + self.assertIn("custom (forced, id ", line) + with contextlib.redirect_stdout(io.StringIO()) as log, patch.object(build.subprocess, "run", + return_value=subprocess.CompletedProcess([], 0)): + build.run(build.device_command(clang, self.custom, "k.hip", "k.co")) + self.assertNotIn(os.path.normcase(self.custom), os.path.normcase(log.getvalue())) + self.assertNotIn(os.path.normcase(self.custom).replace("\\", "/"), os.path.normcase(log.getvalue())) + + +class BuildPy(Env): + def test_every_compile_uses_the_sdk(self): + """build.py (no --rocm, SDK variables elsewhere): every device compile it starts uses the SDK's clang and both + SDK flags, every Model M source is compiled, and the build completes.""" + s = self.install("7.2")["7.2"] + own = os.path.join(s, "bin", "clang" + EXE) cmds = [] - with tempfile.TemporaryDirectory() as out, patch.dict(os.environ, {"ROCM_PATH": "C:\\elsewhere"}), \ - patch.object(build, "run", lambda cmd, *a, **k: cmds.append(cmd)), \ - patch.object(build, "find_clang", lambda rocm=None: "clang"), \ + env = {k: self.custom for k in ("ROCM_PATH", "HIP_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH")} + with tempfile.TemporaryDirectory() as out, patch.dict(os.environ, env), \ + patch.object(build, "run", lambda cmd, *a, **k: cmds.append(list(cmd))), \ + patch.object(build, "run_post_w64", lambda cmd, *a, **k: cmds.append(list(cmd))), \ patch.object(build, "build_runtime", lambda *a, **k: None), \ - patch.object(build, "run_post_w64", lambda cmd, *a, **k: cmds.append(cmd)), \ - patch.object(build, "find_host_cxx", lambda rocm=None: "cxx"): - try: - build.main(["--out", out, "--no-model-k", "--no-resource-gate"]) - except (SystemExit, Exception): - pass - dev = [c for c in cmds if any(str(a).startswith("--rocm-device-lib-path=") for a in c)] - self.assertTrue(dev, "no device compile recorded") - for c in dev: - self.assertIn("--rocm-device-lib-path=" + sdk.device_lib_dir(sdk.OFFICIAL_ROCM), c) - self.assertFalse(any("elsewhere" in str(a) for a in c)) - - def test_no_rocm_path_defaults_left(self): - files = subprocess.run(["git", "ls-files", "clean/*.py", "clean/**/*.py"], cwd=ROOT, capture_output=True, - text=True).stdout.split() - pat = re.compile(r"os\.environ\.get\(\s*[\"']ROCM_PATH[\"']") + contextlib.redirect_stdout(io.StringIO()): + build.main(["--out", out, "--no-model-k", "--no-resource-gate", "--no-runtime"]) + self.assertTrue(cmds) + for c in cmds: + self.assertEqual(c[0], own, c[:2]) + self.assertIn(f"--rocm-path={s}", c) + self.assertIn("--rocm-device-lib-path=" + sdk.device_lib_dir(s), c) + self.assertFalse(any(os.path.normcase(self.custom) in os.path.normcase(str(a)) for a in c)) + outs = {os.path.basename(c[c.index("-o") + 1]) for c in cmds} + for src in build.SOURCES: + self.assertIn(os.path.splitext(src)[0] + ".co", outs) + + +SDK_VARS = ("ROCM_PATH", "HIP_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH", "HIPFIRE_ROCM_PATH", + "ROCM_HOME") +_V = "|".join(SDK_VARS) +READ = [re.compile(rf"""getenv\(\s*["']({_V})["']"""), # os.getenv("X"), getenv("X") + re.compile(rf"""\.get\(\s*["']({_V})["']"""), # os.environ.get / env.get / alias.get + re.compile(rf"""\b\w+\[\s*["']({_V})["']\s*\](?!\s*=[^=])""")] # environ["X"] / alias["X"], not stores +# files whose reads are of a dictionary the code itself built or checks (no environment value used as an SDK): +# hip.py's cleaned copy, and test_hip_env.py, which reads back the value the cleanup set +ALLOWED = {"clean/runner/hip.py", "clean/tests/test_hip_env.py"} + + +def scan_text(text): + """The SDK-variable reads in Python source (lines that delete a variable are not reads).""" + return [m.group(0) for line in text.splitlines() if not line.lstrip().startswith("del ") + for p in READ for m in p.finditer(line)] + + +class Scan(unittest.TestCase): + def test_scanner_negative_controls(self): + for src in ('os.environ["ROCM_PATH"]', "os.getenv('HIP_PATH')", 'env = os.environ\nx = env.get("LLVM_PATH")', + 'from os import environ\ny = environ["HIP_DEVICE_LIB_PATH"]', 'getenv("HIPFIRE_ROCM_PATH", "")', + 'e.get("ROCM_HOME")'): + self.assertTrue(scan_text(src), src) + for src in ('os.environ["ROCM_PATH"] = sdk', 'del os.environ["HIP_PATH"]', 'for k in ("HIP_PATH",): env.pop(k)'): + self.assertFalse(scan_text(src), src) + + def test_no_sdk_variable_read(self): + r = subprocess.run(["git", "ls-files", "--", "clean", "tools"], cwd=ROOT, capture_output=True, text=True, + check=True) + files = [f for f in r.stdout.split() if f.endswith(".py")] + self.assertGreater(len(files), 200) # a scan of nothing is not a pass hits = [] for f in files: + if f in ALLOWED or f == "clean/tests/test_sdk.py": + continue with open(os.path.join(ROOT, f), encoding="utf-8") as fh: - if pat.search(fh.read()): - hits.append(f) + hits += [f"{f}: {h}" for h in scan_text(fh.read())] self.assertEqual(hits, []) From 0a3fded95d57704ee4395e98be73a3caaa9d6747 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:33:15 -0500 Subject: [PATCH 3/4] Builds: --force-sdk on every --rocm CLI, /opt/rocm discovery, redacted compiler output (re-check fixes) - every CLI with --rocm offers --force-sdk and resolves --rocm (and --clang) through the SDK policy right after parsing (sdk.add_force_argument / sdk.apply); children that get --rocm also get --force-sdk - Linux: the newest /opt/rocm- wins; /opt/rocm counts with the version it points to; a standalone /opt/rocm counts with its .info/version (ranked last without one) - build.run / run_post_w64 / post_ablate / post.build_wave64: command echo, compiler output and the post_w64 report go through sdk.redact; redaction matches relative, resolved, 8.3 and home-folder spellings, either separator, any case - test_build_cuid approves its dummy SDK; new tests for all of the above - the native runtime's and installer's HIP runtime lookup stay out of scope (BUILDING.md, issue draft for 0.6.6) Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- BUILDING.md | 9 +- clean/dx12/compare.py | 5 +- clean/dx12/tests/matched_stage_bench.py | 7 +- clean/dx12/tests/native_direct_build.py | 8 +- clean/dx12/tests/native_direct_dll_check.py | 7 +- clean/dx12/tests/native_game_build.py | 8 +- clean/dx12/tests/native_helpers_check.py | 3 + clean/dx12/tests/native_qualification.py | 7 +- clean/dx12/tests/post_tf_d3d12_check.py | 3 + clean/dx12/tests/run_device_test.py | 3 + clean/dx12/tests/run_native_device_test.py | 5 +- clean/dx12/tests/tuned_decoder_check.py | 5 +- clean/dx12/tests/tuned_post_check.py | 15 +- clean/dx12/tests/tuned_swin_check.py | 5 +- clean/dx12/tests/variant_check.py | 5 +- clean/dx12/time_frames.py | 8 +- clean/equiv/model_m.py | 5 +- clean/fg/runner/fg_d3d12_check.py | 3 + clean/fg/runner/fg_net1_gpu.py | 3 + clean/fg/runner/fg_net1_mma.py | 3 + clean/fg/runner/fg_wmma_probe.py | 3 + clean/k/dx12/k_port_check.py | 3 + clean/k/runner/k_bench.py | 3 + clean/k/runner/k_reset_bench.py | 3 + clean/l/runner/l_c32_check.py | 3 + clean/l/runner/l_d3d12_check.py | 3 + clean/l/runner/l_d3d12_fallback_check.py | 3 + clean/l/runner/l_d3d12_ops_check.py | 3 + clean/l/runner/l_enc_check.py | 3 + clean/l/runner/l_enclin_check.py | 3 + clean/l/runner/l_fold_check.py | 3 + clean/l/runner/l_frame_check.py | 5 +- clean/l/runner/l_linfuse_check.py | 3 + clean/l/runner/l_ops_check.py | 3 + clean/l/runner/l_post_check.py | 3 + clean/l/runner/l_post_edge_check.py | 3 + clean/runner/bench_bneck.py | 3 + clean/runner/build.py | 26 ++- clean/runner/build_exact.py | 3 + clean/runner/frame_ab.py | 3 + clean/runner/m_codeobjects_compare.py | 12 +- clean/runner/mlp_probe.py | 3 + clean/runner/motion_hi_check.py | 3 + clean/runner/output_filter_original_check.py | 3 + clean/runner/output_filter_time.py | 3 + clean/runner/output_quality_gpu_check.py | 3 + clean/runner/output_quality_runtime_check.py | 3 + clean/runner/overlap_probe.py | 3 + clean/runner/perf_bench.py | 3 + clean/runner/post.py | 3 +- clean/runner/post_ablate.py | 13 +- clean/runner/post_bench.py | 3 + clean/runner/post_exact.py | 8 + clean/runner/post_fused_bench.py | 3 + clean/runner/post_hdr_check.py | 3 + clean/runner/post_sizes.py | 3 + clean/runner/post_tf_runtime.py | 3 + clean/runner/post_traffic.py | 3 + clean/runner/post_variants.py | 3 + clean/runner/profile_stages.py | 3 + clean/runner/profile_v2.py | 3 + clean/runner/rates.py | 3 + clean/runner/sdk.py | 138 +++++++++++-- clean/runner/selftest.py | 3 + clean/runner/sweep.py | 3 + clean/tests/test_build_cuid.py | 24 ++- clean/tests/test_sdk.py | 207 ++++++++++++++++++- 67 files changed, 605 insertions(+), 62 deletions(-) diff --git a/BUILDING.md b/BUILDING.md index 1d77cf82..83e2c83a 100644 --- a/BUILDING.md +++ b/BUILDING.md @@ -42,12 +42,15 @@ these defaults: | `WRONGGPU_HIP_RUNTIME` | the HIP runtime the launcher's probe loads (tests: the stand-in of step 5.1) | the system's | The build scripts use the newest official AMD HIP SDK 7.x install (`C:\Program Files\AMD\ROCm\`; on -other systems `/opt/rocm-` or `/opt/rocm`), its own clang and device libraries, and load the HIP runtime -library from it by full path. They never take an SDK, a compiler or a library from the environment (`ROCM_PATH`, +other systems `/opt/rocm-` or `/opt/rocm`, which counts with the version it points to, or as a standalone +install with its own), its own clang and device libraries, and the Python tools load the HIP runtime library from it by +full path. They never take an SDK, a compiler or a library from the environment (`ROCM_PATH`, `HIP_PATH`, `LLVM_PATH`, `HIP_DEVICE_LIB_PATH`, `HIP_CLANG_PATH`) or from `PATH`; every HIP compile checks this at one place (`device_command`). Remove those variables from the build's environment anyway: the vendor runtime reads some of them. Another SDK or compiler needs `--rocm ` / `--clang ` with `--force-sdk` (build.py, -k_build.py); every build log names the SDK, compiler and device-library folders, a custom one only by an id. +k_build.py and every script with `--rocm`); every build log names the SDK, compiler and device-library folders, a custom +one only by an id, also in compiler output. The native runtime's and the installer's own HIP runtime lookup are not +part of this rule yet (planned for 0.6.6). Open a "x64 Native Tools Command Prompt for VS 2026" (or call `vcvars64.bat`) for every step below, at the repository root. The commands write into `out\`; any folder outside the repository works the same. diff --git a/clean/dx12/compare.py b/clean/dx12/compare.py index 5d2010a5..dcd8e43a 100644 --- a/clean/dx12/compare.py +++ b/clean/dx12/compare.py @@ -339,7 +339,9 @@ def main(argv=None): ap.add_argument("--candidate-build", help="optional separate native/hybrid build; --build remains the HIP oracle") ap.add_argument("--kernels", required=True, help="WG_DX12_KERNELS for the hybrid run (names as launched, or all)") ap.add_argument("--backend", choices=("dx12", "d3d12"), default="dx12", help="candidate device; HIP stays the oracle") - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--models", default=DEFAULT_MODELS, help="work dir for the synthetic model") ap.add_argument("--frames", type=int, default=4) ap.add_argument("--quick", action="store_true", help="only the small odd-size case") @@ -350,6 +352,7 @@ def main(argv=None): ap.add_argument("--json") ap.add_argument("--trace", help="new trace path; defaults to a unique directory inside the build") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.frames < 1: ap.error("--frames must be positive") if a.probe_unused_capacity and a.frames < 2: diff --git a/clean/dx12/tests/matched_stage_bench.py b/clean/dx12/tests/matched_stage_bench.py index 5784cecb..764153d2 100644 --- a/clean/dx12/tests/matched_stage_bench.py +++ b/clean/dx12/tests/matched_stage_bench.py @@ -19,6 +19,9 @@ import subprocess import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) # the repository, for clean.runner +from clean.runner import sdk # noqa: E402 (the HIP SDK policy: --rocm, --force-sdk) + HERE = Path(__file__).resolve().parent ROOT = HERE.parents[2] @@ -252,7 +255,8 @@ def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--build', required=True, help='frozen complete HIP/native payload build') p.add_argument('--out', required=True, help='private build and evidence directory') - p.add_argument('--rocm', default=r'C:\Program Files\AMD\ROCm\7.2') + p.add_argument('--rocm', default='') + sdk.add_force_argument(p) p.add_argument('--prepare', action='store_true', help='CPU-only compile/input/model generation') p.add_argument('--rebuild', action='store_true', help='with --prepare, recompile while preserving verified original inputs/model') p.add_argument('--run', action='store_true', help='GPU work: use shared lock + activity wrapper') @@ -268,6 +272,7 @@ def main(): p.add_argument('--run-tag', default='paired-r0') p.add_argument('--clock-co', default=os.environ.get('WG_CLOCK_CO', ''), help='original synthetic cycle/realtime probe') a = p.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if not a.prepare and not a.run: p.error('choose --prepare and/or --run') if not (math.isfinite(a.warm) and a.warm >= 3 and math.isfinite(a.minimum_window_seconds) and diff --git a/clean/dx12/tests/native_direct_build.py b/clean/dx12/tests/native_direct_build.py index 19e79875..88fa6c44 100644 --- a/clean/dx12/tests/native_direct_build.py +++ b/clean/dx12/tests/native_direct_build.py @@ -12,15 +12,21 @@ import os from pathlib import Path import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) # the repository, for clean.runner +from clean.runner import sdk # noqa: E402 (the HIP SDK policy: --rocm, --force-sdk) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--out', required=True) - p.add_argument('--rocm', default=r'C:\Program Files\AMD\ROCm\7.2') + p.add_argument('--rocm', default='') + sdk.add_force_argument(p) p.add_argument('--model-l', action='store_true', help='also Model L (WG_MODEL_L, clean/l/runtime): the test then runs it with its last argument L') a = p.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk clean = Path(__file__).resolve().parents[2] rt = clean / 'runtime' out = Path(a.out).resolve() diff --git a/clean/dx12/tests/native_direct_dll_check.py b/clean/dx12/tests/native_direct_dll_check.py index 18815118..093311ec 100644 --- a/clean/dx12/tests/native_direct_dll_check.py +++ b/clean/dx12/tests/native_direct_dll_check.py @@ -14,6 +14,9 @@ import subprocess import sys +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) # the repository, for clean.runner +from clean.runner import sdk # noqa: E402 (the HIP SDK policy: --rocm, --force-sdk) + COMMON = ('swin_block_c64_nc', 'patch_merge_c64', 'swin_block_c96_nc', 'patch_merge_c96', 'bneck/swin_bneck6_nc', 'swin_block_c96_add_ex_nc', 'expand_c96', @@ -193,7 +196,8 @@ def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--prepare', action='store_true') p.add_argument('--out', required=True, help='New private directory; never overwrites evidence') - p.add_argument('--rocm', default=r'C:\Program Files\AMD\ROCm\7.2') + p.add_argument('--rocm', default='') + sdk.add_force_argument(p) p.add_argument('--exe') p.add_argument('--build') p.add_argument('--model') @@ -202,6 +206,7 @@ def main(): p.add_argument('--frames', type=int, default=8) p.add_argument('--output-format', choices=('half', 'float'), default='half') a = p.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.frames < 6 or min(a.shape) < 1: p.error('positive shapes and at least six frames required') return build(a) if a.prepare else run(a) diff --git a/clean/dx12/tests/native_game_build.py b/clean/dx12/tests/native_game_build.py index aa67b7a1..0d3dbb81 100644 --- a/clean/dx12/tests/native_game_build.py +++ b/clean/dx12/tests/native_game_build.py @@ -12,13 +12,19 @@ import os from pathlib import Path import subprocess +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[3])) # the repository, for clean.runner +from clean.runner import sdk # noqa: E402 (the HIP SDK policy: --rocm, --force-sdk) def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--out', required=True) - p.add_argument('--rocm', default=r'C:\Program Files\AMD\ROCm\7.2') + p.add_argument('--rocm', default='') + sdk.add_force_argument(p) a = p.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk clean = Path(__file__).resolve().parents[2] rt = clean / 'runtime' out = Path(a.out).resolve() diff --git a/clean/dx12/tests/native_helpers_check.py b/clean/dx12/tests/native_helpers_check.py index c53c14a6..8e11c054 100644 --- a/clean/dx12/tests/native_helpers_check.py +++ b/clean/dx12/tests/native_helpers_check.py @@ -250,7 +250,9 @@ def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True, type=Path, help="verified HIP reference build containing frame.co") ap.add_argument("--out", required=True, type=Path, help="private owned output directory") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=ROCM) + sdk.add_force_argument(ap) ap.add_argument("--n", default=65539, type=int) ap.add_argument("--seed", default=281, type=int) phase = ap.add_mutually_exclusive_group() @@ -258,6 +260,7 @@ def main(): phase.add_argument("--run-only", action="store_true", help="run existing compiled artifacts; no builds inside GPU lock") ap.add_argument("--verify-mutation", action="store_true", help="also run an intentionally corrupted HLSL candidate") args = ap.parse_args() + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk if not 1 <= args.n < (1 << 24): ap.error("--n must be in [1, 2^24)") args.out = args.out.resolve() diff --git a/clean/dx12/tests/native_qualification.py b/clean/dx12/tests/native_qualification.py index 276117f0..0fc96f28 100644 --- a/clean/dx12/tests/native_qualification.py +++ b/clean/dx12/tests/native_qualification.py @@ -234,13 +234,16 @@ def main(argv=None): ap.add_argument("--native-build", help="standalone native bundle; --build remains the HIP reference") ap.add_argument("--models", required=True) ap.add_argument("--out", required=True, help="new evidence directory (never overwritten)") - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--frames", type=int, default=8) ap.add_argument("--include-4k", action="store_true", help="also qualify 1920x1080 -> 3840x2160 Performance geometry") ap.add_argument("--probe-unused-capacity", action="store_true", help="also poison C capacity during the differential check") ap.add_argument("--worker", action="store_true", help=argparse.SUPPRESS) ap.add_argument("--skip-compare", action="store_true", help="diagnostic only: native process gate without differential run") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk native_build = a.native_build or a.build if a.frames < 4: ap.error("at least four frames are needed to exercise the second reset") @@ -269,7 +272,7 @@ def main(argv=None): if not a.skip_compare: cmd = [sys.executable, str(DX12 / "compare.py"), "--build", a.build, "--models", str(models), "--candidate-build", native_build, - "--rocm", a.rocm, "--backend", "d3d12", "--kernels", "all", "--frames", str(a.frames), + "--rocm", a.rocm, *sdk.forward(a), "--backend", "d3d12", "--kernels", "all", "--frames", str(a.frames), "--json", str(out / "compare.json"), "--trace", str(out / "compare-launches.txt")] if a.include_4k: cmd.append("--include-4k") diff --git a/clean/dx12/tests/post_tf_d3d12_check.py b/clean/dx12/tests/post_tf_d3d12_check.py index 34edc0ac..7dbe73d3 100644 --- a/clean/dx12/tests/post_tf_d3d12_check.py +++ b/clean/dx12/tests/post_tf_d3d12_check.py @@ -490,7 +490,9 @@ def tex_judge(path, w16, ow, oh, w32=None, outp_path=None): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--kernels", default=",".join(KINDS)) @@ -499,6 +501,7 @@ def main(argv=None): ap.add_argument("--ldr", action="store_true", help="with --store: Model L's LDR direct filter (LDR_STORE_FRAMES)") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if not a.run: return 0 kernels = [k for k in a.kernels.split(",") if k] diff --git a/clean/dx12/tests/run_device_test.py b/clean/dx12/tests/run_device_test.py index ab40a3bd..202c5a90 100644 --- a/clean/dx12/tests/run_device_test.py +++ b/clean/dx12/tests/run_device_test.py @@ -21,9 +21,12 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--out", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=ROCM) + sdk.add_force_argument(ap) ap.add_argument("--build-only", action="store_true", help="build without reserving or using the GPU") a = ap.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk dst = os.path.join(a.out, "dx12") os.makedirs(dst, exist_ok=True) asm = build_dx12.build_asm_tool(a.out) diff --git a/clean/dx12/tests/run_native_device_test.py b/clean/dx12/tests/run_native_device_test.py index 8599a266..96a6a7a7 100644 --- a/clean/dx12/tests/run_native_device_test.py +++ b/clean/dx12/tests/run_native_device_test.py @@ -23,10 +23,13 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--out", required=True) - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2", help="host clang++ compiler only") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="", help="host clang++ compiler only") + sdk.add_force_argument(ap) ap.add_argument("--build-only", action="store_true") ap.add_argument("--run-only", action="store_true") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.build_only and a.run_only: ap.error("choose at most one of --build-only and --run-only") out = Path(a.out).resolve() diff --git a/clean/dx12/tests/tuned_decoder_check.py b/clean/dx12/tests/tuned_decoder_check.py index 7d865efd..76c8de06 100644 --- a/clean/dx12/tests/tuned_decoder_check.py +++ b/clean/dx12/tests/tuned_decoder_check.py @@ -27,12 +27,15 @@ def main(argv=None): ap.add_argument("--build", required=True) ap.add_argument("--native-build", required=True) ap.add_argument("--out", required=True) - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--seeds", type=int, default=4) group = ap.add_mutually_exclusive_group() group.add_argument("--build-only", action="store_true") group.add_argument("--run-only", action="store_true") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.seeds < 1: ap.error("--seeds must be positive") out = Path(a.out).resolve() diff --git a/clean/dx12/tests/tuned_post_check.py b/clean/dx12/tests/tuned_post_check.py index 8154ad84..272e7b85 100644 --- a/clean/dx12/tests/tuned_post_check.py +++ b/clean/dx12/tests/tuned_post_check.py @@ -126,11 +126,13 @@ def build(a, out, fields): copy_runtime(Path(a.native_build), native, False) src = target / "post_instrumented.hip" src.write_text(hip_text) - cmd = HB.device_command(str(Path(a.rocm) / "bin" / "clang.exe"), a.rocm, str(src), + cmd = HB.device_command(HB.find_clang(a.rocm), a.rocm, str(src), str(hip / "post_w64__orig.co"), defines={"WG_LZ_PROFILE": 2, "WG_DBG_K": field}, extra=["-mwavefrontsize64", "-I", str(CLEAN / "kernels")]) - subprocess.run(cmd, cwd=out, check=True, stdout=subprocess.DEVNULL) - report["commands"].append(cmd) + r = subprocess.run(cmd, cwd=out, capture_output=True, text=True, errors="replace") + if r.returncode: # a custom (forced) SDK appears by id only, here and in the record + raise SystemExit(f"instrumented post build failed ({r.returncode}): " + HB.sdk.redact(r.stderr)) + report["commands"].append([HB.sdk.redact(c) for c in cmd]) # Some builds retain a second platform-named code object; keep the private pair coherent. alias = hip / "gfx1201.post_w64__orig.co" if alias.exists(): @@ -186,7 +188,9 @@ def main(argv=None): ap.add_argument("--native-build") ap.add_argument("--out", required=True) ap.add_argument("--models") - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--fields", default=",".join(map(str, range(15)))) ap.add_argument("--frames", type=int, default=4) group = ap.add_mutually_exclusive_group() @@ -195,6 +199,7 @@ def main(argv=None): ap.add_argument("--worker-output", help=argparse.SUPPRESS) ap.add_argument("--backend", choices=("hip", "d3d12"), default="d3d12", help=argparse.SUPPRESS) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk for key in ("HIP_PATH", "ROCM_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH"): os.environ.pop(key, None) if a.worker_output: @@ -232,7 +237,7 @@ def main(argv=None): env = os.environ.copy() env["WG_LAUNCH_TRACE"] = str(trace) cmd = [sys.executable, str(Path(__file__).resolve()), "--build", str(target / label), "--out", str(out), - "--models", str(model), "--rocm", a.rocm, "--frames", str(a.frames), + "--models", str(model), "--rocm", a.rocm, *sdk.forward(a), "--frames", str(a.frames), "--backend", "hip" if label == "hip" else "d3d12", "--worker-output", str(target / (label + ".npz"))] result = subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, timeout=900) (target / (label + ".log")).write_text(result.stdout + result.stderr) diff --git a/clean/dx12/tests/tuned_swin_check.py b/clean/dx12/tests/tuned_swin_check.py index 8f046e09..b70b41da 100644 --- a/clean/dx12/tests/tuned_swin_check.py +++ b/clean/dx12/tests/tuned_swin_check.py @@ -93,12 +93,15 @@ def main(argv=None): ap.add_argument("--build", required=True, help="HIP reference code objects") ap.add_argument("--native-build", required=True) ap.add_argument("--out", required=True, help="fresh private work directory, reused only with --run-only") - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--seeds", type=int, default=16) group = ap.add_mutually_exclusive_group() group.add_argument("--build-only", action="store_true") group.add_argument("--run-only", action="store_true") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.seeds < 1: ap.error("--seeds must be positive") out = Path(a.out).resolve() diff --git a/clean/dx12/tests/variant_check.py b/clean/dx12/tests/variant_check.py index e25cbe16..96bba1fc 100644 --- a/clean/dx12/tests/variant_check.py +++ b/clean/dx12/tests/variant_check.py @@ -28,9 +28,12 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--models", default=os.path.join(os.environ.get("WG_BUILD_ROOT", os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))), "build")), "dx12-m", "work", "compare_models")) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk for k in ("HIP_PATH", "ROCM_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH"): os.environ.pop(k, None) if (Path(a.models) / "model.wgcm").exists(): diff --git a/clean/dx12/time_frames.py b/clean/dx12/time_frames.py index a2203f4f..9fcdbcbb 100644 --- a/clean/dx12/time_frames.py +++ b/clean/dx12/time_frames.py @@ -164,7 +164,9 @@ def main(argv=None): ap.add_argument("--native-build", help="optional separate native bundle") ap.add_argument("--shape", choices=("1440p", "4k-performance"), default="1440p") ap.add_argument("--frame-only", action="store_true", help="kernel experiment: preuploaded whole frames only; omit host transfer phases") - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--models", default=DEFAULT_MODELS) ap.add_argument("--configs", default="hip,all") ap.add_argument("--rounds", type=int, default=2) @@ -173,6 +175,7 @@ def main(argv=None): ap.add_argument("--json") ap.add_argument("--child", help=argparse.SUPPRESS) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.warm < 3 or a.repeats < 1 or a.rounds < 1: ap.error("at least 3 seconds of warm-up and positive repeats/rounds are required") for k in ("HIP_PATH", "ROCM_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH"): @@ -184,7 +187,8 @@ def main(argv=None): failures = [] for r in range(a.rounds): for c in configs: - cmd = [sys.executable, os.path.abspath(__file__), "--build", a.build, "--rocm", a.rocm, "--models", a.models, + cmd = [sys.executable, os.path.abspath(__file__), "--build", a.build, "--rocm", a.rocm, *sdk.forward(a), + "--models", a.models, "--repeats", str(a.repeats), "--warm", str(a.warm), "--child", c] cmd += ["--shape", a.shape] if a.native_build: diff --git a/clean/equiv/model_m.py b/clean/equiv/model_m.py index 1e6a25cf..0037d377 100644 --- a/clean/equiv/model_m.py +++ b/clean/equiv/model_m.py @@ -545,7 +545,9 @@ def main(argv=None): for name in ("dump", "stage", "replay"): p = sub.add_parser(name) p.add_argument("--build", required=True) - p.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + p.add_argument("--rocm", default="") + sdk.add_force_argument(p) p.add_argument("--model", required=True) p.add_argument("--out", required=True) if name in ("dump", "replay"): @@ -580,6 +582,7 @@ def main(argv=None): i.add_argument("--stage", required=True) i.add_argument("--out", required=True) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.cmd == "inputs": spec = stage_inputs_from_dump(a.dump, a.frame, a.stage) rel = os.path.relpath(os.path.abspath(a.dump), os.path.dirname(os.path.abspath(a.out))) diff --git a/clean/fg/runner/fg_d3d12_check.py b/clean/fg/runner/fg_d3d12_check.py index 1a89326d..90cf9a79 100644 --- a/clean/fg/runner/fg_d3d12_check.py +++ b/clean/fg/runner/fg_d3d12_check.py @@ -342,13 +342,16 @@ def read(path): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--kernels", default=",".join(KINDS)) ap.add_argument("--json", default=None) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk kernels = a.kernels.split(",") if a.prepare: prepare(a.build, a.rocm, kernels) diff --git a/clean/fg/runner/fg_net1_gpu.py b/clean/fg/runner/fg_net1_gpu.py index 005081d5..a448fea3 100644 --- a/clean/fg/runner/fg_net1_gpu.py +++ b/clean/fg/runner/fg_net1_gpu.py @@ -453,13 +453,16 @@ def setup(w, hgt): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--timing", action="store_true") ap.add_argument("--model", default=None, help="a converter output file of the preset-FG map (execute-only)") ap.add_argument("--json", default=None) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.prepare: K.prepare(a.build, a.rocm, KERNELS) res = None diff --git a/clean/fg/runner/fg_net1_mma.py b/clean/fg/runner/fg_net1_mma.py index 5ebcf153..85f7789b 100644 --- a/clean/fg/runner/fg_net1_mma.py +++ b/clean/fg/runner/fg_net1_mma.py @@ -526,7 +526,9 @@ def prepare(build, rocm): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--timing", action="store_true") @@ -534,6 +536,7 @@ def main(argv=None): ap.add_argument("--model", default=None) ap.add_argument("--json", default=None) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk paths = a.paths.split(",") if any(p not in PATHS for p in paths): ap.error("paths: %s" % ",".join(PATHS)) diff --git a/clean/fg/runner/fg_wmma_probe.py b/clean/fg/runner/fg_wmma_probe.py index 65171722..cc9da404 100644 --- a/clean/fg/runner/fg_wmma_probe.py +++ b/clean/fg/runner/fg_wmma_probe.py @@ -610,13 +610,16 @@ def prepare(build, rocm): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--backends", default="hip,d3d12") ap.add_argument("--scale", type=float, default=1.0) ap.add_argument("--json", default=None) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.prepare: prepare(a.build, a.rocm) if a.run: diff --git a/clean/k/dx12/k_port_check.py b/clean/k/dx12/k_port_check.py index 4f0aae70..19b5d8e2 100644 --- a/clean/k/dx12/k_port_check.py +++ b/clean/k/dx12/k_port_check.py @@ -1002,13 +1002,16 @@ def read(n): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--family", default="expo,warm") ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk families = a.family.split(",") bad = [f for f in families if f not in FAMILY_KERNELS] if bad: diff --git a/clean/k/runner/k_bench.py b/clean/k/runner/k_bench.py index f868a1a3..c64dc347 100644 --- a/clean/k/runner/k_bench.py +++ b/clean/k/runner/k_bench.py @@ -292,7 +292,9 @@ def print_bench(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--backend", default="hip", choices=("hip", "emu")) ap.add_argument("--sizes", nargs="*", default=["1080p", "1440p", "4k"]) ap.add_argument("--s", type=float, default=DEFAULT_S, @@ -325,6 +327,7 @@ def main(argv=None): ap.add_argument("--all-flags", action="store_true") ap.add_argument("--json", help="write all results here") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk dev = open_device(a.backend, a.build, a.rocm) funcs = KFuncs(dev, a.build, a.backend) params = None diff --git a/clean/k/runner/k_reset_bench.py b/clean/k/runner/k_reset_bench.py index 1773e935..5862ba23 100644 --- a/clean/k/runner/k_reset_bench.py +++ b/clean/k/runner/k_reset_bench.py @@ -108,11 +108,14 @@ def run(build, rocm, rounds=5, frames=100, sizes=((1920, 1080), (2560, 1440))): def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--json", required=True) ap.add_argument("--rounds", type=int, default=5) ap.add_argument("--frames", type=int, default=100) a = ap.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.rounds < 1 or a.frames < 20: ap.error("rounds >= 1 and frames >= 20 required") result = run(a.build, a.rocm, a.rounds, a.frames) diff --git a/clean/l/runner/l_c32_check.py b/clean/l/runner/l_c32_check.py index 6878b1b9..ce02bcbe 100644 --- a/clean/l/runner/l_c32_check.py +++ b/clean/l/runner/l_c32_check.py @@ -291,7 +291,9 @@ def run(build, rocm, seeds=(89,)): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--clang") ap.add_argument("--compile", action="store_true") ap.add_argument("--compile-only", action="store_true") @@ -299,6 +301,7 @@ def main(argv=None): ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--json", required=True) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.build, exist_ok=True) if a.compile or a.compile_only: compile_kernels(a.build, a.rocm, a.with_swin_co, a.clang) diff --git a/clean/l/runner/l_d3d12_check.py b/clean/l/runner/l_d3d12_check.py index 03d18083..d1904bff 100644 --- a/clean/l/runner/l_d3d12_check.py +++ b/clean/l/runner/l_d3d12_check.py @@ -235,7 +235,9 @@ def run(build, rocm, seeds, kernels): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) @@ -243,6 +245,7 @@ def main(argv=None): ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.build, exist_ok=True) if a.prepare: print("built", prepare(a.build, a.rocm)) diff --git a/clean/l/runner/l_d3d12_fallback_check.py b/clean/l/runner/l_d3d12_fallback_check.py index cb75b155..afde1f41 100644 --- a/clean/l/runner/l_d3d12_fallback_check.py +++ b/clean/l/runner/l_d3d12_fallback_check.py @@ -220,12 +220,15 @@ def judge(name, case, want, prefix=""): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--ldr", action="store_true", help="the LDR route: l_post_recon_ldr, l_post_filter_ldr") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if not a.run: return 0 res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr) diff --git a/clean/l/runner/l_d3d12_ops_check.py b/clean/l/runner/l_d3d12_ops_check.py index 0d897cfa..0f7fcb87 100644 --- a/clean/l/runner/l_d3d12_ops_check.py +++ b/clean/l/runner/l_d3d12_ops_check.py @@ -552,7 +552,9 @@ def negative(nm): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) @@ -560,6 +562,7 @@ def main(argv=None): ap.add_argument("--no-control", action="store_true", help="skip Model M's swin_block_c96_nc control") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk kernels = [k for k in a.kernels.split(",") if k] + ([] if a.no_control else list(CONTROL_KINDS)) os.makedirs(a.build, exist_ok=True) if a.prepare: diff --git a/clean/l/runner/l_enc_check.py b/clean/l/runner/l_enc_check.py index 31f997d0..b04b6b14 100644 --- a/clean/l/runner/l_enc_check.py +++ b/clean/l/runner/l_enc_check.py @@ -117,10 +117,13 @@ def run(build, rocm, seeds=(89,)): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--json", required=True) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) LR.write_l_record(res, a.json) print({"total": res["total"], "pass": res["pass"]}) diff --git a/clean/l/runner/l_enclin_check.py b/clean/l/runner/l_enclin_check.py index c80d8cb6..c8f58aaa 100644 --- a/clean/l/runner/l_enclin_check.py +++ b/clean/l/runner/l_enclin_check.py @@ -179,12 +179,15 @@ def read(n): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.build, exist_ok=True) if a.prepare: C32.compile_kernels(a.build, a.rocm, with_swin=True) diff --git a/clean/l/runner/l_fold_check.py b/clean/l/runner/l_fold_check.py index fe66c509..13255101 100644 --- a/clean/l/runner/l_fold_check.py +++ b/clean/l/runner/l_fold_check.py @@ -261,12 +261,15 @@ def read(n): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.build, exist_ok=True) if a.prepare: C32.compile_kernels(a.build, a.rocm, with_swin=True) diff --git a/clean/l/runner/l_frame_check.py b/clean/l/runner/l_frame_check.py index 5febcf53..053f03a0 100644 --- a/clean/l/runner/l_frame_check.py +++ b/clean/l/runner/l_frame_check.py @@ -942,7 +942,9 @@ def unwritten(k, n): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) - ap.add_argument("--rocm", default=r"C:\Program Files\AMD\ROCm\7.2") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk + ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) g = ap.add_mutually_exclusive_group(required=True) g.add_argument("--prepare", action="store_true") g.add_argument("--run", action="store_true") @@ -961,6 +963,7 @@ def main(argv=None): ap.add_argument("--ldr", action="store_true", help="--run / --compare / --compare-d3d12: Model L with LDR colour (records *_ldr.pkl)") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk global LDR LDR = a.ldr if (a.model or a.map) and not a.prepare: diff --git a/clean/l/runner/l_linfuse_check.py b/clean/l/runner/l_linfuse_check.py index 97bc2132..2df41729 100644 --- a/clean/l/runner/l_linfuse_check.py +++ b/clean/l/runner/l_linfuse_check.py @@ -179,12 +179,15 @@ def read(n): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--seeds", type=int, default=2) ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.build, exist_ok=True) if a.prepare: C32.compile_kernels(a.build, a.rocm, with_swin=True) diff --git a/clean/l/runner/l_ops_check.py b/clean/l/runner/l_ops_check.py index 065b1d56..d5549313 100644 --- a/clean/l/runner/l_ops_check.py +++ b/clean/l/runner/l_ops_check.py @@ -308,10 +308,13 @@ def run(build, rocm, seeds=(89,)): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--json", required=True) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds))) LR.write_l_record(res, a.json) print({k: v for k, v in res.items() if k.endswith("_total") or k == "pass"}) diff --git a/clean/l/runner/l_post_check.py b/clean/l/runner/l_post_check.py index ef68a443..14499bf8 100644 --- a/clean/l/runner/l_post_check.py +++ b/clean/l/runner/l_post_check.py @@ -244,11 +244,14 @@ def run(build, rocm, seeds=(89,), ldr=False): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--seeds", type=int, default=1, help="run the case list with this many seeds (89, 90, ...)") ap.add_argument("--ldr", action="store_true", help="the LDR form l_post_recon_ldr against the LDR oracle") ap.add_argument("--json", required=True) a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk res = run(a.build, a.rocm, tuple(range(89, 89 + a.seeds)), ldr=a.ldr) with open(a.json, "w") as f: json.dump(res, f, indent=1) diff --git a/clean/l/runner/l_post_edge_check.py b/clean/l/runner/l_post_edge_check.py index 35d989f0..3bd7fb4a 100644 --- a/clean/l/runner/l_post_edge_check.py +++ b/clean/l/runner/l_post_edge_check.py @@ -364,12 +364,15 @@ def run(build, rocm, part="all"): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--prepare", action="store_true") ap.add_argument("--run", action="store_true") ap.add_argument("--part", choices=("all", "replay", "oracle"), default="all") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if a.prepare: prepare(a.build, a.rocm) if a.run: diff --git a/clean/runner/bench_bneck.py b/clean/runner/bench_bneck.py index f8167a62..fa510a2d 100644 --- a/clean/runner/bench_bneck.py +++ b/clean/runner/bench_bneck.py @@ -108,7 +108,9 @@ def format_report(rep): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True, help="output directory of build.py") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variants", help="comma-separated bmega* builds (default: every one built)") ap.add_argument("--groups", help="comma-separated workgroup counts for the megakernel (default: the variant's)") @@ -118,6 +120,7 @@ def main(argv=None): ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--json", help="also write the report as JSON") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk size = tuple(int(v) for v in args.size.lower().split("x")) if args.size else None groups = [int(g) for g in args.groups.split(",")] if args.groups else None reps = [] diff --git a/clean/runner/build.py b/clean/runner/build.py index 824c2083..0e329c1d 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -348,9 +348,22 @@ def build_runtime(cxx, out_dir, embed=True): run(runtime_command(cxx, os.path.join(out_dir, runtime_library_name()), embedded)) +def echo(cmd): + """Print a command line; a custom (forced) SDK or compiler appears by id only.""" + print(sdk.redact(" ".join(f'"{c}"' if " " in c else c for c in cmd)), flush=True) + + +def run_captured(cmd): + """Run a compiler with its output captured (so diagnostics naming a custom SDK can be redacted).""" + return subprocess.run(cmd, capture_output=True, text=True, encoding="utf-8", errors="replace") + + def run(cmd): - print(sdk.redact(" ".join(f'"{c}"' if " " in c else c for c in cmd)), flush=True) # custom SDK paths by id only - r = subprocess.run(cmd) + echo(cmd) + r = run_captured(cmd) + sys.stdout.write(sdk.redact(r.stdout or "")) + sys.stderr.write(sdk.redact(r.stderr or "")) + sys.stdout.flush() if r.returncode != 0: raise SystemExit(f"build failed ({r.returncode})") @@ -413,12 +426,13 @@ def resource_report(text): def run_post_w64(cmd, out_dir, gate=True, report_name=POST_W64_REPORT, gates=None): """Build post_w64.co with the resource report (written to POST_W64_REPORT) and check POST_W64_GATE (or `gates`).""" - print(" ".join(f'"{c}"' if " " in c else c for c in cmd), flush=True) - r = subprocess.run(cmd, stderr=subprocess.PIPE, text=True) + echo(cmd) + r = run_captured(cmd) + sys.stdout.write(sdk.redact(r.stdout or "")) with open(os.path.join(out_dir, report_name), "w") as f: - f.write(r.stderr) + f.write(sdk.redact(r.stderr or "")) if r.returncode != 0: - sys.stderr.write(r.stderr) + sys.stderr.write(sdk.redact(r.stderr or "")) raise SystemExit(f"build failed ({r.returncode})") rep = resource_report(r.stderr) bad = [] diff --git a/clean/runner/build_exact.py b/clean/runner/build_exact.py index 96341951..7e14fc68 100644 --- a/clean/runner/build_exact.py +++ b/clean/runner/build_exact.py @@ -32,11 +32,14 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--a-build", required=True) ap.add_argument("--b-build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--models", required=True, help="work directory for the synthetic models") ap.add_argument("--frames", type=int, default=24) ap.add_argument("--json") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk root = Path(args.models) if (root / "model.wgcm").exists(): raise ValueError("refusing to write synthetic parameters into a converted-model directory") diff --git a/clean/runner/frame_ab.py b/clean/runner/frame_ab.py index 6493486a..18074b09 100644 --- a/clean/runner/frame_ab.py +++ b/clean/runner/frame_ab.py @@ -51,7 +51,9 @@ def synthetic_frame(rw, rh, seed=1): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default="") + sdk.add_force_argument(ap) ap.add_argument("--model", required=True, help="model folder (model.wgcm or raw/)") ap.add_argument("--configs", nargs="+", default=["base"]) ap.add_argument("--frame", help="npz with one frame's inputs (else synthetic)") @@ -64,6 +66,7 @@ def main(argv=None): ap.add_argument("--backend", default="hip", help="hip, or emu (host emulator, for checks only)") ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk rw, rh = (int(v) for v in a.render.lower().split("x")) ow, oh = (int(v) for v in a.output.lower().split("x")) diff --git a/clean/runner/m_codeobjects_compare.py b/clean/runner/m_codeobjects_compare.py index 954c289f..bcdf5ad8 100644 --- a/clean/runner/m_codeobjects_compare.py +++ b/clean/runner/m_codeobjects_compare.py @@ -41,8 +41,9 @@ def hashes(d): return out -def build(tree, out, rocm, env): - r = subprocess.run([sys.executable, "-B", os.path.join("clean", "runner", "build.py"), "--rocm", rocm, "--out", out, +def build(tree, out, rocm, env, forward=()): + r = subprocess.run([sys.executable, "-B", os.path.join("clean", "runner", "build.py"), "--rocm", rocm, *forward, + "--out", out, "--no-runtime", "--variants"], cwd=tree, env=env, capture_output=True, text=True, creationflags=NO_WINDOW) if r.returncode: @@ -56,9 +57,12 @@ def main(argv=None): g.add_argument("--base", help="git ref of the base (checked out as a worktree under --work)") g.add_argument("--base-tree", help="an existing checkout of the base") ap.add_argument("--work", required=True, help="folder for the base worktree and both build outputs") + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=DEFAULT_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--json") a = ap.parse_args(argv) + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk os.makedirs(a.work, exist_ok=True) base_tree = a.base_tree if a.base: @@ -73,8 +77,8 @@ def main(argv=None): env["TMP"] = env["TEMP"] = os.path.join(a.work, "tmp") os.makedirs(env["TMP"], exist_ok=True) outs = {"base": os.path.join(a.work, "base"), "head": os.path.join(a.work, "head")} - build(base_tree, outs["base"], a.rocm, env) - build(ROOT, outs["head"], a.rocm, env) + build(base_tree, outs["base"], a.rocm, env, sdk.forward(a)) # a base without --force-sdk refuses it + build(ROOT, outs["head"], a.rocm, env, sdk.forward(a)) hb, hh = hashes(outs["base"]), hashes(outs["head"]) differ = sorted(k for k in set(hb) | set(hh) if hb.get(k) != hh.get(k)) res = {"base": a.base or base_tree, "base_objects": len(hb), "head_objects": len(hh), "differ": differ} diff --git a/clean/runner/mlp_probe.py b/clean/runner/mlp_probe.py index de160770..6702ac38 100644 --- a/clean/runner/mlp_probe.py +++ b/clean/runner/mlp_probe.py @@ -98,9 +98,12 @@ def format_table(res): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--json") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk res = run(args.build, args.rocm) print(format_table(res)) if args.json: diff --git a/clean/runner/motion_hi_check.py b/clean/runner/motion_hi_check.py index 7a52c743..0927111d 100644 --- a/clean/runner/motion_hi_check.py +++ b/clean/runner/motion_hi_check.py @@ -603,7 +603,9 @@ def main(argv=None): for name in ("probe", "post", "enc", "time", "frame-time"): p = sub.add_parser(name) p.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk p.add_argument("--rocm", default="") + sdk.add_force_argument(p) p.add_argument("--seed", type=int, default=1) p.add_argument("--json") p.add_argument("--frames", type=int, default=5 if name == "post" else 3) @@ -613,6 +615,7 @@ def main(argv=None): p.add_argument("--rounds", type=int, default=5) p.add_argument("--repeats", type=int, default=20) args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk return {"probe": cmd_probe, "post": cmd_post, "enc": cmd_enc, "time": cmd_time, "frame-time": cmd_frame_time}[args.cmd](args) diff --git a/clean/runner/output_filter_original_check.py b/clean/runner/output_filter_original_check.py index c5d0253f..d0ed2e4f 100644 --- a/clean/runner/output_filter_original_check.py +++ b/clean/runner/output_filter_original_check.py @@ -450,13 +450,16 @@ def large_check(args, model, taps, frames=3): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--models", required=True, help="work directory for the synthetic models of this test") ap.add_argument("--json", required=True) ap.add_argument("--baseline-build", help="a runtime build from before this change (test 5)") ap.add_argument("--frames", type=int, default=24) ap.add_argument("--large", action="store_true", help="also 1706x960 -> 2560x1440 (test 6)") args = ap.parse_args() + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk root = Path(args.models) if (root / "model.wgcm").exists(): raise ValueError("refusing to write synthetic parameters into a converted-model directory") diff --git a/clean/runner/output_filter_time.py b/clean/runner/output_filter_time.py index b3698232..1c0af286 100644 --- a/clean/runner/output_filter_time.py +++ b/clean/runner/output_filter_time.py @@ -67,7 +67,9 @@ def post_stage_ms(copy, x, repeats): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--model", required=True) ap.add_argument("--configs", nargs="+", default=["output_filter=analytic", "output_filter=original"]) ap.add_argument("--render", default="1706x960") @@ -80,6 +82,7 @@ def main(): ap.add_argument("--other-map", default=DEFAULT_MAP) ap.add_argument("--other-configs", nargs="+", default=["output_filter=default"]) a = ap.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk R, O = (tuple(int(v) for v in s.split("x")) for s in (a.render, a.output)) if os.path.exists(os.path.join(a.model, "model.wgcm")): raise ValueError("--model is the work directory of a synthetic model, not a converted model") diff --git a/clean/runner/output_quality_gpu_check.py b/clean/runner/output_quality_gpu_check.py index 60c8ac67..d277b424 100644 --- a/clean/runner/output_quality_gpu_check.py +++ b/clean/runner/output_quality_gpu_check.py @@ -68,10 +68,13 @@ def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--build", required=True) ap.add_argument("--baseline", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--json", required=True) ap.add_argument("--timing", action="store_true") args = ap.parse_args() + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk candidate_root, baseline_root = Path(args.build).resolve(), Path(args.baseline).resolve() if candidate_root == baseline_root: raise ValueError("Candidate and baseline build directories must be distinct") diff --git a/clean/runner/output_quality_runtime_check.py b/clean/runner/output_quality_runtime_check.py index 60836453..f6b7cf3b 100644 --- a/clean/runner/output_quality_runtime_check.py +++ b/clean/runner/output_quality_runtime_check.py @@ -43,11 +43,14 @@ def compare_tree(a, b, prefix=""): def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", required=True) + sdk.add_force_argument(ap) ap.add_argument("--model", required=True, help="directory exclusively owned by this synthetic test") ap.add_argument("--json", required=True) ap.add_argument("--profile", choices=("quality9", "quality5"), default="quality9") args = ap.parse_args() + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk model = Path(args.model) if (model / "model.wgcm").exists(): raise ValueError("Refusing to write synthetic parameters into a converted-model directory") diff --git a/clean/runner/overlap_probe.py b/clean/runner/overlap_probe.py index 6fbb7c1e..193a8714 100644 --- a/clean/runner/overlap_probe.py +++ b/clean/runner/overlap_probe.py @@ -129,10 +129,13 @@ def format_report(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--json") ap.add_argument("--repeats", type=int, default=9) args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = run(args.build, args.rocm, repeats=args.repeats) print(format_report(r)) if args.json: diff --git a/clean/runner/perf_bench.py b/clean/runner/perf_bench.py index 10c55c8b..1eb81759 100644 --- a/clean/runner/perf_bench.py +++ b/clean/runner/perf_bench.py @@ -177,7 +177,9 @@ def bench_fusion(r, report, repeats, check=True, built=("fused",)): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--repeats", type=int, default=100) ap.add_argument("--stages", default="enc0,dec0,exposure,dec1dec0", help="comma-separated: enc0, dec0, exposure, dec1dec0 (dec1 + dec0 separate vs fused)") @@ -187,6 +189,7 @@ def main(argv=None): ap.add_argument("--no-timing", action="store_true", help="checks only") ap.add_argument("--backend", default="hip", help="hip (GPU) or emu (host emulator: checks of 'base' only)") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = Runner(args.build, args.rocm, backend=args.backend) report = {"device": getattr(r.dev, "device_name", "?"), "timing_ms": {}, "differ_from_base": {}, "checks": {}} built = {} diff --git a/clean/runner/post.py b/clean/runner/post.py index 35f0c032..877fe8eb 100644 --- a/clean/runner/post.py +++ b/clean/runner/post.py @@ -639,13 +639,12 @@ def build_wave64(build_dir, rocm_dir="", clang=None): """Build post_w64.co: the same kernels in wave64 mode (on RDNA 4 a wave64 FP32 FMA/MUL/ADD issues in one pass, in wave32 only through VOPD pairs). Same compiler command as build.py plus -mwavefrontsize64.""" import os - import subprocess from clean.runner import build as B clang = clang or B.find_clang(rocm_dir) out = os.path.join(build_dir, CODE_OBJECT_W64) cmd = B.device_command(clang, rocm_dir, os.path.join(B.KERNEL_DIR, "post.hip"), out, device_libs=bool(rocm_dir), extra=["-mwavefrontsize64"]) - subprocess.run(cmd, check=True) + B.run(cmd) # echo and compiler output with a custom (forced) SDK by id only return out diff --git a/clean/runner/post_ablate.py b/clean/runner/post_ablate.py index c744a256..907c8eaf 100644 --- a/clean/runner/post_ablate.py +++ b/clean/runner/post_ablate.py @@ -16,7 +16,6 @@ import json import os import re -import subprocess import sys HERE = os.path.dirname(os.path.abspath(__file__)) @@ -24,7 +23,8 @@ import numpy as np # noqa: E402 -from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 (the official HIP SDK, never ROCM_PATH) +from clean.runner import sdk # noqa: E402 (the HIP SDK policy, never ROCM_PATH) +from clean.runner.sdk import OFFICIAL_ROCM # noqa: E402 from clean.runner import build as B # noqa: E402 from clean.runner import post as PS # noqa: E402 from clean.runner.post_bench import big_inputs # noqa: E402 @@ -69,9 +69,10 @@ def build_all(build_dir, rocm, clang, names): out = os.path.join(build_dir, code_object(name)) cmd = B.device_command(clang, rocm, os.path.join(B.KERNEL_DIR, "post.hip"), out, device_libs=bool(rocm), report=True, defines={d: 1 for d in ABLATIONS[name]}) - r = subprocess.run(cmd, capture_output=True, text=True) - if r.returncode != 0: - raise SystemExit(f"build of {name} failed:\n{r.stderr}") + B.echo(cmd) + r = B.run_captured(cmd) + if r.returncode != 0: # a custom (forced) SDK appears by id only + raise SystemExit(f"build of {name} failed:\n{sdk.redact(r.stderr or '')}") info, cur = {}, None for line in r.stderr.splitlines(): m = re.search(r"Function Name: (\w+)", line) @@ -105,6 +106,7 @@ def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--clang", help="clang (default: from --rocm)") ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=1) @@ -114,6 +116,7 @@ def main(argv=None): ap.add_argument("--json", help="also write the results to this file") ap.add_argument("--build-only", action="store_true", help="build and print the compiler resource report only") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk names = args.only.split(",") if args.only else list(ABLATIONS) if "base" not in names: names = ["base"] + names diff --git a/clean/runner/post_bench.py b/clean/runner/post_bench.py index 45b7081c..084275e8 100644 --- a/clean/runner/post_bench.py +++ b/clean/runner/post_bench.py @@ -90,13 +90,16 @@ def check_small(r, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--json", help="also write the results to this file") ap.add_argument("--no-check", action="store_true", help="timing only") ap.add_argument("--no-timing", action="store_true", help="checks only") ap.add_argument("--backend", default="hip", help="hip (GPU) or emu (host emulator, checks only)") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = Runner(args.build, args.rocm, backend=args.backend) report = {"device": getattr(r.dev, "device_name", "?"), "timing_ms": {}, "checks": []} if not args.no_check: diff --git a/clean/runner/post_exact.py b/clean/runner/post_exact.py index 132e83db..3c0771cd 100644 --- a/clean/runner/post_exact.py +++ b/clean/runner/post_exact.py @@ -36,8 +36,12 @@ ("odd-4", (1200, 700), (1000, 600))] # R > 2/3 D: recon2_supported false, the post_recon fallback +FORWARD = [] # --force-sdk for the children when this process got it (main) + + def dump(python, repo, build, rocm, render, output, frames, paths, motion, seed, out, save=None, extra=()): cmd = [python, os.path.join(repo, "clean", "runner", "post_sizes.py"), "dump", "--build", build, "--rocm", rocm, + *FORWARD, "--render", render, "--output", output, "--frames", str(frames), "--paths", paths, "--motion", motion, "--seed", str(seed), "--out", out] + list(extra) if save: @@ -51,7 +55,9 @@ def dump(python, repo, build, rocm, render, output, frames, paths, motion, seed, def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--a-repo", required=True) ap.add_argument("--a-build", required=True) ap.add_argument("--b-repo", required=True) @@ -66,6 +72,8 @@ def main(argv=None): ap.add_argument("--work", help="folder for the dumps (default: a temporary folder)") ap.add_argument("--json", help="write the comparison summary here") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk + FORWARD[:] = sdk.forward(args) python = sys.executable if args.render or args.output: cfgs = [("custom", tuple(map(int, args.render.split("x"))), tuple(map(int, args.output.split("x"))))] diff --git a/clean/runner/post_fused_bench.py b/clean/runner/post_fused_bench.py index c0dfffd7..9dcc85c6 100644 --- a/clean/runner/post_fused_bench.py +++ b/clean/runner/post_fused_bench.py @@ -92,7 +92,9 @@ def check_small(r, code_objects, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--clang", help="clang for building post_w64.co (default: from --rocm)") ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--json", help="also write the results to this file") @@ -101,6 +103,7 @@ def main(argv=None): ap.add_argument("--no-w64", action="store_true", help="wave32 code object only") ap.add_argument("--backend", default="hip", help="hip (GPU) or emu (host emulator, checks only)") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk code_objects = [PS.CODE_OBJECT] if not args.no_w64 and args.backend == "hip": if not os.path.exists(os.path.join(args.build, PS.CODE_OBJECT_W64)): diff --git a/clean/runner/post_hdr_check.py b/clean/runner/post_hdr_check.py index 2725b3cb..6050c784 100644 --- a/clean/runner/post_hdr_check.py +++ b/clean/runner/post_hdr_check.py @@ -17,6 +17,7 @@ from clean.reference import post as ref from clean.runner import post as ps from clean.runner.stage_runner import Runner +from clean.runner import sdk # noqa: E402 (the HIP SDK policy: --rocm, --force-sdk) def tf_plan(dev, ow, oh, brightness, exposure, pi, half): @@ -65,8 +66,10 @@ def main(): ap.add_argument('--build', required=True) ap.add_argument('--baseline') ap.add_argument('--rocm', required=True) + sdk.add_force_argument(ap) ap.add_argument('--json', required=True) a = ap.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk for key in ('LLVM_PATH', 'ROCM_PATH', 'HIP_PATH', 'HIP_DEVICE_LIB_PATH'): os.environ.pop(key, None) runners = {'candidate': Runner(a.build, a.rocm, backend='hip')} diff --git a/clean/runner/post_sizes.py b/clean/runner/post_sizes.py index 7306934b..1fd66a4d 100644 --- a/clean/runner/post_sizes.py +++ b/clean/runner/post_sizes.py @@ -630,7 +630,9 @@ def main(argv=None): for name in ("time", "dump"): p = sub.add_parser(name) p.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk p.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(p) p.add_argument("--code-object", default=W64, help="code object of the post kernels (default post_w64.co)") p.add_argument("--render", help="render size WxH (with --output: one configuration)") p.add_argument("--output", help="output size WxH") @@ -650,6 +652,7 @@ def main(argv=None): d.add_argument("--save", help="also save the canonical outputs (.npy) into this folder") d.add_argument("--verbose", action="store_true") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk if args.cmd == "dump" and not (args.render and args.output): ap.error("dump needs --render and --output") (cmd_time if args.cmd == "time" else cmd_dump)(args) diff --git a/clean/runner/post_tf_runtime.py b/clean/runner/post_tf_runtime.py index 856b31c7..30354738 100644 --- a/clean/runner/post_tf_runtime.py +++ b/clean/runner/post_tf_runtime.py @@ -119,13 +119,16 @@ def frame_inputs(rng, rw, rh, reset, pi): def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--model", required=True) ap.add_argument("--output", default="2560x1440") ap.add_argument("--frames", type=int, default=14) ap.add_argument("--seed", type=int, default=5) ap.add_argument("--paths", default="copy,own") a = ap.parse_args() + sdk.apply(a) # --rocm: an official SDK, or a custom one with --force-sdk if not os.path.isdir(os.path.join(a.model, "raw")): write_raw(a.model, synthetic_tensors(1)) ow, oh = (int(v) for v in a.output.lower().split("x")) diff --git a/clean/runner/post_traffic.py b/clean/runner/post_traffic.py index 5bf89831..2a379752 100644 --- a/clean/runner/post_traffic.py +++ b/clean/runner/post_traffic.py @@ -216,13 +216,16 @@ def run(r, sizes, repeats=50, rounds=3, reference=True, report=None, flush_bytes def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=3) ap.add_argument("--no-reference", action="store_true", help="only the traffic layouts, not the real kernels") ap.add_argument("--no-w64", action="store_true", help="skip the wave64 (post_w64.co) reference timings") ap.add_argument("--json", help="also write the results to this file") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = Runner(args.build, args.rocm) w64 = not args.no_w64 and not args.no_reference if w64: # always rebuilt: an older post_w64.co may lack kernels of this post.hip diff --git a/clean/runner/post_variants.py b/clean/runner/post_variants.py index ae9a92cc..bc97d58d 100644 --- a/clean/runner/post_variants.py +++ b/clean/runner/post_variants.py @@ -104,7 +104,9 @@ def check(r, report): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--repeats", type=int, default=50) ap.add_argument("--rounds", type=int, default=3) ap.add_argument("--json", help="also write the results to this file") @@ -112,6 +114,7 @@ def main(argv=None): ap.add_argument("--no-timing", action="store_true") ap.add_argument("--backend", default="hip", help="hip (GPU) or emu (host emulator, checks only)") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = Runner(args.build, args.rocm, backend=args.backend) report = {"device": getattr(r.dev, "device_name", "?"), "budget_ms": BUDGET_MS, "checks": [], "timing_ms": {}} if not args.no_check: diff --git a/clean/runner/profile_stages.py b/clean/runner/profile_stages.py index cbf42fc1..59876702 100644 --- a/clean/runner/profile_stages.py +++ b/clean/runner/profile_stages.py @@ -146,7 +146,9 @@ def block_ms(times): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variant", default="prof") ap.add_argument("--stages", help="comma-separated (default: enc1, enc2, b0, dec2, dec1)") @@ -156,6 +158,7 @@ def main(argv=None): ap.add_argument("--no-full", action="store_true", help="skip the stride-1 comparison run") ap.add_argument("--json") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk runner = Runner(args.build, args.rocm, backend=args.backend, variant=args.variant) base = None try: diff --git a/clean/runner/profile_v2.py b/clean/runner/profile_v2.py index 33ab6a45..c818a5b0 100644 --- a/clean/runner/profile_v2.py +++ b/clean/runner/profile_v2.py @@ -131,12 +131,15 @@ def format_report(rep): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--backend", default="hip", choices=["hip", "emu"]) ap.add_argument("--variant", default="v2prof") ap.add_argument("--stages", help="comma-separated (default: all of sweep.GAME_SIZES)") ap.add_argument("--json") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk rep = profile(args.build, args.rocm, args.backend, args.variant, args.stages.split(",") if args.stages else None) print(format_report(rep)) diff --git a/clean/runner/rates.py b/clean/runner/rates.py index 5e8d611f..fa43ac86 100644 --- a/clean/runner/rates.py +++ b/clean/runner/rates.py @@ -103,9 +103,12 @@ def format_table(r): def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--build", required=True) + from clean.runner import sdk # the HIP SDK policy: --rocm, --force-sdk ap.add_argument("--rocm", default=OFFICIAL_ROCM) + sdk.add_force_argument(ap) ap.add_argument("--json") args = ap.parse_args(argv) + sdk.apply(args) # --rocm: an official SDK, or a custom one with --force-sdk r = run(args.build, args.rocm) print(format_table(r)) if args.json: diff --git a/clean/runner/sdk.py b/clean/runner/sdk.py index e10e5310..7bd802db 100644 --- a/clean/runner/sdk.py +++ b/clean/runner/sdk.py @@ -8,9 +8,13 @@ lie inside the SDK); - anything else needs a deliberate override (--force-sdk), which approves exactly that SDK folder or compiler for this process (approve); -- the HIP runtime library is loaded from the selected SDK only (runtime_library), never by a bare name the loader would - search for; -- logs name official folders as they are and custom ones only as "custom SDK (forced, id )" (label, redact). +- on other systems /opt/rocm counts with the version of the /opt/rocm- it points to, a standalone /opt/rocm + with its own version (.info/version; without one it ranks after every versioned install); +- the Python HIP runtime library is loaded from the selected SDK only (runtime_library), never by a bare name the loader + would search for (the native runtime and the installer have their own lookup, not covered here yet); +- every CLI with --rocm offers --force-sdk (add_force_argument) and resolves --rocm right after parsing (apply); +- logs name official folders as they are and custom ones only as "custom (forced, id )" (label, redact: every + spelling of the path, also in compiler output). device_command / find_clang (clean/runner/build.py) and the HIP runtime loader (clean/runner/hip.py) call check(): every entry point that compiles HIP code or loads the HIP runtime passes it, whichever script started it.""" @@ -26,6 +30,7 @@ _VERSION = re.compile(r"^(\d+)\.(\d+)(?:\.(\d+))?$") _approved = set() # normalized folders / compilers approved with --force-sdk in this process +_spellings = {} # approved normalized path -> the ways a log may write it (redact) _bannered = set() @@ -55,17 +60,40 @@ def official_installs(windows_root=None, linux_root=None, windows=None): names = [(n[len("rocm-"):], os.path.join(root, n)) for n in (os.listdir(root) if os.path.isdir(root) else []) if n.startswith("rocm-")] for ver, path in names: - m = _VERSION.match(ver) - if m and int(m.group(1)) == SUPPORTED_MAJOR and _valid_sdk(path): - out.append((tuple(int(g or 0) for g in m.groups()), path)) - out.sort(reverse=True) - if not windows: # /opt/rocm: official when it is (or links to) one of them + v = _parse_version(ver) + if v and v[0] == SUPPORTED_MAJOR and _valid_sdk(path): + out.append((v, path)) + if not windows: + # /opt/rocm: with the version of the install it points to; a standalone one (no versioned sibling it resolves + # to) with its own version (.info/version), and without one ranked after every versioned install link = os.path.join(root, "rocm") - if os.path.isdir(link) and any(_norm(link) == _norm(p) for _, p in out): - out.insert(0, (out[0][0], link)) + if _valid_sdk(link): + target = [v for v, p in out if _norm(p) == _norm(link)] + v = target[0] if target else _install_version(link) + if v is None: + out.append(((-1,), link)) + elif v[0] == SUPPORTED_MAJOR: + out.append((v, link)) + # newest first; at the same version /opt/rocm before the folder it points to + out.sort(key=lambda e: (e[0], os.path.basename(e[1]) == "rocm"), reverse=True) return out +def _parse_version(text): + m = _VERSION.match(text) + return tuple(int(g or 0) for g in m.groups()) if m else None + + +def _install_version(path): + """The version a standalone install states in .info/version (e.g. "7.2.0-43"), or None.""" + try: + with open(os.path.join(path, ".info", "version"), encoding="utf-8", errors="replace") as f: + m = re.match(r"\s*(\d+\.\d+(?:\.\d+)?)", f.read()) + except OSError: + return None + return _parse_version(m.group(1)) if m else None + + def default_rocm(): """The newest official compatible install, or None.""" found = official_installs() @@ -93,8 +121,53 @@ def label(path): def approve(path): - """--force-sdk: accept this exact SDK folder or compiler in this process.""" - _approved.add(_norm(path)) + """--force-sdk: accept this exact SDK folder or compiler in this process (and redact every spelling of it).""" + n = _norm(path) + _approved.add(n) + _spellings.setdefault(n, set()).update(_spellings_of(path) | {n}) + + +def _short_name(path): + """The Windows 8.3 short spelling of an existing path, or None.""" + if sys.platform != "win32": + return None + try: + import ctypes + buf = ctypes.create_unicode_buffer(32768) + n = ctypes.windll.kernel32.GetShortPathNameW(str(path), buf, len(buf)) + return buf.value if 0 < n < len(buf) else None + except (OSError, AttributeError, ValueError): + return None + + +def _spellings_of(path): + """The ways a log may write `path`: as given, absolute, normalized, resolved, relative to the working folder, the + 8.3 short form, and under the home folder with the ~ shorthand (separators and case are matched freely; variables + like the home variable are expanded by the shell before a tool sees them).""" + absp = os.path.abspath(path) + out = {path, absp, os.path.normpath(path), os.path.realpath(absp), _short_name(absp)} + try: + out.add(os.path.relpath(absp)) + except ValueError: # another drive + pass + out |= {os.path.join(".", p) for p in list(out) if p and not os.path.isabs(p) and not p.startswith(".")} + home = os.path.expanduser("~") + for p in list(out): + if p and home and home != "~" and _inside(p, home) and _norm(p) != _norm(home): + rest = os.path.relpath(os.path.abspath(p), home) + out.add(os.path.join("~", rest)) + return {p.rstrip("\\/") for p in out if p and p.rstrip("\\/") not in ("", ".", "..")} + + +def _spelling_pattern(spelling): + """A spelling as a pattern: any run of either separator between its parts; not inside a longer name.""" + parts = [re.escape(x) for x in re.split(r"[\\/]+", spelling)] + body = r"[\\/]+".join(parts) + if os.path.isabs(spelling) or re.match(r"^(~|%|\$|[A-Za-z]:)", spelling): + lead = "" # distinctive by itself (also right after -I, -L, = ...) + else: # a relative spelling: a whole path, or right after -I / -L / -B + lead = r"(?:(? wins over a /opt/rocm that points to an older one; /opt/rocm carries the version + it points to; a standalone /opt/rocm counts (its .info/version, or ranked last without one); +- --force-sdk: post_ablate accepts a forced SDK end to end; every tracked CLI with --rocm offers --force-sdk and resolves + --rocm through the policy (scan); +- logs: custom folders appear by id only (banner, refusal, command echo, compiler output of run / run_post_w64 / + post_ablate, the post_w64 report), in every spelling (relative, either separator, case, 8.3, under the home folder); - build.py: with ROCM_PATH and friends pointing elsewhere, every compile it starts uses the SDK's clang and its device libraries, and the build completes; - a scan of clean/ and tools/: no Python file reads an SDK variable (ROCM_PATH, HIP_PATH, LLVM_PATH, @@ -242,6 +247,206 @@ def test_every_compile_uses_the_sdk(self): self.assertIn(os.path.splitext(src)[0] + ".co", outs) +class LinuxDiscovery(Env): + """/opt/rocm next to /opt/rocm- (official_installs(windows=False), on any test platform).""" + + def linux(self, *versions): + got = {v: fake_sdk(os.path.join(self.root, "rocm-" + v)) for v in versions} + for p in got.values(): # the Linux spelling of the compiler too + open(os.path.join(p, "bin", "clang"), "w").close() + return got + + def standalone(self, version_text=None): + link = fake_sdk(os.path.join(self.root, "rocm")) + open(os.path.join(link, "bin", "clang"), "w").close() + if version_text is not None: + os.makedirs(os.path.join(link, ".info"), exist_ok=True) + with open(os.path.join(link, ".info", "version"), "w") as f: + f.write(version_text) + return link + + def found(self): + return sdk.official_installs(linux_root=self.root, windows=False) + + def points_to(self, link, target): + """/opt/rocm resolving to `target` (a real symlink where the platform allows, else _norm patched).""" + orig = sdk._norm + return patch.object(sdk, "_norm", lambda p: orig(target) if orig(p) == orig(link) else orig(p)) + + def test_newest_wins_over_the_link(self): + got = self.linux("7.1", "7.2") + link = self.standalone() # stands in for the link: resolves to rocm-7.1 + with self.points_to(link, got["7.1"]): + found = self.found() + self.assertEqual(found[0], ((7, 2, 0), got["7.2"])) # the newest, not the link's + self.assertIn(((7, 1, 0), link), found) # the link with the version it points to + self.assertLess(found.index(((7, 1, 0), link)), found.index(((7, 1, 0), got["7.1"]))) + + def test_link_to_the_newest(self): + got = self.linux("7.1", "7.2") + link = self.standalone() + with self.points_to(link, got["7.2"]): + found = self.found() + self.assertEqual(found[0], ((7, 2, 0), link)) + self.assertEqual(found[1], ((7, 2, 0), got["7.2"])) + + def test_real_symlink(self): + got = self.linux("7.1", "7.2") + link = os.path.join(self.root, "rocm") + try: + os.symlink(got["7.1"], link, target_is_directory=True) + except (OSError, NotImplementedError): + self.skipTest("no symlinks on this system") + found = self.found() + self.assertEqual(found[0], ((7, 2, 0), got["7.2"])) + self.assertIn(((7, 1, 0), link), found) + + def test_standalone_counts(self): + link = self.standalone("7.2.0-43\n") + self.assertEqual(self.found(), [((7, 2, 0), link)]) + got = self.linux("7.1") + self.assertEqual(self.found(), [((7, 2, 0), link), ((7, 1, 0), got["7.1"])]) + + def test_standalone_without_version_ranks_last(self): + link = self.standalone() + got = self.linux("7.0") + self.assertEqual(self.found(), [((7, 0, 0), got["7.0"]), ((-1,), link)]) + + def test_standalone_incompatible_or_incomplete(self): + self.standalone("6.4.1") + self.assertEqual(self.found(), []) + os.makedirs(os.path.join(self.tmp.name, "other"), exist_ok=True) + self.assertEqual(sdk.official_installs(linux_root=os.path.join(self.tmp.name, "other"), windows=False), []) + + +class ForceCli(Env): + """--force-sdk on the CLIs with --rocm: post_ablate end to end, and a scan of every tracked CLI.""" + + def test_post_ablate_accepts_a_forced_sdk(self): + self.install("7.2") + from clean.runner import post_ablate + out = os.path.join(self.tmp.name, "out") + seen = [] + + class Stop(Exception): + pass + + def build_all(build_dir, rocm, clang, names): + seen.append((rocm, clang)) + raise Stop() + with patch.object(post_ablate, "build_all", build_all), contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit): # refused without the override + post_ablate.main(["--build", out, "--build-only", "--rocm", self.custom]) + self.assertEqual(seen, []) + with self.assertRaises(Stop): + post_ablate.main(["--build", out, "--build-only", "--rocm", self.custom, "--force-sdk"]) + self.assertEqual(seen, [(os.path.abspath(self.custom), None)]) + sdk.check_rocm(self.custom) # approved for this process + + def test_post_ablate_forced_build_output_is_redacted(self): + from clean.runner import post_ablate + clang = os.path.join(self.custom, "bin", "clang" + EXE) + sdk.resolve(self.custom, force=True) + err = f"clang: error: {clang}: bad\nIn file included from {self.custom.replace(os.sep, '/')}/include/x.h" + with patch.object(build, "run_captured", lambda cmd: subprocess.CompletedProcess(cmd, 1, "", err)), \ + contextlib.redirect_stdout(io.StringIO()) as log: + with self.assertRaises(SystemExit) as e: + post_ablate.build_all(os.path.join(self.tmp.name, "out"), self.custom, clang, ["base"]) + for text in (str(e.exception), log.getvalue()): + self.assertNotIn("custom-sdk", text) + self.assertIn("custom (forced, id ", text) + + def test_every_rocm_cli_has_the_override(self): + """Every tracked Python file with a --rocm option also offers --force-sdk and resolves --rocm through the + policy after parsing (sdk.add_arguments + sdk.resolve, or sdk.add_force_argument + sdk.apply).""" + r = subprocess.run(["git", "ls-files", "--", "clean", "tools"], cwd=ROOT, capture_output=True, text=True, + check=True) + files = [f for f in r.stdout.split() if f.endswith(".py")] + self.assertGreater(len(files), 200) + clis, missing = 0, [] + for f in files: + with open(os.path.join(ROOT, f), encoding="utf-8") as fh: + s = fh.read() + if not re.search(r"""add_argument\(\s*["']--rocm["']""", s) or f == "clean/runner/sdk.py": + continue + clis += 1 + if not (("sdk.add_force_argument(" in s and "sdk.apply(" in s) or + ("sdk.add_arguments(" in s and "sdk.resolve(" in s)): + missing.append(f) + self.assertGreater(clis, 40) + self.assertEqual(missing, []) + with open(os.path.join(ROOT, "clean", "runner", "build.py"), encoding="utf-8") as fh: + self.assertIn("sdk.add_arguments(ap)", fh.read()) + + +class Redaction(Env): + """Custom (forced) paths in logs: every spelling, and build.py's post_w64 path (echo, stderr, report).""" + + def test_spellings(self): + sdk.resolve(self.custom, force=True) + c = self.custom + texts = [c, c.replace("\\", "/"), c.upper(), c.replace(os.sep, os.sep * 2), "-I" + c + os.sep + "include", + "--rocm-path=" + c, '"' + os.path.join(c, "bin", "clang" + EXE) + '"'] + for t in texts: + out = sdk.redact(t) + self.assertNotIn("custom-sdk", out.lower(), t) + self.assertIn("custom (forced, id ", out, t) + for t in (c + "2", os.path.join(self.tmp.name, "custom-sdk-old"), "custom-sdk.txt"): + self.assertEqual(sdk.redact(t), t) # another name stays + + def test_relative_and_home_spellings(self): + cwd = os.getcwd() + os.chdir(self.tmp.name) + try: + with patch.object(sdk.os.path, "expanduser", lambda p: self.tmp.name if p == "~" else p): + sdk.resolve("custom-sdk", force=True) + for t in ("custom-sdk", "custom-sdk/bin/clang", "-Icustom-sdk\\include", "." + os.sep + "custom-sdk", + os.path.join("~", "custom-sdk", "bin")): + out = sdk.redact(t) + self.assertNotIn("custom-sdk", out, t) + for t in ("notcustom-sdk", "custom-sdk2", "custom-sdk.py"): + self.assertEqual(sdk.redact(t), t) + finally: + os.chdir(cwd) + + def test_short_name(self): + short = sdk._short_name(self.custom) + if not short or os.path.normcase(short) == os.path.normcase(self.custom): + self.skipTest("no distinct 8.3 name here") + sdk.resolve(self.custom, force=True) + self.assertNotIn(short.lower(), sdk.redact(short).lower()) + + def test_run_post_w64(self): + sdk.resolve(self.custom, force=True) + clang = sdk.compiler(self.custom, os.path.join(self.custom, "bin", "clang" + EXE), force=True) + cmd = [clang, f"--rocm-path={self.custom}", f"--rocm-device-lib-path={sdk.device_lib_dir(self.custom)}", + "k.hip"] + err = (f"remark: {self.custom.replace(os.sep, '/')}/amdgcn/bitcode/ocml.bc\n" + f"{os.path.relpath(self.custom)}{os.sep}include: warning\n") + out = os.path.join(self.tmp.name, "out") + os.makedirs(out) + for rc in (1, 0): + with patch.object(build, "run_captured", lambda c, rc=rc: subprocess.CompletedProcess(c, rc, "", err)), \ + contextlib.redirect_stdout(io.StringIO()) as log, contextlib.redirect_stderr(io.StringIO()) as elog: + with contextlib.suppress(SystemExit): + build.run_post_w64(cmd, out, gates={}) + with open(os.path.join(out, build.POST_W64_REPORT)) as f: + report = f.read() + for text in (log.getvalue(), elog.getvalue(), report): + self.assertNotIn("custom-sdk", text) + self.assertIn("custom (forced, id ", log.getvalue()) + self.assertIn("custom (forced, id ", report) + + def test_run(self): + sdk.resolve(self.custom, force=True) + err = f"In file included from {self.custom}{os.sep}include{os.sep}hip.h\n" + with patch.object(build, "run_captured", lambda c: subprocess.CompletedProcess(c, 1, err, err)), \ + contextlib.redirect_stdout(io.StringIO()) as log, contextlib.redirect_stderr(io.StringIO()) as elog: + with self.assertRaises(SystemExit): + build.run([os.path.join(self.custom, "bin", "clang" + EXE), "-c", "k.hip"]) + self.assertNotIn("custom-sdk", log.getvalue() + elog.getvalue()) + + SDK_VARS = ("ROCM_PATH", "HIP_PATH", "LLVM_PATH", "HIP_DEVICE_LIB_PATH", "HIP_CLANG_PATH", "HIPFIRE_ROCM_PATH", "ROCM_HOME") _V = "|".join(SDK_VARS) From 0393ec3276f5d554454efabec36a13024beb7138 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Sun, 15 Nov 2026 06:52:34 -0500 Subject: [PATCH 4/4] Builds: the LUT oracle builders under the SDK policy; --emu-only needs no HIP SDK (re-check 2) - clean/dx12/gelu_lut/run.py, f16lut/run.py, f16lut/run_in.py: --rocm / --force-sdk through sdk.resolve, the SDK's clang from sdk.compiler, sdk.check before the compile, logs redacted (no fixed 7.2 path) - build.py and k_build.py resolve the HIP SDK only when a device build needs it: --emu-only builds the host emulator with the host compiler alone, also with no SDK installed - tests: emu-only builds without an SDK (and the device build still refused); a scan that every file running the HIP compiler itself goes through the policy Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- clean/dx12/f16lut/run.py | 21 +++++++++++--- clean/dx12/f16lut/run_in.py | 21 +++++++++++--- clean/dx12/gelu_lut/run.py | 24 ++++++++++++---- clean/k/runner/k_build.py | 6 ++-- clean/runner/build.py | 8 ++++-- clean/tests/test_sdk.py | 57 +++++++++++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 19 deletions(-) diff --git a/clean/dx12/f16lut/run.py b/clean/dx12/f16lut/run.py index 6e070b90..bf5688b2 100644 --- a/clean/dx12/f16lut/run.py +++ b/clean/dx12/f16lut/run.py @@ -28,7 +28,10 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH +sys.path.insert(0, os.path.dirname(CLEAN)) +from clean.runner import sdk # noqa: E402 (the HIP SDK policy) + +ROCM = sdk.OFFICIAL_ROCM # main: --rocm through sdk.resolve (an official SDK; another one only with --force-sdk) # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") @@ -49,11 +52,18 @@ def hip_env(): def run(cmd, **kw): r = subprocess.run(cmd, capture_output=True, text=True, **kw) - if r.returncode not in (0, 5): - print(r.stdout[-2000:], r.stderr[-2000:]) + if r.returncode not in (0, 5): # a custom (forced) SDK appears by id only + print(sdk.redact(r.stdout[-2000:]), sdk.redact(r.stderr[-2000:])) return r +def hip_clang(): + """The SDK's own clang, through the shared SDK policy check (banner once).""" + clang = sdk.compiler(ROCM, None) + sdk.check(ROCM, clang) + return clang + + def gpu(args, **kw): return run(_locked(args), timeout=1800, **kw) @@ -83,11 +93,14 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--work", required=True) ap.add_argument("--batch", type=int, default=150) + sdk.add_arguments(ap) a = ap.parse_args() + global ROCM + ROCM = sdk.resolve(a.rocm, a.force_sdk) w = os.path.abspath(a.work) os.makedirs(w, exist_ok=True) co = os.path.join(w, "oracle.co") - r = run([os.path.join(ROCM, "bin", "clang.exe"), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", + r = run([hip_clang(), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", "-nogpuinc", "-O3", "--no-gpu-bundle-output", f"--rocm-path={ROCM}", f"--rocm-device-lib-path={os.path.join(ROCM, 'amdgcn', 'bitcode')}", "-I", os.path.join(CLEAN, "kernels"), os.path.join(HERE, "oracle.hip"), "-o", co], env=hip_env()) diff --git a/clean/dx12/f16lut/run_in.py b/clean/dx12/f16lut/run_in.py index ef6ac054..b3630557 100644 --- a/clean/dx12/f16lut/run_in.py +++ b/clean/dx12/f16lut/run_in.py @@ -33,7 +33,10 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH +sys.path.insert(0, os.path.dirname(CLEAN)) +from clean.runner import sdk # noqa: E402 (the HIP SDK policy) + +ROCM = sdk.OFFICIAL_ROCM # main: --rocm through sdk.resolve (an official SDK; another one only with --force-sdk) # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") @@ -54,11 +57,18 @@ def hip_env(): def run(cmd, **kw): r = subprocess.run(cmd, capture_output=True, text=True, **kw) - if r.returncode not in (0, 5): - print(r.stdout[-2000:], r.stderr[-2000:]) + if r.returncode not in (0, 5): # a custom (forced) SDK appears by id only + print(sdk.redact(r.stdout[-2000:]), sdk.redact(r.stderr[-2000:])) return r +def hip_clang(): + """The SDK's own clang, through the shared SDK policy check (banner once).""" + clang = sdk.compiler(ROCM, None) + sdk.check(ROCM, clang) + return clang + + def gpu(args, **kw): return run(_locked(args), timeout=1800, **kw) @@ -89,11 +99,14 @@ def main(): ap = argparse.ArgumentParser() ap.add_argument("--work", required=True) ap.add_argument("--batch", type=int, default=150) + sdk.add_arguments(ap) a = ap.parse_args() + global ROCM + ROCM = sdk.resolve(a.rocm, a.force_sdk) w = os.path.abspath(a.work) os.makedirs(w, exist_ok=True) co = os.path.join(w, "oracle_in.co") - r = run([os.path.join(ROCM, "bin", "clang.exe"), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", + r = run([hip_clang(), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", "-nogpuinc", "-O3", "--no-gpu-bundle-output", f"--rocm-path={ROCM}", f"--rocm-device-lib-path={os.path.join(ROCM, 'amdgcn', 'bitcode')}", "-I", os.path.join(CLEAN, "kernels"), os.path.join(HERE, "oracle_in.hip"), "-o", co], env=hip_env()) diff --git a/clean/dx12/gelu_lut/run.py b/clean/dx12/gelu_lut/run.py index c6ccb49b..040b00ec 100644 --- a/clean/dx12/gelu_lut/run.py +++ b/clean/dx12/gelu_lut/run.py @@ -32,7 +32,10 @@ sys.path.insert(0, DX12) import build_dx12 # noqa: E402 -ROCM = r"C:\Program Files\AMD\ROCm\7.2" # the official HIP SDK (clean/runner/sdk.py), never ROCM_PATH +sys.path.insert(0, os.path.dirname(CLEAN)) +from clean.runner import sdk # noqa: E402 (the HIP SDK policy) + +ROCM = sdk.OFFICIAL_ROCM # main: --rocm through sdk.resolve (an official SDK; another one only with --force-sdk) # Optional GPU lock (WG_GPU_LOCK: a script run as `python --timeout -- `, e.g. to keep # GPU jobs of several shells apart); unset: the command runs directly. GPU_LOCK = os.environ.get("WG_GPU_LOCK", "") @@ -52,21 +55,29 @@ def hip_env(): def run(cmd, **kw): - print(">", " ".join(cmd) if isinstance(cmd, list) else cmd, flush=True) + # a custom (forced) SDK appears by id only, in the command and in the tools' output + print(">", sdk.redact(" ".join(cmd) if isinstance(cmd, list) else cmd), flush=True) r = subprocess.run(cmd, capture_output=True, text=True, **kw) - print(r.stdout.strip()) + print(sdk.redact(r.stdout.strip())) if r.stderr.strip(): - print(r.stderr.strip()) + print(sdk.redact(r.stderr.strip())) return r +def hip_clang(): + """The SDK's own clang, through the shared SDK policy check (banner once).""" + clang = sdk.compiler(ROCM, None) + sdk.check(ROCM, clang) + return clang + + def gpu(args, **kw): return run(_locked(args), **kw) def build_tools(w): co = os.path.join(w, "oracle.co") - r = run([os.path.join(ROCM, "bin", "clang.exe"), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", "-nogpuinc", + r = run([hip_clang(), "-x", "hip", "--offload-arch=gfx1201", "--offload-device-only", "-nogpuinc", "-O3", "--no-gpu-bundle-output", f"--rocm-path={ROCM}", f"--rocm-device-lib-path={os.path.join(ROCM, 'amdgcn', 'bitcode')}", "-I", os.path.join(CLEAN, "kernels"), os.path.join(HERE, "oracle.hip"), "-o", co], env=hip_env()) @@ -102,7 +113,10 @@ def main(): ap.add_argument("--skip-oracle", action="store_true", help="reuse /rle.bin") ap.add_argument("--table", help="compact design: this table (mk_headers.py output) instead of running gen.py") ap.add_argument("--define", action="append", default=[], help="NAME=VALUE for the verify shader (WG_GELU_LUT_ALT=1)") + sdk.add_arguments(ap) a = ap.parse_args() + global ROCM + ROCM = sdk.resolve(a.rocm, a.force_sdk) w = os.path.abspath(a.work) os.makedirs(w, exist_ok=True) proof = {} diff --git a/clean/k/runner/k_build.py b/clean/k/runner/k_build.py index 45888464..f0bb4259 100644 --- a/clean/k/runner/k_build.py +++ b/clean/k/runner/k_build.py @@ -73,10 +73,10 @@ def main(argv=None): ap.add_argument("--emu", action="store_true", help="also the host emulator library " + EMU_LIBRARY) ap.add_argument("--cxx", default=os.environ.get("CXX", "clang++")) a = ap.parse_args(argv) - a.rocm = sdk.resolve(a.rocm, a.force_sdk) os.makedirs(a.out, exist_ok=True) - if not a.emu_only: # the SDK's own compiler; another one only with --force-sdk (device_command checks it again) - clang = sdk.compiler(a.rocm, a.clang, a.force_sdk) + if not a.emu_only: # the HIP SDK only for the device build (--emu-only: the host compiler alone) + a.rocm = sdk.resolve(a.rocm, a.force_sdk) + clang = sdk.compiler(a.rocm, a.clang, a.force_sdk) # another one only with --force-sdk build_device(clang, a.rocm, a.out, not a.no_device_libs, a.report, io_wave64=not a.io_wave32) if a.emu or a.emu_only: run(emu_command(a.cxx, os.path.join(a.out, EMU_LIBRARY))) diff --git a/clean/runner/build.py b/clean/runner/build.py index 0e329c1d..a9fe306e 100644 --- a/clean/runner/build.py +++ b/clean/runner/build.py @@ -498,9 +498,11 @@ def main(argv=None): "first, for every target); its D3D12 kernels come from clean/dx12/build_dx12.py --model-l") args = ap.parse_args(argv) rocm_given = bool(args.rocm) # release runtime builds reuse pinned Model L code objects unless --rocm is given - args.rocm = sdk.resolve(args.rocm, args.force_sdk) - if args.clang: # the SDK's own compiler (a bare name means the SDK's file); another one only with --force-sdk - args.clang = sdk.compiler(args.rocm, args.clang, args.force_sdk) + # the HIP SDK only when something needs it: --emu-only builds the host emulator with the host compiler alone + if not args.emu_only or args.runtime_only or args.motion_outres_only: + args.rocm = sdk.resolve(args.rocm, args.force_sdk) + if args.clang: # the SDK's own compiler (a bare name means the SDK's file); another one only with --force-sdk + args.clang = sdk.compiler(args.rocm, args.clang, args.force_sdk) for name in OPTIONAL_MODELS: set_model(name, not getattr(args, "no_model_" + name.lower())) set_model_l(args.model_l) diff --git a/clean/tests/test_sdk.py b/clean/tests/test_sdk.py index 6e49ba98..c12a5ec6 100644 --- a/clean/tests/test_sdk.py +++ b/clean/tests/test_sdk.py @@ -379,6 +379,63 @@ def test_every_rocm_cli_has_the_override(self): self.assertIn("sdk.add_arguments(ap)", fh.read()) +class EmuOnly(Env): + """--emu-only (the host emulator, host compiler only) needs no HIP SDK; the device build still does.""" + + def capture(self, module): + cmds = [] + return cmds, patch.object(module, "run", lambda cmd, *a, **k: cmds.append(list(cmd))) + + def test_build_py_emu_only_without_sdk(self): + self.assertIsNone(sdk.default_rocm()) # no official SDK anywhere + out = os.path.join(self.tmp.name, "out") + cmds, p = self.capture(build) + with p, patch.dict(os.environ, {}), contextlib.redirect_stdout(io.StringIO()): + build.main(["--out", out, "--emu-only", "--cxx", "g++"]) + self.assertTrue(cmds) + self.assertTrue(all(c[0] == "g++" for c in cmds), [c[0] for c in cmds]) + with self.assertRaises(SystemExit): # the device build still needs one + build.main(["--out", out, "--no-runtime"]) + + def test_k_build_emu_only_without_sdk(self): + from clean.k.runner import k_build + from clean.runner import build as B + self.assertIsNone(sdk.default_rocm()) + out = os.path.join(self.tmp.name, "out") + cmds, p = self.capture(B) + with p, contextlib.redirect_stdout(io.StringIO()): + k_build.main(["--out", out, "--emu-only", "--cxx", "g++"]) + self.assertEqual([c[0] for c in cmds], ["g++"]) + with self.assertRaises(SystemExit): + k_build.main(["--out", out]) + + +class DirectCompiles(unittest.TestCase): + def test_every_hip_compile_passes_the_policy(self): + """A tracked Python file that runs the HIP compiler itself (--offload-arch) takes the SDK and compiler from the + policy and calls sdk.check (build.py does it in device_command); none names the SDK's clang by a fixed path.""" + r = subprocess.run(["git", "ls-files", "--", "clean", "tools"], cwd=ROOT, capture_output=True, text=True, + check=True) + files = [f for f in r.stdout.split() if f.endswith(".py") and f != "clean/tests/test_sdk.py"] + self.assertGreater(len(files), 200) + direct, bad = [], [] + for f in files: + with open(os.path.join(ROOT, f), encoding="utf-8") as fh: + s = fh.read() + if "--offload-arch" not in s: + continue + direct.append(f) + if f == "clean/runner/build.py": + ok = "sdk.check(" in s + else: + ok = "sdk.check(" in s and "sdk.compiler(" in s and "sdk.resolve(" in s + if not ok or re.search(r"""["']clang(\.exe)?["']\s*\)""", s) or "AMD\\ROCm\\7" in s: + bad.append(f) + self.assertIn("clean/runner/build.py", direct) + self.assertGreaterEqual(len(direct), 4) # build.py and the three LUT oracle builders + self.assertEqual(bad, []) + + class Redaction(Env): """Custom (forced) paths in logs: every spelling, and build.py's post_w64 path (echo, stderr, report)."""