From d6bcc9768cd79f4e60c63928ae41df09d1f1fa6a Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:04:39 -0500 Subject: [PATCH 1/2] CI: split CPU build and tests into parallel jobs The single CPU job (about 27 minutes, 17 of them the Python test set) becomes parallel jobs: converter and shared settings; FSR front end; launcher; Ray Reconstruction plugin (with the one Python module that needs its build, test_optimal.py); the Python test set in three shards. Each job builds what it needs; no build output is shared. A final job keeps the required check name "CPU build and tests" and passes only when every split job passed. run_python_tests.py: --shard K/N and --group proxy run parts of the set; without options it runs the whole set as before. test_stages.py (812 s) is split by test id into six lists. Every shard computes the whole plan and checks that each module or test id is in exactly one part; --plan N prints it. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- .github/workflows/ci.yml | 102 ++++++++++++++++++++++++++-- tools/ci/run_python_tests.py | 124 ++++++++++++++++++++++++++++++++--- 2 files changed, 213 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c2a577e..1a83df37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,10 +32,12 @@ jobs: python -m pip install --disable-pip-version-check reuse==5.0.2 reuse lint - build-and-test: - name: CPU build and tests + # The CPU builds and tests run as parallel jobs; each builds what it needs (no build output is shared between jobs). + # "CPU build and tests" (the required check) passes only when all of them pass. + converter-shared: + name: "CPU: converter and shared settings" runs-on: windows-2022 - timeout-minutes: 90 + timeout-minutes: 60 env: CMAKE_BUILD_PARALLEL_LEVEL: "4" PYTHONDONTWRITEBYTECODE: "1" @@ -61,12 +63,48 @@ jobs: cmake -S clean/wgshared -B out/wgshared -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded cmake --build out/wgshared ctest --test-dir out/wgshared --output-on-failure + + frontend: + name: "CPU: FSR front end" + runs-on: windows-2022 + timeout-minutes: 60 + env: + CMAKE_BUILD_PARALLEL_LEVEL: "4" + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@v4.2.2 + - uses: actions/setup-python@v5.3.0 + with: + python-version: "3.12" + - name: Python packages + run: python -m pip install --disable-pip-version-check numpy ninja + - uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: x64 - name: FSR front end (CPU tests) run: | $PSNativeCommandUseErrorActionPreference = $true cmake -S clean/fsr -B out/frontend -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DWG_MODEL_K=OFF -DWG_BUILD_ID=ci -DWG_BUILD_ASI=ON cmake --build out/frontend ctest --test-dir out/frontend --output-on-failure -R "unit|hosted|diagnostic|support" -E "overlay|split|d3d12" + + launcher: + name: "CPU: launcher" + runs-on: windows-2022 + timeout-minutes: 60 + env: + CMAKE_BUILD_PARALLEL_LEVEL: "4" + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@v4.2.2 + - uses: actions/setup-python@v5.3.0 + with: + python-version: "3.12" + - name: Python packages + run: python -m pip install --disable-pip-version-check numpy ninja + - uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: x64 - name: HIP runtime stand-in for the launcher tests run: | $PSNativeCommandUseErrorActionPreference = $true @@ -79,6 +117,24 @@ jobs: cmake -S clean/installer -B out/launcher-tests -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POLICY_DEFAULT_CMP0091=NEW -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DWRONGGPU_PAYLOAD_DIR="$PWD/out/no-payload" -DWRONGGPU_TESTING=OFF -DWRONGGPU_PRESET_K=OFF -DWRONGGPU_DIRECT_ROUTE=OFF cmake --build out/launcher-tests --target wg-core-tests launcher-core-tests coexist-core-tests convert-ignore-tests ctest --test-dir out/launcher-tests --output-on-failure -R "^(wg-core|launcher-core|coexist-core|convert-ignore|source-size|wording)$" + + rr-plugin: + name: "CPU: Ray Reconstruction plugin" + runs-on: windows-2022 + timeout-minutes: 60 + env: + CMAKE_BUILD_PARALLEL_LEVEL: "4" + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@v4.2.2 + - uses: actions/setup-python@v5.3.0 + with: + python-version: "3.12" + - name: Python packages + run: python -m pip install --disable-pip-version-check numpy ninja + - uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: x64 - name: Ray Reconstruction plugin (build and CPU tests) run: | $PSNativeCommandUseErrorActionPreference = $true @@ -86,5 +142,43 @@ jobs: python clean/rr_f/proxy/tools/build.py --out out/rr-proxy --no-tests python tools/ci/run_proxy_core_tests.py out/rr-proxy "WG_RR_PROXY_BUILD=$PWD\out\rr-proxy" | Out-File -FilePath $env:GITHUB_ENV -Append -Encoding utf8 + - name: Python tests that need the plugin build (CPU) + run: python tools/ci/run_python_tests.py --group proxy + + python: + name: "CPU: Python tests ${{ matrix.shard }}/3" + runs-on: windows-2022 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + CMAKE_BUILD_PARALLEL_LEVEL: "4" + PYTHONDONTWRITEBYTECODE: "1" + steps: + - uses: actions/checkout@v4.2.2 + - uses: actions/setup-python@v5.3.0 + with: + python-version: "3.12" + - name: Python packages + run: python -m pip install --disable-pip-version-check numpy ninja + - uses: ilammy/msvc-dev-cmd@v1.13.0 + with: + arch: x64 - name: Python tests (CPU) - run: python tools/ci/run_python_tests.py --jobs 3 + run: python tools/ci/run_python_tests.py --jobs 3 --shard ${{ matrix.shard }}/3 + + build-and-test: + name: CPU build and tests + if: ${{ !cancelled() }} + needs: [converter-shared, frontend, launcher, rr-plugin, python] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: All CPU jobs passed + env: + RESULTS: ${{ toJSON(needs) }} + run: | + echo "$RESULTS" + test "$(echo "$RESULTS" | jq -r '[.[].result] | all(. == "success")')" = true diff --git a/tools/ci/run_python_tests.py b/tools/ci/run_python_tests.py index c0383024..bbed5d27 100644 --- a/tools/ci/run_python_tests.py +++ b/tools/ci/run_python_tests.py @@ -5,7 +5,17 @@ with `discover` in that folder). Skips are fine (each module states its reason: the host lane emulator needs Linux, a compiler or a file that is not in the repository, an opt-in GPU variable); a failure or an error fails the run. `clean/wgshared/tests/*.py` are scripts that the wgshared ctest runs, so they are not run here. -Run from the repository root: python tools/ci/run_python_tests.py [--jobs N] + +CI runs the set in parallel jobs, which together run exactly the same tests as one run without options: + --group proxy only the modules that need the Ray Reconstruction plugin's CPU build (WG_RR_PROXY_BUILD), run in the + job that builds the plugin; + --shard K/N part K of N of all other modules. The modules in SPLIT_BY_TEST (long, independent test methods) are + split into SPLIT_CHUNKS lists of test ids; units are spread over the N parts by their measured time + (WEIGHTS; a module without an entry counts DEFAULT_WEIGHT). Times only change the balance: every + shard computes the whole plan and checks that each module (or each test id of a split module) is in + exactly one part. + --plan N print the whole plan for N shards (and the proxy group) and exit. +Run from the repository root: python tools/ci/run_python_tests.py [--jobs N] [--group proxy | --shard K/N | --plan N] """ import argparse import concurrent.futures as cf @@ -24,14 +34,50 @@ 'clean/rr_f/proxy/tests/test_capture_replay.py', 'clean/rr_f/runtime/tests/test_model_pack.py', } RUN_ELSEWHERE = {'clean/wgshared/tests/test_dummy_extension.py', 'clean/wgshared/tests/test_gen_registry.py'} +PROXY_GROUP = {'clean/rr_f/proxy/tests/test_optimal.py'} # reads test_proxy_core.exe from WG_RR_PROXY_BUILD +SPLIT_BY_TEST = {'clean/rr_f/tests/test_stages.py'} +SPLIT_CHUNKS = 6 +# seconds per module in CI (run 43531456640, 3 modules at a time); balance only +WEIGHTS = { + 'clean/rr_f/tests/test_stages.py': 812, 'clean/rr_f/tests/test_vectors.py': 240, + 'clean/tests/test_post_tf_geometry.py': 145, 'clean/rr_f/head/tests/test_stage.py': 98, + 'clean/rr_f/tests/test_scalar.py': 80, 'clean/rr_f/tests/test_input_stage.py': 72, + 'clean/k/tests/test_k_runtime_pack.py': 51, 'clean/rr_f/tests/test_network.py': 48, + 'clean/rr_f/tests/test_colour_symmetry.py': 46, 'clean/k/tests/test_k_runtime_tables.py': 42, + 'clean/rr_f/head/tests/test_scalar_rules.py': 41, 'clean/rr_f/tests/test_gpu_pipeline.py': 40, + 'clean/rr_f/tests/test_gpu_network.py': 40, 'clean/rr_f/tests/test_prepared_pipeline.py': 39, + 'clean/rr_f/tests/test_input_lifecycle.py': 39, 'clean/rr_f/tests/test_resolve.py': 30, + 'clean/tests/test_encoder_dither.py': 27, 'clean/rr_f/tests/test_block.py': 25, + 'clean/tests/test_model_filter.py': 17, 'clean/tests/test_runtime_frame_trace.py': 14, +} +DEFAULT_WEIGHT = 5 -def run_one(root, rel, timeout): +def module_args(root, rel): d = (root / rel).parent if (d / '__init__.py').exists(): - args = [sys.executable, '-B', '-m', 'unittest', '-q', rel[:-3].replace('/', '.')] - else: - args = [sys.executable, '-B', '-m', 'unittest', 'discover', '-q', '-s', str(d), '-t', str(d), '-p', Path(rel).name] + return [sys.executable, '-B', '-m', 'unittest', '-q', rel[:-3].replace('/', '.')] + return [sys.executable, '-B', '-m', 'unittest', 'discover', '-q', '-s', str(d), '-t', str(d), '-p', Path(rel).name] + + +def test_ids(root, rel): + """The test ids of a module in a package folder, sorted.""" + code = ('import sys, unittest\n' + 'def walk(s):\n' + ' for t in s:\n' + ' yield from (walk(t) if isinstance(t, unittest.TestSuite) else [t.id()])\n' + 'print("\\n".join(sorted(walk(unittest.defaultTestLoader.loadTestsFromName(sys.argv[1])))))\n') + r = subprocess.run([sys.executable, '-B', '-c', code, rel[:-3].replace('/', '.')], cwd=root, capture_output=True, + text=True, timeout=600) + if r.returncode != 0: + raise SystemExit('cannot list the tests of %s:\n%s' % (rel, (r.stdout + r.stderr)[-4000:])) + return [l for l in r.stdout.splitlines() if l.strip()] + + +def run_one(root, unit, timeout): + """unit: (label, rel, test ids or None).""" + label, rel, ids = unit + args = module_args(root, rel) if ids is None else [sys.executable, '-B', '-m', 'unittest', '-q'] + ids t0 = time.time() try: r = subprocess.run(args, cwd=root, capture_output=True, text=True, timeout=timeout) @@ -42,13 +88,51 @@ def run_one(root, rel, timeout): rc = 'timeout' ran = re.findall(r'Ran (\d+) tests?', out) skipped = re.findall(r'skipped=(\d+)', out) - return rel, rc, int(ran[-1]) if ran else 0, int(skipped[-1]) if skipped else 0, time.time() - t0, out + return label, rc, int(ran[-1]) if ran else 0, int(skipped[-1]) if skipped else 0, time.time() - t0, out + + +def plan(root, todo, n): + """Split todo (minus the proxy group) into n parts: [[unit, ...], ...] and the proxy group's units.""" + units = [] + for rel in todo: + if rel in PROXY_GROUP: + continue + if rel in SPLIT_BY_TEST and n > 1: + ids = test_ids(root, rel) + chunks = [ids[i::SPLIT_CHUNKS] for i in range(SPLIT_CHUNKS)] + w = WEIGHTS.get(rel, DEFAULT_WEIGHT) / SPLIT_CHUNKS + units += [(w, ('%s [%d/%d]' % (rel, i + 1, SPLIT_CHUNKS), rel, c)) for i, c in enumerate(chunks) if c] + else: + units.append((WEIGHTS.get(rel, DEFAULT_WEIGHT), (rel, rel, None))) + parts, load = [[] for _ in range(n)], [0.0] * n + for w, u in sorted(units, key=lambda x: (-x[0], x[1][0])): # longest first, each to the least loaded part + k = min(range(n), key=lambda i: (load[i], i)) + parts[k].append(u) + load[k] += w + # every module, or every test id of a split module, in exactly one part + seen = {} + for k, p in enumerate(parts): + for label, rel, ids in p: + for key in ([rel] if ids is None else [rel + '::' + t for t in ids]): + if key in seen: + raise SystemExit('plan error: %s in parts %d and %d' % (key, seen[key] + 1, k + 1)) + seen[key] = k + covered = {key.split('::')[0] for key in seen} + missing = [rel for rel in todo if rel not in PROXY_GROUP and rel not in covered] + if missing: + raise SystemExit('plan error: not in any part: %s' % missing) + proxy = [(rel, rel, None) for rel in todo if rel in PROXY_GROUP] + return parts, load, proxy def main(): ap = argparse.ArgumentParser() ap.add_argument('--jobs', type=int, default=2) ap.add_argument('--timeout', type=int, default=2400) + g = ap.add_mutually_exclusive_group() + g.add_argument('--shard', help='K/N: part K of N of the modules outside the proxy group') + g.add_argument('--group', choices=['proxy'], help='only the modules that need the plugin build') + g.add_argument('--plan', type=int, metavar='N', help='print the plan for N shards and exit') a = ap.parse_args() root = Path(subprocess.check_output(['git', 'rev-parse', '--show-toplevel'], text=True).strip()) os.environ.setdefault('PYTHONDONTWRITEBYTECODE', '1') @@ -56,14 +140,36 @@ def main(): todo = [f for f in files if f not in GPU_MODULES and f not in RUN_ELSEWHERE] print('python tests: %d files, %d run here, %d GPU modules left out, %d run by ctest' % (len(files), len(todo), len(GPU_MODULES & set(files)), len(RUN_ELSEWHERE & set(files))), flush=True) + if a.plan: + parts, load, proxy = plan(root, todo, a.plan) + for k, p in enumerate(parts): + print('shard %d/%d: %d units, about %.0f s of test time' % (k + 1, a.plan, len(p), load[k])) + for label, rel, ids in p: + print(' ' + label + ('' if ids is None else ' (%d tests)' % len(ids))) + print('proxy group: ' + ', '.join(u[0] for u in proxy)) + return 0 + if a.shard: + m = re.fullmatch(r'(\d+)/(\d+)', a.shard) + k, n = (int(m.group(1)), int(m.group(2))) if m else (0, 0) + if not 1 <= k <= n: + raise SystemExit('--shard K/N with 1 <= K <= N') + parts, load, _ = plan(root, todo, n) + units = parts[k - 1] + print('python tests: shard %d/%d, %d units (about %.0f s of test time)' % (k, n, len(units), load[k - 1]), + flush=True) + elif a.group == 'proxy': + units = [(rel, rel, None) for rel in todo if rel in PROXY_GROUP] + print('python tests: proxy group, %d module(s)' % len(units), flush=True) + else: + units = [(rel, rel, None) for rel in todo] failed, total_ran, total_skipped = [], 0, 0 with cf.ThreadPoolExecutor(max(1, a.jobs)) as ex: - for rel, rc, ran, skipped, secs, out in ex.map(lambda f: run_one(root, f, a.timeout), todo): + for label, rc, ran, skipped, secs, out in ex.map(lambda u: run_one(root, u, a.timeout), units): total_ran += ran total_skipped += skipped - print('%-60s exit %-7s ran %4d skipped %3d %5.0f s' % (rel, rc, ran, skipped, secs), flush=True) + print('%-60s exit %-7s ran %4d skipped %3d %5.0f s' % (label, rc, ran, skipped, secs), flush=True) if rc != 0: - failed.append(rel) + failed.append(label) print(out[-6000:], flush=True) print('python tests: %d ran, %d skipped, %d module(s) failed%s' % (total_ran, total_skipped, len(failed), (': ' + ', '.join(failed)) if failed else ''), flush=True) From 6ee7d996e272bdd5e2813fb5be8536776bc93d95 Mon Sep 17 00:00:00 2001 From: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:18:48 -0500 Subject: [PATCH 2/2] CI: the final CPU check always runs A cancelled run skipped the gate job, and a skipped required check counts as passing. With always() the gate runs and fails unless every split job reports success. Signed-off-by: phantoslayer21 <156617979+phantoslayer21@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a83df37..58424210 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -171,7 +171,7 @@ jobs: build-and-test: name: CPU build and tests - if: ${{ !cancelled() }} + if: ${{ always() }} needs: [converter-shared, frontend, launcher, rr-plugin, python] runs-on: ubuntu-24.04 timeout-minutes: 5