From f9bb490c1e4986363b850ddf945cbad9d52d7456 Mon Sep 17 00:00:00 2001 From: chaowick Date: Fri, 14 Aug 2026 00:01:25 +0800 Subject: [PATCH 1/3] feat(moonep): add managed MindSpeed model runner --- .gitignore | 2 + docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md | 2 + .../moonep_torch/tilexr_moonep/abi.py | 11 + .../moonep_torch/tilexr_moonep/runtime.py | 8 +- tests/moonep/python/test_ffi_unittest.py | 21 + .../python/test_mindspeed_model_runner.py | 213 ++++++++++ tools/moonep/mindspeed/README.md | 55 ++- tools/moonep/mindspeed/run_model.sh | 375 ++++++++++++++++++ tools/moonep/mindspeed/run_model_node.sh | 366 +++++++++++++++++ .../mindspeed/tilexr_mindspeed_adapter.py | 40 +- 10 files changed, 1083 insertions(+), 10 deletions(-) create mode 100644 tests/moonep/python/test_mindspeed_model_runner.py create mode 100755 tools/moonep/mindspeed/run_model.sh create mode 100755 tools/moonep/mindspeed/run_model_node.sh diff --git a/.gitignore b/.gitignore index 2e797e7..ec07d61 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,8 @@ op-simulator/test_template *.log run/ +# Cached answers for the interactive MindSpeed model runner are runtime-only. +run/moonep/mindspeed/model_runner.env env/cann/ env/temp/ env/ diff --git a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md index fd4a72a..d6694d5 100644 --- a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md +++ b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md @@ -40,6 +40,7 @@ Python 侧的 `plan.status.zero_()` 不是可靠修复：NPU task queue 和 Kern | ReduceGrad 第二轮 CQ 失败，`entryIdx=0x4000` | `entryIdx` 携带 SQ cycle，原实现错误地要求它小于 ring depth | 先按 depth 归一化，再依据绝对 SQ tail 计算完成 BB 数 | 两轮同 QP 精确复现；dump raw CQE、SQ head/tail、outstanding | | Combine 后复用 plan 的反向 Dispatch 超时 | 旧 `status=3000` 违反 URMA 输入协议，异步 Python reset 又存在跨 stream 竞态 | consumer Host 在同一 stream reset；先检查旧首错误，不能掩盖真实失败 | 生产规模 oracle 对比不清理、异步清理、同步清理和 Host same-stream reset | | 4K grouped Dispatch 超时或 SQ 满 | 全量 route 无法一次装入 UB，WQE 也不能一次塞入 SQ | route tiling、WQE 分批发布、每批 CQ 回收，以 `head-tail` 计算 outstanding | case 15 生产规模单算子和 H=7168 grouped oracle | +| 双机 ReduceGrad prepare 返回 `-4`，HCCP 为 `528101` | packed adapter 的占位 Up source 只有 2 KiB，逻辑 view 同时被当作 MR 注册范围 | 保持 source shape/bytes 不变，为 view 提供实机验证过的 2 MiB backing，并在 FFI 中分别描述逻辑 source 与 registration storage | 双机日志定位失败 region/bytes/返回码；两机 NPU probe 证明 source 2 KiB、MR 2 MiB；2 机 16 卡完整模型 8/8 迭代通过 | “重复 MR 注册泄漏”“完全没有 poll CQ”“peer 调度不对称”都曾是合理假设，但被后续 A/B、原始队列状态和成功对照推翻，不应继续作为既定根因传播。 @@ -179,6 +180,7 @@ reset 等单一变量。一次同时修改 Kernel、Host、timeout 和路由， - 不要用 toy shape 证明生产规模的 UB/SQ 容量安全。 - 不要依赖已消费 SQE 的内容推导 CQ completion。 - 不要在调试运行和性能运行之间复用未审计的环境变量。 +- 不要把小 Tensor view 的逻辑字节数等同于 HCCP MR 范围；扩大 backing storage 时保持逻辑 shape 不变，否则会污染算子工作量和性能数据。 - 不要因某次补丁通过完整模型就跳过最小 reproducer；最小 reproducer 才能证明因果。 - 不要删除被推翻的假设记录。保留否定证据可以防止后续重复猜测。 diff --git a/integrations/moonep_torch/tilexr_moonep/abi.py b/integrations/moonep_torch/tilexr_moonep/abi.py index 7033101..6730104 100644 --- a/integrations/moonep_torch/tilexr_moonep/abi.py +++ b/integrations/moonep_torch/tilexr_moonep/abi.py @@ -260,6 +260,17 @@ def tensor_ptr(tensor) -> ctypes.c_void_p: return ctypes.c_void_p(int(tensor.data_ptr())) +def tensor_registration_range(tensor) -> tuple[ctypes.c_void_p, int]: + backing = getattr(tensor, "_tilexr_registration_backing", tensor) + base = int(backing.data_ptr()) + size = tensor_nbytes(backing) + data = int(tensor.data_ptr()) + logical_bytes = tensor_nbytes(tensor) + if base <= data and logical_bytes <= size - (data - base): + return ctypes.c_void_p(base), size + raise ValueError("tensor view is outside its registration backing") + + def make_tensor_v1(tensor) -> TileXRMoonEPTensorV1: shape = tuple(int(value) for value in tensor.shape) if not shape or len(shape) > TILEXR_MOONEP_MAX_TENSOR_RANK: diff --git a/integrations/moonep_torch/tilexr_moonep/runtime.py b/integrations/moonep_torch/tilexr_moonep/runtime.py index 2493039..f6430d6 100644 --- a/integrations/moonep_torch/tilexr_moonep/runtime.py +++ b/integrations/moonep_torch/tilexr_moonep/runtime.py @@ -30,6 +30,7 @@ make_tensor_v1, tensor_nbytes, tensor_ptr, + tensor_registration_range, void_p, ) @@ -893,11 +894,14 @@ def _reduce_grad_source_slices( ) -> tuple[TileXRMoonEPReduceGradSourceSliceV2, ...]: slices = [] for source, registration in zip(sources, registrations): + registration_base, registration_bytes = tensor_registration_range( + registration + ) value = TileXRMoonEPReduceGradSourceSliceV2() value.data = tensor_ptr(source) value.bytes = tensor_nbytes(source) - value.registrationBase = tensor_ptr(registration) - value.registrationBytes = tensor_nbytes(registration) + value.registrationBase = registration_base + value.registrationBytes = registration_bytes slices.append(value) return tuple(slices) diff --git a/tests/moonep/python/test_ffi_unittest.py b/tests/moonep/python/test_ffi_unittest.py index 30340ec..bdc6a3e 100644 --- a/tests/moonep/python/test_ffi_unittest.py +++ b/tests/moonep/python/test_ffi_unittest.py @@ -425,6 +425,27 @@ def tensor(shape, dtype): class FfiAbiTests(unittest.TestCase): + def test_reduce_grad_registration_uses_storage_without_expanding_source(self): + backing = FakeTensor((2 * 1024 * 1024 // 4,), "float32") + backing._ptr = 0x200000 + registration = backing.narrow(0, 1024, 512) + registration._tilexr_registration_backing = backing + + source = FakeTensor((4, 16), "float32") + slices = TileXRMoonEPRuntime._reduce_grad_source_slices( + (source, source, source), + (registration, registration, registration), + ) + self.assertEqual(tuple(int(value.bytes) for value in slices), (256,) * 3) + self.assertEqual( + tuple(int(value.registrationBase) for value in slices), + (0x200000,) * 3, + ) + self.assertEqual( + tuple(int(value.registrationBytes) for value in slices), + (2 * 1024 * 1024,) * 3, + ) + def test_dispatch_completion_flag_matrix_format(self): flags = bytearray(512 * 2 * 8) struct.pack_into(" str: + candidate = shutil.which("bash") + if candidate: + return candidate + if GIT_BASH.is_file(): + return str(GIT_BASH) + pytest.skip("bash is unavailable") + + +def run_controller(*args: str, stdin: str = "") -> subprocess.CompletedProcess[str]: + return subprocess.run( + [bash_executable(), str(CONTROLLER), *args], + cwd=ROOT, + input=stdin, + text=True, + capture_output=True, + timeout=20, + check=False, + ) + + +def answers(nodes: str = "node-a node-b", tilexr_home: str = "/srv/Tile XR") -> str: + return "\n".join( + ( + nodes, + "root", + "8", + tilexr_home, + "/srv/model stack", + f"{tilexr_home}/install", + "/home/pkg/b131/cann/set_env.sh", + "/home/miniconda3/etc/profile.d/conda.sh", + "ai_moe_test", + "/srv/model stack/native.env", + "/home/dataset/deepseek3", + "/home/dataset/deepseek3/enwiki_text_document", + ) + ) + "\n" + + +def test_runner_scripts_expose_the_supported_interface_and_validated_shape() -> None: + controller = CONTROLLER.read_text(encoding="utf-8") + node = NODE_RUNNER.read_text(encoding="utf-8") + + for option in ( + "--mode", + "--backend", + "--profile", + "--stage-barrier", + "--configure", + "--config", + "--dry-run", + "--idle-wait", + ): + assert option in controller + for option in ( + "--node-count", + "--node-rank", + "--master-addr", + "--master-port", + "--devices-per-node", + ): + assert option in node + + for argument in ( + "--num-layers 4", + "--seq-length 4096", + "--hidden-size 7168", + "--num-experts 32", + "--moe-router-topk 8", + "--moonep-token-padding 1", + "--train-iters 8", + ): + assert argument in node + assert "TILEXR_MOONEP_DISPATCH_PEER_MODE=group" in node + assert "TILEXR_MOONEP_DISPATCH_GROUP_WIDTH=16" in node + assert "TILEXR_MOONEP_COMBINE_VERSION=2" in node + assert "unset TILEXR_MOONEP_DISPATCH_TRANSPORT" in node + + +def test_first_run_prompts_and_subsequent_dry_run_reuses_cached_answers(tmp_path: Path) -> None: + config = tmp_path / "runner.env" + first = run_controller( + "--mode", + "multi", + "--backend", + "tilexr", + "--config", + str(config), + "--dry-run", + stdin=answers(), + ) + assert first.returncode == 0, first.stderr + assert config.is_file() + assert "MODEL_RUNNER_NODES=node-a\\ node-b" in config.read_text(encoding="utf-8") + assert "password" not in config.read_text(encoding="utf-8").lower() + assert "node_rank=0 host=node-a" in first.stdout + assert "node_rank=1 host=node-b" in first.stdout + assert "--node-count 2" in first.stdout + assert "--node-rank 1" in first.stdout + assert "/srv/Tile\\ XR/tools/moonep/mindspeed/run_model_node.sh" in first.stdout + + reused = run_controller( + "--mode", + "multi", + "--backend", + "tilexr", + "--config", + str(config), + "--dry-run", + ) + assert reused.returncode == 0, reused.stderr + assert reused.stdout == first.stdout + assert "Enter" not in reused.stderr + + +def test_configure_is_the_only_way_to_replace_cached_answers(tmp_path: Path) -> None: + config = tmp_path / "runner.env" + created = run_controller( + "--mode", "multi", "--config", str(config), "--dry-run", stdin=answers() + ) + assert created.returncode == 0, created.stderr + + ignored_input = run_controller( + "--mode", + "multi", + "--config", + str(config), + "--dry-run", + stdin=answers(nodes="replacement-a replacement-b"), + ) + assert ignored_input.returncode == 0, ignored_input.stderr + assert "host=node-a" in ignored_input.stdout + assert "replacement-a" not in ignored_input.stdout + + updated = run_controller( + "--mode", + "multi", + "--config", + str(config), + "--configure", + "--dry-run", + stdin=answers(nodes="replacement-a replacement-b"), + ) + assert updated.returncode == 0, updated.stderr + assert "host=replacement-a" in updated.stdout + assert "host=node-a" not in updated.stdout + + +def test_single_mode_is_local_and_multi_mode_uses_system_ssh(tmp_path: Path) -> None: + config = tmp_path / "runner.env" + configured = run_controller( + "--mode", "single", "--config", str(config), "--dry-run", stdin=answers() + ) + assert configured.returncode == 0, configured.stderr + assert "mode=single local=1 node_rank=0" in configured.stdout + assert "ssh " not in configured.stdout + assert "--node-count 1" in configured.stdout + + multi = run_controller( + "--mode", "multi", "--config", str(config), "--dry-run" + ) + assert multi.returncode == 0, multi.stderr + assert multi.stdout.count("ssh ") == 2 + assert "root@node-a" in multi.stdout + assert "root@node-b" in multi.stdout + assert "--master-addr node-a" in multi.stdout + + +def test_scripts_do_not_invoke_file_transfer_tools_and_define_failure_cleanup() -> None: + controller = CONTROLLER.read_text(encoding="utf-8") + node = NODE_RUNNER.read_text(encoding="utf-8") + for script in (controller, node): + assert "scp " not in script + assert "rsync " not in script + assert "cleanup_remote_runs" in controller + assert "wait_for_stable_idle" in controller + assert 'consecutive=$((consecutive + 1))' in controller + assert 'wait "${cleanup_pid}" || true' in controller + assert "trap 'handle_signal" in controller + assert "remaining=$((remaining - 1))" in controller + assert "No node reported completion of iteration 8/8" in controller + assert "status=91" not in node + assert "runner.pid" in node + assert "kill -- -\"${model_pid}\"" in node + + +def test_cached_config_is_explicitly_ignored() -> None: + ignored = subprocess.run( + ["git", "check-ignore", "-v", "run/moonep/mindspeed/model_runner.env"], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + assert ignored.returncode == 0 + assert "run/moonep/mindspeed/model_runner.env" in ignored.stdout diff --git a/tools/moonep/mindspeed/README.md b/tools/moonep/mindspeed/README.md index 900cc66..703b055 100644 --- a/tools/moonep/mindspeed/README.md +++ b/tools/moonep/mindspeed/README.md @@ -1,7 +1,7 @@ # MindSpeed validation tools This directory contains reusable tools developed while validating the TileXR MoonEP -backend with MindSpeed on a single eight-device Ascend 950 node. +backend with MindSpeed on Ascend 950 nodes. - `tilexr_mindspeed_adapter.py`: MindSpeed backend adapter that keeps communication buffers owned by TileXR. @@ -13,6 +13,10 @@ backend with MindSpeed on a single eight-device Ascend 950 node. oracle at the production `S=4096`, `K=8`, EP8 route shape. - `run_case15_32.sh`: 32-iteration case 15 runner. - `run_grouped_oracle.sh`: eight-rank grouped-URMA oracle runner. +- `run_model.sh`: single entry point for local single-node runs and controller-side + multi-node SSH orchestration. +- `run_model_node.sh`: non-interactive per-node runner based on the validated + 4K/8P model command. The runners use the current repository by default and write results below `run/moonep/mindspeed`. Override paths with `TILEXR_HOME`, @@ -31,3 +35,52 @@ MINDSPEED_HOME="${MINDSPEED_HOME}" \ The preflight installs `tilexr_mindspeed_adapter.py` into the checkout named by `MINDSPEED_HOME`; use a disposable or task-owned MindSpeed checkout. + +## Full model runner + +The first invocation asks for the node list and deployed paths, then writes the +answers to the ignored runtime file +`run/moonep/mindspeed/model_runner.env`. Later invocations reuse it without +prompting. Use `--configure` only when the saved deployment information must be +replaced. The file contains paths and hostnames, never SSH passwords. + +Run a single-node TileXR model locally: + +```bash +bash tools/moonep/mindspeed/run_model.sh \ + --mode single --backend tilexr +``` + +From the controller host, start all configured nodes concurrently over system +SSH: + +```bash +bash tools/moonep/mindspeed/run_model.sh \ + --mode multi --backend tilexr +``` + +The controller maps node ranks in the saved host order; the first host is rank 0 +and supplies `MASTER_ADDR`. It verifies that `run_model_node.sh` exists on every +node before starting any model process, records one controller log per node, and +stops peer process groups if a node fails or the controller is interrupted. +It also waits for three consecutive all-node idle samples before launch; use +`--idle-wait` to bound that wait on a shared test machine. + +Useful optional modes: + +```bash +# Inspect the exact local or SSH commands without launching. +bash tools/moonep/mindspeed/run_model.sh --mode multi --dry-run + +# Replace the cached answers. +bash tools/moonep/mindspeed/run_model.sh --configure --mode multi --dry-run + +# Enable one profiler window. The diagnostic stage barrier is off by default. +bash tools/moonep/mindspeed/run_model.sh --mode multi --profile +bash tools/moonep/mindspeed/run_model.sh --mode multi --stage-barrier +``` + +SSH authentication is owned by the system client. Configure a key, agent, +terminal password prompt, or askpass outside this tool. The runner does not copy +the repository: deploy identical code and builds to every node with Mutagen +before starting a multi-node run. diff --git a/tools/moonep/mindspeed/run_model.sh b/tools/moonep/mindspeed/run_model.sh new file mode 100755 index 0000000..9f6c340 --- /dev/null +++ b/tools/moonep/mindspeed/run_model.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +default_tilexr_home=$(cd "${script_dir}/../../.." && pwd) +default_config=${default_tilexr_home}/run/moonep/mindspeed/model_runner.env + +usage() { + cat <<'EOF' +Usage: run_model.sh [options] + +Options: + --mode single|multi Launch locally or orchestrate all nodes over SSH. + --backend tilexr|native Select the MoonEP backend (default: tilexr). + --profile Enable the one-iteration NPU profiler window. + --stage-barrier Add diagnostic world barriers before Dispatch/Combine. + --configure Replace the cached configuration interactively. + --config PATH Use an alternate cached configuration file. + --master-port PORT Distributed launcher port (default: 29501). + --run-tag TAG Result directory name (default: timestamped). + --timeout SECONDS Per-node model timeout (default: 900). + --idle-wait SECONDS Wait for a stable all-node idle window (default: 600). + --dry-run Print commands without connecting or launching. + -h, --help Show this help. + +The first run prompts for deployment paths and nodes, then caches the answers. +Only --configure updates an existing cache. SSH authentication remains external +to this script; passwords are never stored. Deploy code separately with Mutagen. +EOF +} + +mode=single +backend=tilexr +profile=0 +stage_barrier=0 +configure=0 +dry_run=0 +config=${TILEXR_MODEL_RUNNER_CONFIG:-${default_config}} +master_port=29501 +timeout_sec=900 +idle_wait_sec=600 +run_tag= + +while [[ $# -gt 0 ]]; do + case "$1" in + --mode) mode=${2:?--mode requires a value}; shift 2 ;; + --backend) backend=${2:?--backend requires a value}; shift 2 ;; + --profile) profile=1; shift ;; + --stage-barrier) stage_barrier=1; shift ;; + --configure) configure=1; shift ;; + --config) config=${2:?--config requires a value}; shift 2 ;; + --master-port) master_port=${2:?--master-port requires a value}; shift 2 ;; + --run-tag) run_tag=${2:?--run-tag requires a value}; shift 2 ;; + --timeout) timeout_sec=${2:?--timeout requires a value}; shift 2 ;; + --idle-wait) idle_wait_sec=${2:?--idle-wait requires a value}; shift 2 ;; + -h|--help) usage; exit 0 ;; + --dry-run) dry_run=1; shift ;; + *) printf 'Unknown argument: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +case "${mode}" in single|multi) ;; *) printf 'Invalid mode: %s\n' "${mode}" >&2; exit 2 ;; esac +case "${backend}" in tilexr|native) ;; *) printf 'Invalid backend: %s\n' "${backend}" >&2; exit 2 ;; esac +if [[ "${stage_barrier}" -eq 1 && "${backend}" != tilexr ]]; then + echo "--stage-barrier is supported only by the TileXR backend" >&2 + exit 2 +fi +if [[ ! "${master_port}" =~ ^[0-9]+$ ]] || (( master_port < 1 || master_port > 65535 )); then + printf 'Invalid master port: %s\n' "${master_port}" >&2 + exit 2 +fi +if [[ ! "${timeout_sec}" =~ ^[0-9]+$ ]] || (( timeout_sec < 1 )); then + printf 'Invalid timeout: %s\n' "${timeout_sec}" >&2 + exit 2 +fi +if [[ ! "${idle_wait_sec}" =~ ^[0-9]+$ ]]; then + printf 'Invalid idle wait: %s\n' "${idle_wait_sec}" >&2 + exit 2 +fi +if [[ -n "${run_tag}" && ! "${run_tag}" =~ ^[A-Za-z0-9_.-]+$ ]]; then + printf 'Unsafe run tag: %s\n' "${run_tag}" >&2 + exit 2 +fi + +if [[ "${config}" =~ ^[A-Za-z]:[\\/] ]] && command -v cygpath >/dev/null 2>&1; then + config=$(cygpath -u "${config}") +fi + +prompt_value() { + local variable=$1 + local label=$2 + local default_value=$3 + local answer= + printf '%s [%s]: ' "${label}" "${default_value}" >&2 + IFS= read -r answer || true + answer=${answer%$'\r'} + printf -v "${variable}" '%s' "${answer:-${default_value}}" +} + +write_config() { + local config_dir + config_dir=$(dirname "${config}") + mkdir -p "${config_dir}" + umask 077 + { + printf '# Generated by tools/moonep/mindspeed/run_model.sh --configure\n' + local variable + for variable in \ + MODEL_RUNNER_NODES MODEL_RUNNER_SSH_USER MODEL_RUNNER_DEVICES_PER_NODE \ + MODEL_RUNNER_TILEXR_HOME MODEL_RUNNER_MODEL_ROOT MODEL_RUNNER_INSTALL_PREFIX \ + MODEL_RUNNER_CANN_ENV MODEL_RUNNER_CONDA_SH MODEL_RUNNER_CONDA_ENV \ + MODEL_RUNNER_NATIVE_ENV MODEL_RUNNER_TOKENIZER_PATH MODEL_RUNNER_DATA_PATH; do + printf '%s=%q\n' "${variable}" "${!variable}" + done + } >"${config}" + chmod 600 "${config}" +} + +configure_runner() { + local host_default + host_default=$( (hostname -I 2>/dev/null || true) | awk '{print $1}') + host_default=${host_default:-127.0.0.1} + prompt_value MODEL_RUNNER_NODES "Node IPs or hostnames (space separated)" "${host_default}" + prompt_value MODEL_RUNNER_SSH_USER "SSH user" root + prompt_value MODEL_RUNNER_DEVICES_PER_NODE "Devices per node" 8 + prompt_value MODEL_RUNNER_TILEXR_HOME "TileXR path on every node" /home/c30061605/ai/TileXR + prompt_value MODEL_RUNNER_MODEL_ROOT "Model stack root" "${MODEL_RUNNER_TILEXR_HOME}/run/multinode-validation/model" + prompt_value MODEL_RUNNER_INSTALL_PREFIX "TileXR install prefix" "${MODEL_RUNNER_TILEXR_HOME}/install" + prompt_value MODEL_RUNNER_CANN_ENV "CANN environment script" /home/pkg/b131/cann/set_env.sh + prompt_value MODEL_RUNNER_CONDA_SH "Conda shell script" /home/miniconda3/etc/profile.d/conda.sh + prompt_value MODEL_RUNNER_CONDA_ENV "Conda environment" ai_moe_test + prompt_value MODEL_RUNNER_NATIVE_ENV "Native MoonEP environment script" "${MODEL_RUNNER_MODEL_ROOT}/moonep-native-build-97350ce0/moonep-native.env" + prompt_value MODEL_RUNNER_TOKENIZER_PATH "Tokenizer path" /home/dataset/deepseek3 + prompt_value MODEL_RUNNER_DATA_PATH "Training data prefix" /home/dataset/deepseek3/enwiki_text_document + write_config +} + +if [[ "${configure}" -eq 1 || ! -f "${config}" ]]; then + configure_runner +fi + +# shellcheck disable=SC1090 +source "${config}" +required_variables=( + MODEL_RUNNER_NODES MODEL_RUNNER_SSH_USER MODEL_RUNNER_DEVICES_PER_NODE + MODEL_RUNNER_TILEXR_HOME MODEL_RUNNER_MODEL_ROOT MODEL_RUNNER_INSTALL_PREFIX + MODEL_RUNNER_CANN_ENV MODEL_RUNNER_CONDA_SH MODEL_RUNNER_CONDA_ENV + MODEL_RUNNER_NATIVE_ENV MODEL_RUNNER_TOKENIZER_PATH MODEL_RUNNER_DATA_PATH +) +for variable in "${required_variables[@]}"; do + if [[ -z "${!variable:-}" ]]; then + printf 'Missing %s in %s; run with --configure\n' "${variable}" "${config}" >&2 + exit 2 + fi +done +if [[ ! "${MODEL_RUNNER_DEVICES_PER_NODE}" =~ ^[0-9]+$ ]] || \ + (( MODEL_RUNNER_DEVICES_PER_NODE < 1 )); then + printf 'Invalid devices per node in %s\n' "${config}" >&2 + exit 2 +fi + +node_spec=${MODEL_RUNNER_NODES//,/ } +read -r -a nodes <<<"${node_spec}" +if [[ ${#nodes[@]} -eq 0 ]]; then + printf 'No nodes configured in %s\n' "${config}" >&2 + exit 2 +fi +if [[ "${mode}" == multi && ${#nodes[@]} -lt 2 ]]; then + printf 'Multi-node mode needs at least two configured nodes\n' >&2 + exit 2 +fi + +if [[ -z "${run_tag}" ]]; then + if [[ "${dry_run}" -eq 1 ]]; then + run_tag=dry-run + else + run_tag="model_${backend}_${mode}_$(date +%Y%m%d-%H%M%S)_$$" + fi +fi + +quote_command() { + local quoted=() + local value + for value in "$@"; do + printf -v value '%q' "${value}" + quoted+=("${value}") + done + local IFS=' ' + printf '%s' "${quoted[*]}" +} + +node_arguments() { + local node_count=$1 + local node_rank=$2 + local master_addr=$3 + local args=( + --backend "${backend}" + --node-count "${node_count}" + --node-rank "${node_rank}" + --master-addr "${master_addr}" + --master-port "${master_port}" + --devices-per-node "${MODEL_RUNNER_DEVICES_PER_NODE}" + --tilexr-home "${MODEL_RUNNER_TILEXR_HOME}" + --model-root "${MODEL_RUNNER_MODEL_ROOT}" + --install-prefix "${MODEL_RUNNER_INSTALL_PREFIX}" + --cann-env "${MODEL_RUNNER_CANN_ENV}" + --conda-sh "${MODEL_RUNNER_CONDA_SH}" + --conda-env "${MODEL_RUNNER_CONDA_ENV}" + --native-env "${MODEL_RUNNER_NATIVE_ENV}" + --tokenizer-path "${MODEL_RUNNER_TOKENIZER_PATH}" + --data-path "${MODEL_RUNNER_DATA_PATH}" + --run-tag "${run_tag}" + --timeout "${timeout_sec}" + ) + [[ "${profile}" -eq 1 ]] && args+=(--profile) + [[ "${stage_barrier}" -eq 1 ]] && args+=(--stage-barrier) + quote_command "${args[@]}" +} + +remote_runner=${MODEL_RUNNER_TILEXR_HOME}/tools/moonep/mindspeed/run_model_node.sh +local_runner=${script_dir}/run_model_node.sh +ssh_options=(-o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=3) + +if [[ "${mode}" == single ]]; then + args=$(node_arguments 1 0 127.0.0.1) + command=$(quote_command bash "${local_runner}") + command+=" ${args}" + if [[ "${dry_run}" -eq 1 ]]; then + printf 'mode=single local=1 node_rank=0\n%s\n' "${command}" + exit 0 + fi + eval "${command}" + exit $? +fi + +node_count=${#nodes[@]} +master_addr=${nodes[0]} +commands=() +targets=() +for node_rank in "${!nodes[@]}"; do + target=${MODEL_RUNNER_SSH_USER}@${nodes[${node_rank}]} + args=$(node_arguments "${node_count}" "${node_rank}" "${master_addr}") + command=$(quote_command bash "${remote_runner}") + command+=" ${args}" + targets+=("${target}") + commands+=("${command}") + if [[ "${dry_run}" -eq 1 ]]; then + printf 'node_rank=%s host=%s\n' "${node_rank}" "${nodes[${node_rank}]}" + printf 'ssh %q %s\n' "${target}" "${command}" + fi +done +[[ "${dry_run}" -eq 1 ]] && exit 0 + +controller_dir=${default_tilexr_home}/run/moonep/mindspeed/${run_tag}/controller +mkdir -p "${controller_dir}" + +for index in "${!targets[@]}"; do + probe=$(quote_command test -f "${remote_runner}") + if ! ssh "${ssh_options[@]}" "${targets[${index}]}" "${probe}"; then + printf 'SSH preflight failed for %s\n' "${targets[${index}]}" >&2 + exit 1 + fi +done + +wait_for_stable_idle() { + local deadline=$((SECONDS + idle_wait_sec)) + local consecutive=0 + local index idle all_idle + while true; do + all_idle=1 + for index in "${!targets[@]}"; do + idle=$(ssh "${ssh_options[@]}" "${targets[${index}]}" \ + "npu-smi info | grep -c 'No running processes found in NPU' || true") + if [[ "${idle}" -ne "${MODEL_RUNNER_DEVICES_PER_NODE}" ]]; then + all_idle=0 + fi + printf 'idle_gate host=%s idle=%s/%s stable=%s/3\n' \ + "${nodes[${index}]}" "${idle}" "${MODEL_RUNNER_DEVICES_PER_NODE}" \ + "${consecutive}" | tee -a "${controller_dir}/idle_gate.log" + done + if [[ "${all_idle}" -eq 1 ]]; then + consecutive=$((consecutive + 1)) + [[ "${consecutive}" -ge 3 ]] && return 0 + else + consecutive=0 + fi + if (( SECONDS >= deadline )); then + printf 'No stable all-node idle window within %s seconds\n' \ + "${idle_wait_sec}" >&2 + return 1 + fi + sleep 5 + done +} + +wait_for_stable_idle + +declare -a pids=() +cleanup_started=0 +cleanup_remote_runs() { + [[ "${cleanup_started}" -eq 1 ]] && return + cleanup_started=1 + local index stop_args stop_command cleanup_pid + local cleanup_pids=() + for index in "${!targets[@]}"; do + stop_args=$(quote_command \ + --stop --backend "${backend}" --node-rank "${index}" \ + --tilexr-home "${MODEL_RUNNER_TILEXR_HOME}" --run-tag "${run_tag}") + stop_command=$(quote_command bash "${remote_runner}") + stop_command+=" ${stop_args}" + ssh "${ssh_options[@]}" "${targets[${index}]}" "${stop_command}" \ + >"${controller_dir}/cleanup_${index}.log" 2>&1 & + cleanup_pids+=("$!") + done + for cleanup_pid in "${cleanup_pids[@]}"; do + wait "${cleanup_pid}" || true + done +} + +handle_signal() { + local signal=$1 + printf 'Received %s; stopping all remote model jobs\n' "${signal}" >&2 + cleanup_remote_runs + exit 130 +} +trap 'handle_signal INT' INT +trap 'handle_signal TERM' TERM +trap 'handle_signal HUP' HUP + +for index in "${!targets[@]}"; do + status_file=${controller_dir}/status_${index} + ( + set +e + ssh "${ssh_options[@]}" "${targets[${index}]}" "${commands[${index}]}" \ + >"${controller_dir}/node_${index}.log" 2>&1 + status=$? + printf '%s\n' "${status}" >"${status_file}" + exit "${status}" + ) & + pids+=("$!") +done + +remaining=${#pids[@]} +failed=0 +declare -a completed=() +while (( remaining > 0 )); do + for index in "${!pids[@]}"; do + [[ "${completed[${index}]:-0}" -eq 1 ]] && continue + status_file=${controller_dir}/status_${index} + [[ -f "${status_file}" ]] || continue + status=$(<"${status_file}") + wait "${pids[${index}]}" || true + completed[${index}]=1 + remaining=$((remaining - 1)) + printf 'node_rank=%s host=%s exit_code=%s log=%s\n' \ + "${index}" "${nodes[${index}]}" "${status}" \ + "${controller_dir}/node_${index}.log" + if [[ "${status}" -ne 0 && "${failed}" -eq 0 ]]; then + failed=1 + cleanup_remote_runs + fi + done + (( remaining > 0 )) && sleep 1 +done + +if [[ "${failed}" -eq 0 ]] && ! grep -Eq \ + 'iteration[[:space:]]+8/[[:space:]]*8' "${controller_dir}"/node_*.log; then + echo "No node reported completion of iteration 8/8" >&2 + failed=1 +fi +if [[ "${failed}" -ne 0 ]]; then + printf 'Multi-node model failed; see %s\n' "${controller_dir}" >&2 + exit 1 +fi +printf 'Multi-node model completed: %s\n' "${controller_dir}" diff --git a/tools/moonep/mindspeed/run_model_node.sh b/tools/moonep/mindspeed/run_model_node.sh new file mode 100755 index 0000000..5a4f5bf --- /dev/null +++ b/tools/moonep/mindspeed/run_model_node.sh @@ -0,0 +1,366 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: run_model_node.sh [options] + +This non-interactive node runner is normally invoked by run_model.sh. + +Required options: + --backend tilexr|native + --node-count COUNT --node-rank RANK --master-addr ADDRESS --master-port PORT + --devices-per-node COUNT --tilexr-home PATH --model-root PATH + --install-prefix PATH --cann-env PATH --conda-sh PATH --conda-env NAME + --native-env PATH --tokenizer-path PATH --data-path PATH --run-tag TAG + +Optional: --profile --stage-barrier --timeout SECONDS +Internal cleanup: --stop --backend BACKEND --node-rank RANK + --tilexr-home PATH --run-tag TAG +EOF +} + +backend= +node_count= +node_rank= +master_addr= +master_port= +devices_per_node= +tilexr_home= +model_root= +install_prefix= +cann_env= +conda_sh= +conda_env= +native_env= +tokenizer_path= +data_path= +run_tag= +timeout_sec=900 +profile=0 +stage_barrier=0 +stop=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --backend) backend=${2:?--backend requires a value}; shift 2 ;; + --node-count) node_count=${2:?--node-count requires a value}; shift 2 ;; + --node-rank) node_rank=${2:?--node-rank requires a value}; shift 2 ;; + --master-addr) master_addr=${2:?--master-addr requires a value}; shift 2 ;; + --master-port) master_port=${2:?--master-port requires a value}; shift 2 ;; + --devices-per-node) devices_per_node=${2:?--devices-per-node requires a value}; shift 2 ;; + --tilexr-home) tilexr_home=${2:?--tilexr-home requires a value}; shift 2 ;; + --model-root) model_root=${2:?--model-root requires a value}; shift 2 ;; + --install-prefix) install_prefix=${2:?--install-prefix requires a value}; shift 2 ;; + --cann-env) cann_env=${2:?--cann-env requires a value}; shift 2 ;; + --conda-sh) conda_sh=${2:?--conda-sh requires a value}; shift 2 ;; + --conda-env) conda_env=${2:?--conda-env requires a value}; shift 2 ;; + --native-env) native_env=${2:?--native-env requires a value}; shift 2 ;; + --tokenizer-path) tokenizer_path=${2:?--tokenizer-path requires a value}; shift 2 ;; + --data-path) data_path=${2:?--data-path requires a value}; shift 2 ;; + --run-tag) run_tag=${2:?--run-tag requires a value}; shift 2 ;; + --timeout) timeout_sec=${2:?--timeout requires a value}; shift 2 ;; + --profile) profile=1; shift ;; + --stage-barrier) stage_barrier=1; shift ;; + --stop) stop=1; shift ;; + -h|--help) usage; exit 0 ;; + *) printf 'Unknown argument: %s\n' "$1" >&2; usage >&2; exit 2 ;; + esac +done + +case "${backend}" in tilexr|native) ;; *) printf 'Invalid backend: %s\n' "${backend}" >&2; exit 2 ;; esac +if [[ -z "${tilexr_home}" || -z "${run_tag}" || -z "${node_rank}" ]]; then + echo "--tilexr-home, --run-tag, and --node-rank are required" >&2 + exit 2 +fi +if [[ ! "${run_tag}" =~ ^[A-Za-z0-9_.-]+$ ]] || [[ ! "${node_rank}" =~ ^[0-9]+$ ]]; then + echo "unsafe run tag or node rank" >&2 + exit 2 +fi + +output=${tilexr_home}/run/moonep/mindspeed/${run_tag}/${backend}/node_${node_rank} +stop_existing_run() { + local model_pid runner_pid + if [[ -f "${output}/model.pid" ]]; then + model_pid=$(<"${output}/model.pid") + if [[ "${model_pid}" =~ ^[0-9]+$ ]] && kill -0 "${model_pid}" 2>/dev/null; then + kill -- -"${model_pid}" 2>/dev/null || true + sleep 2 + kill -KILL -- -"${model_pid}" 2>/dev/null || true + fi + fi + if [[ -f "${output}/runner.pid" ]]; then + runner_pid=$(<"${output}/runner.pid") + if [[ "${runner_pid}" =~ ^[0-9]+$ ]] && kill -0 "${runner_pid}" 2>/dev/null; then + kill "${runner_pid}" 2>/dev/null || true + fi + fi +} +if [[ "${stop}" -eq 1 ]]; then + stop_existing_run + exit 0 +fi + +required_values=( + node_count master_addr master_port devices_per_node model_root install_prefix + cann_env conda_sh conda_env native_env tokenizer_path data_path +) +for variable in "${required_values[@]}"; do + if [[ -z "${!variable:-}" ]]; then + printf 'Missing required option for %s\n' "${variable}" >&2 + exit 2 + fi +done +for variable in node_count node_rank master_port devices_per_node timeout_sec; do + if [[ ! "${!variable}" =~ ^[0-9]+$ ]]; then + printf '%s must be an integer\n' "${variable}" >&2 + exit 2 + fi +done +if (( node_count < 1 || node_rank >= node_count || devices_per_node < 1 || \ + master_port < 1 || master_port > 65535 || timeout_sec < 1 )); then + echo "invalid distributed launcher dimensions" >&2 + exit 2 +fi +if [[ "${stage_barrier}" -eq 1 && "${backend}" != tilexr ]]; then + echo "stage barriers require the TileXR backend" >&2 + exit 2 +fi + +for path in "${cann_env}" "${conda_sh}" "${native_env}"; do + [[ -f "${path}" ]] || { printf 'Required file not found: %s\n' "${path}" >&2; exit 1; } +done +for path in "${model_root}/MindSpeed" "${model_root}/MindSpeed-LLM" \ + "${model_root}/shmem/src/python" "${tokenizer_path}"; do + [[ -e "${path}" ]] || { printf 'Required path not found: %s\n' "${path}" >&2; exit 1; } +done + +mkdir -p "${output}" +printf '%s\n' "$$" >"${output}/runner.pid" +model_pid= +cleanup_model() { + if [[ -n "${model_pid}" ]] && kill -0 "${model_pid}" 2>/dev/null; then + kill -- -"${model_pid}" 2>/dev/null || true + sleep 2 + kill -KILL -- -"${model_pid}" 2>/dev/null || true + wait "${model_pid}" 2>/dev/null || true + fi +} +finish_runner() { + if [[ -f "${output}/runner.pid" ]] && [[ "$(<"${output}/runner.pid")" == "$$" ]]; then + rm -f "${output}/runner.pid" "${output}/model.pid" + fi +} +trap 'cleanup_model; finish_runner' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM HUP +exec > >(tee "${output}/controller.log") 2>&1 + +# shellcheck disable=SC1090 +source "${cann_env}" +# shellcheck disable=SC1090 +source "${conda_sh}" +conda activate "${conda_env}" +# shellcheck disable=SC1090 +source "${native_env}" + +for gate in 1 2; do + npu-smi info >"${output}/npu_gate_${gate}.log" + idle=$(grep -c 'No running processes found in NPU' "${output}/npu_gate_${gate}.log" || true) + if [[ "${idle}" -ne "${devices_per_node}" ]]; then + printf 'exit_code=90\nreason=npu_busy\ngate=%s\nidle=%s\n' "${gate}" "${idle}" \ + | tee "${output}/result.txt" + exit 90 + fi + [[ "${gate}" -eq 1 ]] && sleep 5 +done + +shmem_python=${model_root}/shmem/src/python +shmem_backend=${shmem_python}/shmem/backends/950 +mindspeed_home=${model_root}/MindSpeed +mindspeed_llm_home=${model_root}/MindSpeed-LLM +export PYTHONPATH="${mindspeed_home}:${shmem_python}:${mindspeed_llm_home}${PYTHONPATH:+:${PYTHONPATH}}" +export LD_LIBRARY_PATH="${shmem_backend}:${LD_LIBRARY_PATH:-}" +[[ -f /usr/lib64/libstdc++.so.6 ]] && export LD_PRELOAD=/usr/lib64/libstdc++.so.6 + +interface=${MODEL_RUNNER_SOCKET_IFNAME:-} +if [[ -z "${interface}" ]]; then + interface=$(ip route get "${master_addr}" 2>/dev/null | awk '{for (i=1; i<=NF; ++i) if ($i == "dev") {print $(i+1); exit}}') +fi +if [[ -z "${interface}" || "${interface}" == lo ]]; then + interface=$(ip -4 -brief address | awk '$1 != "lo" && $3 != "" {print $1; exit}') +fi +[[ -n "${interface}" ]] || { echo "Unable to determine communication interface" >&2; exit 1; } + +export HCCL_HOST_SOCKET_PORT_RANGE=auto +export HCCL_SOCKET_IFNAME=${interface} +export GLOO_SOCKET_IFNAME=${interface} +export HCCL_BUFFSIZE=200 +export HCCL_XN_RES_NUM=2000 +export HCCL_DISABLE_NHR=1 +export HCCL_DFS_CONFIG=task_exception:off +export HCCL_CONNECT_TIMEOUT=120 +export HCCL_EXEC_TIMEOUT=120 +export CUDA_DEVICE_MAX_CONNECTIONS=1 +export TASK_QUEUE_ENABLE=2 +export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True +export STREAMS_PER_DEVICE=32 +if [[ "${devices_per_node}" -eq 8 ]]; then + export CPU_AFFINITY_CONF=${CPU_AFFINITY_CONF:-1,npu0:192-215,npu1:216-239,npu2:0-23,npu3:24-47,npu4:48-71,npu5:72-95,npu6:240-263,npu7:264-287} +fi + +unset TILEXR_MOONEP_TRACE_STAGES TILEXR_MOONEP_TRACE_PLANNER_MAGIC +unset TILEXR_MOONEP_DEBUG_SYNC_COMBINE TILEXR_MOONEP_DEBUG_SYNC_DISPATCH_STATUS +unset TILEXR_MOONEP_DEBUG_PREFETCH_UDMA TILEXR_MOONEP_DUMP_DFX_ON_ERROR +unset TILEXR_MOONEP_FLAG_DUMP_DIR TILEXR_MOONEP_FLAG_DUMP_MODE +unset TILEXR_MINDSPEED_TRACE TILEXR_MINDSPEED_PLAN_DUMP_DIR +unset TILEXR_MINDSPEED_FORCE_DUMMY_UDMA ASCEND_LAUNCH_BLOCKING +unset PROFILING_MODE PROFILING_OPTIONS ASCEND_MOONEP_DISPATCH_ENABLE_DFX +unset ASCEND_MOONEP_DISPATCH_ENABLE_TRACE ASCEND_MOONEP_DISPATCH_TRACE + +world_size=$((node_count * devices_per_node)) +global_batch_size=${world_size} +backend_args=() +if [[ "${backend}" == tilexr ]]; then + MINDSPEED_HOME=${mindspeed_home} TILEXR_HOME=${tilexr_home} \ + TILEXR_INSTALL_PREFIX=${install_prefix} \ + bash "${tilexr_home}/tools/moonep/mindspeed/preflight_adapter.sh" + export PYTHONPATH="${mindspeed_home}:${tilexr_home}/integrations/moonep_torch:${shmem_python}:${shmem_backend}:${mindspeed_llm_home}:${tilexr_home}${PYTHONPATH:+:${PYTHONPATH}}" + export LD_LIBRARY_PATH="${install_prefix}/lib64:${shmem_backend}:${LD_LIBRARY_PATH:-}" + export TILEXR_INSTALL_PREFIX=${install_prefix} + export TILEXR_UDMA_QP_ROUTE_SPEC=port_count:6,port_count:2 + export TILEXR_UDMA_ATTACH_EXISTING_RA=1 + export TILEXR_MOONEP_DISPATCH_PEER_MODE=group + export TILEXR_MOONEP_DISPATCH_GROUP_WIDTH=16 + export TILEXR_MOONEP_COMBINE_VERSION=2 + export TILEXR_MINDSPEED_STAGE_BARRIER=${stage_barrier} + unset TILEXR_MOONEP_DISPATCH_TRANSPORT + export TILEXR_COMM_ID="${master_addr}:$((master_port + 10000))" + ep_size=${world_size} + backend_args+=( + --moonep-token-padding 1 + --moonep-backend-factory + mindspeed.core.transformer.moe.tilexr_mindspeed_adapter:create_tilexr_moonep_backend + ) +else + ep_size=${devices_per_node} +fi + +profile_output=${output}/profiling +profile_args=() +if [[ "${profile}" -eq 1 ]]; then + mkdir -p "${profile_output}" + profile_args+=( + --profile --profile-step-start 6 --profile-step-end 7 + --profile-with-cpu --profile-ranks -1 --profile-level level1 + --profile-export-type text --profile-save-path "${profile_output}" + ) +fi + +{ + printf 'backend=%s\nrun_tag=%s\nnode_rank=%s\nworld_size=%s\nrank_per_dev=1\n' \ + "${backend}" "${run_tag}" "${node_rank}" "${world_size}" + printf 'ep_size=%s\nprofile=%s\nstage_barrier=%s\ninterface=%s\n' \ + "${ep_size}" "${profile}" "${stage_barrier}" "${interface}" + printf 'shape=4K/8P layers=4 hidden=7168 experts=32 token_padding=1 iterations=8\n' + printf 'debug=disabled flag_dump=disabled plan_dump=disabled\n' + env | grep -E '^(TILEXR_|MOONEP_|ASCEND_MOONEP_|HCCL_|GLOO_SOCKET)' | sort +} >"${output}/provenance.log" + +cd "${mindspeed_llm_home}" +set +e +setsid timeout --signal=TERM --kill-after=20s "${timeout_sec}s" \ +python -m torch.distributed.launch \ + --nproc_per_node "${devices_per_node}" \ + --nnodes "${node_count}" \ + --node_rank "${node_rank}" \ + --master_addr "${master_addr}" \ + --master_port "${master_port}" \ + pretrain_gpt.py \ + --te-gmm-mode performance \ + --transformer-impl transformer_engine \ + --disable-gloo-group \ + --no-check-for-nan-in-loss-and-grad \ + --spec mindspeed_llm.tasks.models.spec.deepseek_spec layer_spec \ + --gemm-gradient-accumulation-fusion \ + --manual-gc --manual-gc-interval 50 \ + --use-distributed-optimizer --use-flash-attn --use-mcore-models \ + --tensor-model-parallel-size 1 \ + --pipeline-model-parallel-size 1 \ + --expert-model-parallel-size "${ep_size}" \ + --expert-tensor-parallel-size 1 \ + --sequence-parallel \ + --context-parallel-size 1 \ + --context-parallel-algo ulysses_cp_algo \ + --num-layers 4 \ + --hidden-size 7168 \ + --ffn-hidden-size 18432 \ + --num-attention-heads 128 \ + --tokenizer-type PretrainedFromHF \ + --tokenizer-name-or-path "${tokenizer_path}" \ + --seq-length 4096 \ + --max-position-embeddings 163840 \ + --micro-batch-size 1 \ + --global-batch-size "${global_batch_size}" \ + --make-vocab-size-divisible-by 1 \ + --lr 1.0e-5 --train-iters 8 --lr-decay-style cosine \ + --untie-embeddings-and-output-weights --disable-bias-linear \ + --attention-dropout 0.0 --hidden-dropout 0.0 --init-method-std 0.02 \ + --position-embedding-type rope --normalization RMSNorm \ + --use-fused-rotary-pos-emb --use-rotary-position-embeddings \ + --use-fused-swiglu --use-fused-rmsnorm --swiglu \ + --no-masked-softmax-fusion --attention-softmax-in-fp32 \ + --min-lr 1.0e-7 --weight-decay 1e-2 --lr-warmup-iters 0 \ + --clip-grad 1.0 --adam-beta1 0.9 --adam-beta2 0.999 \ + --initial-loss-scale 65536 \ + --vocab-size 129280 --padded-vocab-size 129280 \ + --rotary-base 10000 --norm-epsilon 1e-6 \ + --no-load-optim --no-load-rng --bf16 --distributed-timeout-minutes 2 \ + --data-path "${data_path}" \ + --split 100,0,0 \ + --log-interval 1 --save-interval 20000 --eval-interval 20000 --eval-iters 0 \ + --no-save-optim --no-save-rng --no-shared-storage --exit-interval 8 \ + --multi-latent-attention --qk-pos-emb-head-dim 64 --qk-head-dim 128 \ + --q-lora-rank 1536 --kv-lora-rank 512 --v-head-dim 128 \ + --qk-layernorm --mla-mm-split --mla-fa-without-pad \ + --moe-grouped-gemm --moe-token-dispatcher-type flex \ + --first-k-dense-replace 0 --moe-enable-moonep --moe-layer-freq 1 \ + --moe-shared-expert-intermediate-size 2048 --num-experts 32 \ + --moe-router-topk 8 --moe-ffn-hidden-size 2048 \ + --moe-router-load-balancing-type seq_aux_loss \ + --moe-router-num-groups 8 --moe-router-group-topk 4 \ + --moe-router-topk-scaling-factor 2.5 --moe-aux-loss-coeff 0.0001 \ + --norm-topk-prob --moe-router-score-function sigmoid \ + --moe-router-enable-expert-bias --moe-router-dtype fp32 \ + --mtp-num-layers 1 --mtp-loss-scaling-factor 0.3 \ + --mtp-mem-efficient-logits --recompute-activation-function \ + --recompute-mla-up-proj --swap-optimizer --swap-optimizer-times 16 \ + --beta-fast 32 --beta-slow 1 --rope-scaling-factor 40 \ + --rope-scaling-mscale 1.0 --rope-scaling-mscale-all-dim 1.0 \ + --rope-scaling-original-max-position-embeddings 4096 \ + --rope-scaling-type yarn \ + --moonep-full-vmm-mode performance --moonep-zero-copy-recompute \ + "${profile_args[@]}" \ + "${backend_args[@]}" \ + --distributed-backend nccl & +model_pid=$! +printf '%s\n' "${model_pid}" >"${output}/model.pid" +wait "${model_pid}" +status=$? +set -e +model_pid= + +iterations=$(grep -Ec 'iteration[[:space:]]+[0-9]+/[[:space:]]*[0-9]+' "${output}/controller.log" || true) +last_iteration=$(grep -E 'iteration[[:space:]]+[0-9]+/[[:space:]]*[0-9]+' "${output}/controller.log" | tail -1 || true) +skipped=$(grep -Ec 'number of skipped iterations:[[:space:]]+[1-9]' "${output}/controller.log" || true) +nan=$(grep -Ec 'number of nan iterations:[[:space:]]+[1-9]' "${output}/controller.log" || true) +profile_done=$(find "${profile_output}" -type f -name analyse.done 2>/dev/null | wc -l || true) +npu-smi info >"${output}/npu_after.log" || true +post_idle=$(grep -c 'No running processes found in NPU' "${output}/npu_after.log" || true) +if [[ "${status}" -eq 0 && ( "${skipped}" -ne 0 || "${nan}" -ne 0 ) ]]; then + status=92 +fi +printf 'exit_code=%s\niterations=%s\nlast_iteration=%s\nskipped_nonzero=%s\nnan_nonzero=%s\nprofile_done=%s\npost_idle=%s\ncompleted=%s\n' \ + "${status}" "${iterations}" "${last_iteration}" "${skipped}" "${nan}" \ + "${profile_done}" "${post_idle}" "$(date '+%F %T %z')" | tee "${output}/result.txt" +exit "${status}" diff --git a/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py b/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py index f17373b..04e4ccf 100644 --- a/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py +++ b/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py @@ -12,6 +12,9 @@ ) +_UDMA_COMPAT_REGISTRATION_BYTES = 2 * 1024 * 1024 + + def _reject_upstream_buffer_init(*args, **kwargs): del args, kwargs raise RuntimeError( @@ -57,6 +60,7 @@ def __init__(self, *args, token_buffer_count=1, **kwargs): self._packed_projection_signature = None self._reduce_dummy = None self._reduce_dummy_buffer = None + self._reduce_dummy_buffer_allocation = None self._tilexr_remote_prefetches = 0 self._plan_owner_token = object() self._dispatch_generation = 0 @@ -140,6 +144,7 @@ def _dump_native_plan_once(self, plan): ) def dispatch(self, *args, hidden_buffer=None, **kwargs): + self._optional_stage_barrier() async_finish = bool(kwargs.pop("async_finish", False)) zero_copy = bool(kwargs.pop("zero_copy", False)) result = super().dispatch( @@ -206,6 +211,7 @@ def _stage_route_weights(self, route_weights, *, hidden_buffer): return route_boundary def combine(self, *args, hidden_buffer=None, **kwargs): + self._optional_stage_barrier() zero_copy = bool(kwargs.pop("zero_copy", False)) if hidden_buffer is not None: self._validate_boundary(hidden_buffer) @@ -347,20 +353,39 @@ def _prefetch_weight_packed_batch2( ) return self._record_event() if async_finish else None + def _optional_stage_barrier(self): + if os.environ.get("TILEXR_MINDSPEED_STAGE_BARRIER", "0") != "1": + return + distributed = self._torch.distributed + if not distributed.is_available() or not distributed.is_initialized(): + raise RuntimeError( + "TileXR stage barrier requires an initialized process group" + ) + distributed.barrier() + def _ensure_reduce_dummy(self, full_fc1, reduce_fc1): - dummy_width = ( - 262160 - if os.environ.get("TILEXR_MINDSPEED_FORCE_DUMMY_UDMA", "0") == "1" - else 16 - ) + dummy_width = 16 full_shape = (self.E + self.B, dummy_width) reduce_shape = (self.R, self.B, dummy_width) if self._reduce_dummy is None: self._reduce_dummy = self._torch.zeros( full_shape, dtype=self._torch.float32, device=full_fc1.device ) - self._reduce_dummy_buffer = self._torch.zeros( - reduce_shape, dtype=self._torch.float32, device=reduce_fc1.device + reduce_elements = self.R * self.B * dummy_width + allocation_elements = max( + reduce_elements, + _UDMA_COMPAT_REGISTRATION_BYTES // 4, + ) + self._reduce_dummy_buffer_allocation = self._torch.zeros( + (allocation_elements,), + dtype=self._torch.float32, + device=reduce_fc1.device, + ) + self._reduce_dummy_buffer = self._reduce_dummy_buffer_allocation.narrow( + 0, 0, reduce_elements + ).reshape(reduce_shape) + self._reduce_dummy_buffer._tilexr_registration_backing = ( + self._reduce_dummy_buffer_allocation ) else: self._reduce_dummy.zero_() @@ -432,6 +457,7 @@ def destroy(self): self._packed_projections = None self._reduce_dummy = None self._reduce_dummy_buffer = None + self._reduce_dummy_buffer_allocation = None self.token_buffers = () self.route_buffers = () self._route_by_token_ptr = {} From 23002707ee5a5ca4cacd27025c95ce0fd6611111 Mon Sep 17 00:00:00 2001 From: chaowick Date: Fri, 14 Aug 2026 00:01:25 +0800 Subject: [PATCH 2/3] fix(moonep): wait for complete combine inputs --- docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md | 25 +++++++++++ .../kernels/tilexr_moonep_combine_v2_kernel.h | 45 +++++++++++-------- .../python/test_mindspeed_model_runner.py | 7 ++- .../unit/test_combine_v2_source_guard.cpp | 28 +++++++++++- tools/moonep/mindspeed/run_model.sh | 10 ++++- tools/moonep/mindspeed/run_model_node.sh | 14 +++++- 6 files changed, 104 insertions(+), 25 deletions(-) diff --git a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md index d6694d5..2d158c4 100644 --- a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md +++ b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md @@ -184,6 +184,31 @@ reset 等单一变量。一次同时修改 Kernel、Host、timeout 和路由， - 不要因某次补丁通过完整模型就跳过最小 reproducer；最小 reproducer 才能证明因果。 - 不要删除被推翻的假设记录。保留否定证据可以防止后续重复猜测。 +## Combine V2 共享 scratch 的 completion 约束 + +ring 调度可以把发送工作分给不同 AIV core，但不能据此把接收完成条件也按 core +分片。每个 core 的 reduction 都会读取包含所有 source 写入的共享 scratch，因此每个 +consumer core 必须在 reduction 前观察全部 source 和 lane 的 Done token；只等待本 core +负责发送的 source 会在远端写仍在进行时提前读取，表现为偶发少聚合，而不是稳定超时。 + +本地 Self copy 也必须进入同一 completion 协议。只有在 Self 的最后一笔 MTE3 写完成后 +才能发布本地 Done，然后才能发布 step grant。不能把 `source == rank` 直接视为 ready， +否则其他 core 仍可能早于本地 copy 完成开始 reduction。 + +不要用 launch-wide barrier 修复这个问题。Host 的 launch block 数可能大于运行时 active +block 数，inactive block 会提前返回，`SyncAll` 的参与者集合因而无法收敛。应使用现有 +magic/epoch/step 编码的 GM Done token 表达真实 producer-consumer 依赖。验证至少覆盖： + +- 非 2 次幂 rank 的轮询游标，不能用位与代替取模； +- route weights、Prefetch、额外 plan、registration 切换和 plan reuse； +- 所有 rank 的 Dispatch/Combine hidden 与 weight 逐元素比较； +- 完整前反向模型的有限 loss/gradient，而不能只看进程退出码。 + +2026-08-13 的根因 oracle 使用单机 8 rank、S=4096、K=8、H=7168、model-skew +路由。修复后 5 轮严格 oracle 全 rank exact，随后单机 8 卡和双机 16 卡完整模型均达到 +8/8 且 loss/gradient 有限。该结果只证明已测试的 Ascend950PR、B131 CANN 和对应拓扑； +其他硬件、rank 规模和 topology 仍需按相同测试阶梯验证。 + ## Dispatch V2 fused epoch 约束与验证边界 一次 paired `TileXRMoonEpDispatchV2` 应只有一个 magic、一次 AICore launch 和 diff --git a/src/moonep/combine_v2/kernels/tilexr_moonep_combine_v2_kernel.h b/src/moonep/combine_v2/kernels/tilexr_moonep_combine_v2_kernel.h index a48c747..a99ac78 100644 --- a/src/moonep/combine_v2/kernels/tilexr_moonep_combine_v2_kernel.h +++ b/src/moonep/combine_v2/kernels/tilexr_moonep_combine_v2_kernel.h @@ -400,6 +400,7 @@ class MoonEpCombineV2 { uint32_t targetRank, uint64_t remoteOffset, __gm__ uint64_t *localSource, uint32_t flag); __aicore__ inline void PublishLocalGrant(uint32_t step, uint32_t lane); + __aicore__ inline void PublishSelfDone(uint32_t step); __aicore__ inline void CopyIssueToSq(LocalTensor issue, MoonEpCombineV2LaneState &state, uint32_t count); __aicore__ inline bool SubmitPair(uint32_t peer, uint32_t step, @@ -1118,6 +1119,21 @@ __aicore__ inline void MoonEpCombineV2::PublishLocalGrant( TileXRMoonEp::kMoonEpCombineV2TokenStrideBytes); } +__aicore__ inline void MoonEpCombineV2::PublishSelfDone(uint32_t step) +{ + for (uint32_t lane = 0U; + lane < TileXRMoonEp::kMoonEpCombineV2LaneCount; ++lane) { + const uint64_t index = TileXRMoonEp::MoonEpCombineV2DoneIndex( + epoch_, rank_, lane); + __gm__ uint64_t *done = reinterpret_cast<__gm__ uint64_t *>( + doneBase_ + index * + TileXRMoonEp::kMoonEpCombineV2TokenStrideBytes); + *done = TileXRMoonEp::MoonEpCombineV2Token(magic_, step); + TileXR::UDMACleanCacheLines(reinterpret_cast<__gm__ uint8_t *>(done), + TileXRMoonEp::kMoonEpCombineV2TokenStrideBytes); + } +} + __aicore__ inline void MoonEpCombineV2::CopyIssueToSq( LocalTensor issue, MoonEpCombineV2LaneState &state, uint32_t count) @@ -1649,35 +1665,32 @@ __aicore__ inline bool MoonEpCombineV2::SendSelfStep( firstPass = false; } while (pausedThreadCount != 0U); } + PublishSelfDone(step); return SubmitSelfGrant(step); } __aicore__ inline bool MoonEpCombineV2::WaitInboundDone() { - bool ready[TileXRMoonEp::kMoonEpCombineV2MaxSourcesPerCore] + bool ready[TileXRMoonEp::kMoonEpCombineV2RankCount] [TileXRMoonEp::kMoonEpCombineV2LaneCount] = {}; - uint64_t observed[TileXRMoonEp::kMoonEpCombineV2MaxSourcesPerCore] + uint64_t observed[TileXRMoonEp::kMoonEpCombineV2RankCount] [TileXRMoonEp::kMoonEpCombineV2LaneCount] = {}; uint32_t remaining = 0U; for (uint32_t sourceIndex = 0U; - sourceIndex < sourcesPerCore_; ++sourceIndex) { - const uint32_t source = TileXRMoonEp::MoonEpCombineV2SourceForCore( - core_, sourceIndex, rankSize_); + sourceIndex < rankSize_; ++sourceIndex) { for (uint32_t lane = 0U; lane < TileXRMoonEp::kMoonEpCombineV2LaneCount; ++lane) { - ready[sourceIndex][lane] = source == rank_; - if (!ready[sourceIndex][lane]) { - ++remaining; - } + ready[sourceIndex][lane] = false; + ++remaining; } } - const uint32_t conditionCount = sourcesPerCore_ * + const uint32_t conditionCount = rankSize_ * TileXRMoonEp::kMoonEpCombineV2LaneCount; uint32_t cursor = 0U; while (remaining != 0U) { for (uint32_t offset = 0U; offset < conditionCount; ++offset) { const uint32_t condition = - (cursor + offset) & (conditionCount - 1U); + (cursor + offset) % conditionCount; const uint32_t sourceIndex = condition / TileXRMoonEp::kMoonEpCombineV2LaneCount; const uint32_t lane = condition & @@ -1685,9 +1698,7 @@ __aicore__ inline bool MoonEpCombineV2::WaitInboundDone() if (ready[sourceIndex][lane]) { continue; } - const uint32_t source = - TileXRMoonEp::MoonEpCombineV2SourceForCore( - core_, sourceIndex, rankSize_); + const uint32_t source = sourceIndex; const uint32_t step = TileXRMoonEp::MoonEpCombineV2ReceiveStep( rank_, source, rankSize_, kCombineV2ScheduleMode); @@ -1704,7 +1715,7 @@ __aicore__ inline bool MoonEpCombineV2::WaitInboundDone() --remaining; } } - cursor = (cursor + 1U) & (conditionCount - 1U); + cursor = (cursor + 1U) % conditionCount; if (kEnableSafetyChecks && TimedOut(operationStartCycles_)) { for (uint32_t condition = 0U; condition < conditionCount; ++condition) { @@ -1715,9 +1726,7 @@ __aicore__ inline bool MoonEpCombineV2::WaitInboundDone() if (ready[sourceIndex][lane]) { continue; } - const uint32_t source = - TileXRMoonEp::MoonEpCombineV2SourceForCore( - core_, sourceIndex, rankSize_); + const uint32_t source = sourceIndex; const uint32_t step = TileXRMoonEp::MoonEpCombineV2ReceiveStep( rank_, source, rankSize_, kCombineV2ScheduleMode); diff --git a/tests/moonep/python/test_mindspeed_model_runner.py b/tests/moonep/python/test_mindspeed_model_runner.py index 7b77d27..5cf7ab9 100644 --- a/tests/moonep/python/test_mindspeed_model_runner.py +++ b/tests/moonep/python/test_mindspeed_model_runner.py @@ -195,8 +195,13 @@ def test_scripts_do_not_invoke_file_transfer_tools_and_define_failure_cleanup() assert 'wait "${cleanup_pid}" || true' in controller assert "trap 'handle_signal" in controller assert "remaining=$((remaining - 1))" in controller - assert "No node reported completion of iteration 8/8" in controller + assert "A node reported a non-finite gradient norm" in controller + assert "No node reported iteration 8/8 with a finite language-model loss" in controller assert "status=91" not in node + assert "nonfinite_grad=$(grep -Eic" in node + assert "finite_final_loss=$(grep -Ec" in node + assert '"${node_count}" -eq 1' in node + assert "status=93" in node assert "runner.pid" in node assert "kill -- -\"${model_pid}\"" in node diff --git a/tests/moonep_combine_v2/unit/test_combine_v2_source_guard.cpp b/tests/moonep_combine_v2/unit/test_combine_v2_source_guard.cpp index b34c694..90c332f 100644 --- a/tests/moonep_combine_v2/unit/test_combine_v2_source_guard.cpp +++ b/tests/moonep_combine_v2/unit/test_combine_v2_source_guard.cpp @@ -125,6 +125,9 @@ int main() const std::string selfGrant = Section(kernelImpl, "__aicore__ inline bool MoonEpCombineV2::SubmitSelfGrant(", "__aicore__ inline bool MoonEpCombineV2::SendSelfStep("); + const std::string inboundDone = Section(kernelImpl, + "__aicore__ inline bool MoonEpCombineV2::WaitInboundDone()", + "__aicore__ inline void MoonEpCombineV2::InitReduceBuffers()"); const std::string localGrant = Section(kernelImpl, "__aicore__ inline void MoonEpCombineV2::PublishLocalGrant(", "__aicore__ inline void MoonEpCombineV2::CopyIssueToSq("); @@ -243,6 +246,11 @@ int main() "Combine V2 Self step does not consume compacted route batches"); ok &= Require(selfStep, "SubmitSelfGrant(step)", "Combine V2 Self step does not publish its step grant"); + ok &= Require(selfStep, "PublishSelfDone(step)", + "Combine V2 Self step does not publish local completion"); + ok &= RequireBefore(selfStep, "PublishSelfDone(step)", + "SubmitSelfGrant(step)", + "Combine V2 Self completion is not published before its grant"); ok &= Require(selfGrant, "TileXR::TILEXR_UDMA_SQE_FLAG_ORDERED_COMPLETION", "Combine V2 Self grant does not request ordered completion"); @@ -315,6 +323,22 @@ int main() ok &= RequireBefore(process, "WaitStepGrant(step)", "WaitInboundDone()", "Combine V2 final grant wait does not precede finalization"); + ok &= Require(inboundDone, + "bool ready[TileXRMoonEp::kMoonEpCombineV2RankCount]", + "Combine V2 Done wait does not cover every source rank"); + ok &= Require(inboundDone, "sourceIndex < rankSize_", + "Combine V2 Done wait stops at the per-core source partition"); + ok &= Require(inboundDone, "const uint32_t source = sourceIndex;", + "Combine V2 Done wait does not address every source directly"); + ok &= Require(inboundDone, + "const uint32_t conditionCount = rankSize_ *", + "Combine V2 Done condition count is not derived from all sources"); + ok &= Require(inboundDone, "% conditionCount", + "Combine V2 Done cursor assumes a power-of-two rank size"); + ok &= Reject(inboundDone, "source == rank_", + "Combine V2 Done wait treats unfinished Self copies as ready"); + ok &= Reject(inboundDone, "MoonEpCombineV2SourceForCore(", + "Combine V2 Done wait still observes only its core-owned sources"); ok &= Require(kernelImpl, "st_dev(", "Combine V2 implementation does not ring device doorbells"); ok &= Require(kernelImpl, "rank_, 0U, core_, rankSize_", @@ -338,8 +362,8 @@ int main() "Combine V2 failure convergence does not use active cores"); ok &= Require(kernelImpl, "encoded, slots_, rankSize_", "Combine V2 destination validation does not use runtime rank size"); - ok &= Require(kernelImpl, "kMoonEpCombineV2MaxSourcesPerCore", - "Combine V2 inbound Done polling is not rank generalized"); + ok &= Require(inboundDone, "kMoonEpCombineV2RankCount", + "Combine V2 inbound Done storage is not rank generalized"); ok &= Require(kernelImpl, "MOONEP_COMBINE_V2_METRIC_SELF_COPY", "Combine V2 detailed self-copy profiling is missing"); ok &= Require(kernelImpl, "MOONEP_COMBINE_V2_METRIC_REMOTE_WQE_BUILD", diff --git a/tools/moonep/mindspeed/run_model.sh b/tools/moonep/mindspeed/run_model.sh index 9f6c340..fae0170 100755 --- a/tools/moonep/mindspeed/run_model.sh +++ b/tools/moonep/mindspeed/run_model.sh @@ -363,9 +363,15 @@ while (( remaining > 0 )); do (( remaining > 0 )) && sleep 1 done +if [[ "${failed}" -eq 0 ]] && grep -Eiq \ + 'grad norm:[[:space:]]*(-?inf|nan)' "${controller_dir}"/node_*.log; then + echo "A node reported a non-finite gradient norm" >&2 + failed=1 +fi if [[ "${failed}" -eq 0 ]] && ! grep -Eq \ - 'iteration[[:space:]]+8/[[:space:]]*8' "${controller_dir}"/node_*.log; then - echo "No node reported completion of iteration 8/8" >&2 + 'iteration[[:space:]]+8/[[:space:]]*8.*lm loss:[[:space:]]*[0-9]' \ + "${controller_dir}"/node_*.log; then + echo "No node reported iteration 8/8 with a finite language-model loss" >&2 failed=1 fi if [[ "${failed}" -ne 0 ]]; then diff --git a/tools/moonep/mindspeed/run_model_node.sh b/tools/moonep/mindspeed/run_model_node.sh index 5a4f5bf..8bad85c 100755 --- a/tools/moonep/mindspeed/run_model_node.sh +++ b/tools/moonep/mindspeed/run_model_node.sh @@ -354,13 +354,23 @@ iterations=$(grep -Ec 'iteration[[:space:]]+[0-9]+/[[:space:]]*[0-9]+' "${output last_iteration=$(grep -E 'iteration[[:space:]]+[0-9]+/[[:space:]]*[0-9]+' "${output}/controller.log" | tail -1 || true) skipped=$(grep -Ec 'number of skipped iterations:[[:space:]]+[1-9]' "${output}/controller.log" || true) nan=$(grep -Ec 'number of nan iterations:[[:space:]]+[1-9]' "${output}/controller.log" || true) +nonfinite_grad=$(grep -Eic 'grad norm:[[:space:]]*(-?inf|nan)' \ + "${output}/controller.log" || true) +finite_final_loss=$(grep -Ec \ + 'iteration[[:space:]]+8/[[:space:]]*8.*lm loss:[[:space:]]*[0-9]' \ + "${output}/controller.log" || true) profile_done=$(find "${profile_output}" -type f -name analyse.done 2>/dev/null | wc -l || true) npu-smi info >"${output}/npu_after.log" || true post_idle=$(grep -c 'No running processes found in NPU' "${output}/npu_after.log" || true) if [[ "${status}" -eq 0 && ( "${skipped}" -ne 0 || "${nan}" -ne 0 ) ]]; then status=92 fi -printf 'exit_code=%s\niterations=%s\nlast_iteration=%s\nskipped_nonzero=%s\nnan_nonzero=%s\nprofile_done=%s\npost_idle=%s\ncompleted=%s\n' \ +if [[ "${status}" -eq 0 && "${node_count}" -eq 1 && \ + ( "${nonfinite_grad}" -ne 0 || "${finite_final_loss}" -eq 0 ) ]]; then + status=93 +fi +printf 'exit_code=%s\niterations=%s\nlast_iteration=%s\nskipped_nonzero=%s\nnan_nonzero=%s\nnonfinite_grad=%s\nfinite_final_loss=%s\nprofile_done=%s\npost_idle=%s\ncompleted=%s\n' \ "${status}" "${iterations}" "${last_iteration}" "${skipped}" "${nan}" \ - "${profile_done}" "${post_idle}" "$(date '+%F %T %z')" | tee "${output}/result.txt" + "${nonfinite_grad}" "${finite_final_loss}" "${profile_done}" "${post_idle}" \ + "$(date '+%F %T %z')" | tee "${output}/result.txt" exit "${status}" From 1396448988bb416f96cd193cca0aa4c8663c60cb Mon Sep 17 00:00:00 2001 From: chaowick Date: Fri, 14 Aug 2026 00:01:25 +0800 Subject: [PATCH 3/3] perf(moonep): disable dispatch diagnostics by default --- docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md | 11 ++++++++++- src/moonep/dispatch/CMakeLists.txt | 2 +- tests/moonep/python/test_mindspeed_model_runner.py | 4 ++++ tests/moonep/unit/test_tilexr_moonep_sources.cpp | 2 ++ tools/moonep/mindspeed/run_model_node.sh | 1 + 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md index 2d158c4..958d00c 100644 --- a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md +++ b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md @@ -169,7 +169,16 @@ reset 等单一变量。一次同时修改 Kernel、Host、timeout 和路由， - 同时覆盖 1-BB 和多 BB WQE； - balanced、model-skew、sparse 和 unique routing； - `S=4096`、`K=8`、`H=7168`、EP8 的生产规模； -- 正确性运行开启失败时 DFX，性能运行关闭 trace、dump、DFX 和 profiler。 +- 正确性运行可按需开启失败时 DFX。纯吞吐性能运行关闭 trace、dump、DFX 和 profiler； + 需要算子耗时统计时，保持 trace、dump、DFX、调试同步和 stage barrier 关闭，只开启 + 框架 NPU profiler，并单独标注为 profiling-on 数据。 + +性能复测不能只检查运行时环境变量。Dispatch DFX 和部分 profiling 是编译期 CMake +选项；即使 provenance 显示 trace/dump/profile 都关闭，已嵌入 `.so` 的 AICore binary +仍可能包含 DFX 路径。性能运行前必须同时保存并核对 CMake cache、实际加载库哈希和 +运行时环境，普通 Release 构建应默认关闭 DFX，需要诊断时再显式开启并重新构建。 +框架 NPU profiler 与 Kernel 编译期 DFX/profiling 必须分开记录：前者可用于算子计时， +后者会改变被测 Kernel 路径，不能在默认性能构建中开启。 ## 避免重复踩坑 diff --git a/src/moonep/dispatch/CMakeLists.txt b/src/moonep/dispatch/CMakeLists.txt index 8dfcfcd..80fb540 100644 --- a/src/moonep/dispatch/CMakeLists.txt +++ b/src/moonep/dispatch/CMakeLists.txt @@ -9,7 +9,7 @@ include(${CMAKE_SOURCE_DIR}/src/moonep/cmake/MoonEpKernel.cmake) option(TILEXR_MOONEP_DISPATCH_ENABLE_PROFILING "Enable per-AIV profiling in the URMA MoonEP Dispatch kernel" OFF) option(TILEXR_MOONEP_DISPATCH_ENABLE_DFX - "Enable detailed per-AIV DFX in the URMA MoonEP Dispatch kernel" ON) + "Enable detailed per-AIV DFX in the URMA MoonEP Dispatch kernel" OFF) if(TILEXR_MOONEP_DISPATCH_ENABLE_PROFILING AND NOT TILEXR_MOONEP_DISPATCH_ENABLE_DFX) message(FATAL_ERROR diff --git a/tests/moonep/python/test_mindspeed_model_runner.py b/tests/moonep/python/test_mindspeed_model_runner.py index 5cf7ab9..ac9fa98 100644 --- a/tests/moonep/python/test_mindspeed_model_runner.py +++ b/tests/moonep/python/test_mindspeed_model_runner.py @@ -92,6 +92,10 @@ def test_runner_scripts_expose_the_supported_interface_and_validated_shape() -> assert "TILEXR_MOONEP_DISPATCH_GROUP_WIDTH=16" in node assert "TILEXR_MOONEP_COMBINE_VERSION=2" in node assert "unset TILEXR_MOONEP_DISPATCH_TRANSPORT" in node + assert ( + "HCCL_NPU_SOCKET_PORT_RANGE=${MODEL_RUNNER_HCCL_NPU_SOCKET_PORT_RANGE:-47000-47100}" + in node + ) def test_first_run_prompts_and_subsequent_dry_run_reuses_cached_answers(tmp_path: Path) -> None: diff --git a/tests/moonep/unit/test_tilexr_moonep_sources.cpp b/tests/moonep/unit/test_tilexr_moonep_sources.cpp index 99d4117..5e6f29d 100644 --- a/tests/moonep/unit/test_tilexr_moonep_sources.cpp +++ b/tests/moonep/unit/test_tilexr_moonep_sources.cpp @@ -163,6 +163,8 @@ int main() Contains("dispatch CMake", dispatchCmake, "--cce-auto-sync"); Contains("dispatch CMake", dispatchCmake, "-DCATLASS_ARCH=3510"); Contains("dispatch CMake", dispatchCmake, "TILEXR_MOONEP_DISPATCH_ENABLE_DFX"); + Contains("dispatch CMake", dispatchCmake, + "Enable detailed per-AIV DFX in the URMA MoonEP Dispatch kernel\" OFF)"); Contains("dispatch CMake", dispatchCmake, "cxx_std_14"); Contains("dispatch CMake", dispatchCmake, "BUILD_WITH_INSTALL_RPATH TRUE"); Contains("dispatch CMake", dispatchCmake, "INSTALL_RPATH \"$ORIGIN\""); diff --git a/tools/moonep/mindspeed/run_model_node.sh b/tools/moonep/mindspeed/run_model_node.sh index 8bad85c..f7784ee 100755 --- a/tools/moonep/mindspeed/run_model_node.sh +++ b/tools/moonep/mindspeed/run_model_node.sh @@ -193,6 +193,7 @@ fi [[ -n "${interface}" ]] || { echo "Unable to determine communication interface" >&2; exit 1; } export HCCL_HOST_SOCKET_PORT_RANGE=auto +export HCCL_NPU_SOCKET_PORT_RANGE=${MODEL_RUNNER_HCCL_NPU_SOCKET_PORT_RANGE:-47000-47100} export HCCL_SOCKET_IFNAME=${interface} export GLOO_SOCKET_IFNAME=${interface} export HCCL_BUFFSIZE=200