From 0e70eba69f8c0433a703e8f623eac215badc4e46 Mon Sep 17 00:00:00 2001
From: chaowick <winnochan@163.com>
Date: Fri, 14 Aug 2026 14:57:35 +0800
Subject: [PATCH] feat(moonep): support 128-rank MindSpeed runs

---
 docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md |  16 +
 .../moonep_torch/tilexr_moonep/runtime.py     |   9 +-
 .../moonep_torch/tilexr_moonep/torch_api.py   | 124 ++++-
 tests/moonep/python/fakes.py                  |  23 +
 tests/moonep/python/test_ffi_unittest.py      |  25 +
 .../python/test_mindspeed_model_runner.py     | 110 +++-
 .../python/test_mindspeed_rank_table.py       | 231 ++++++++
 tools/moonep/mindspeed/README.md              |  61 ++-
 tools/moonep/mindspeed/generate_rank_table.py | 499 ++++++++++++++++++
 .../mindspeed/mindspeed_stage_barrier.py      |  47 ++
 tools/moonep/mindspeed/preflight_adapter.sh   |   4 +
 tools/moonep/mindspeed/probe_idle.sh          |  54 ++
 tools/moonep/mindspeed/run_model.sh           |  49 +-
 tools/moonep/mindspeed/run_model_node.sh      | 102 +++-
 .../mindspeed/tilexr_mindspeed_adapter.py     |  84 ++-
 15 files changed, 1387 insertions(+), 51 deletions(-)
 create mode 100644 tests/moonep/python/test_mindspeed_rank_table.py
 create mode 100644 tools/moonep/mindspeed/generate_rank_table.py
 create mode 100644 tools/moonep/mindspeed/mindspeed_stage_barrier.py
 create mode 100644 tools/moonep/mindspeed/probe_idle.sh

diff --git a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md
index 958d00c..0089c30 100644
--- a/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md
+++ b/docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md
@@ -189,9 +189,25 @@ reset 等单一变量。一次同时修改 Kernel、Host、timeout 和路由，
 - 不要用 toy shape 证明生产规模的 UB/SQ 容量安全。
 - 不要依赖已消费 SQE 的内容推导 CQ completion。
 - 不要在调试运行和性能运行之间复用未审计的环境变量。
+- 多机 HCCL rank table 与 TileXR UDMA RootInfo 是两种不同配置：前者描述全局
+  rank/topology，后者描述每台主机的本地 UDMA EID/port。不能用一个文件替代另一个；
+  应分别保存路径、哈希和 parser/init 证据。
+- 多机 launcher 全部退出 0 只证明进程完成，不证明模型数值正确。至少同时门禁完整迭代数、
+  最终有限 loss、全程有限 gradient norm、skip/NaN 计数、profiler artifact 数和退出后 idle。
+- 每次实机 A/B 必须同时保存源码 commit/diff 与实际加载 `.so` 路径和哈希。源码同步成功
+  不代表远端安装树已重编译；如果重新构建后行为变化而代码修复尚未被单变量证明，应明确
+  记录为 binary-provenance 问题，不能把诊断插桩本身误报成根因修复。
 - 不要把小 Tensor view 的逻辑字节数等同于 HCCP MR 范围；扩大 backing storage 时保持逻辑 shape 不变，否则会污染算子工作量和性能数据。
 - 不要因某次补丁通过完整模型就跳过最小 reproducer；最小 reproducer 才能证明因果。
 - 不要删除被推翻的假设记录。保留否定证据可以防止后续重复猜测。
+- 多阶段共享普通 UDMA 注册时，不能在热路径中按 stage 反复切换单一 active MR；不同
+  rank 的 stage 到达顺序可能不同，使无标签注册 collective 串轮。可预测的工作区应合并
+  到一个持久注册 arena，算子继续使用各自逻辑子区间；子区间激活必须命中已有 MR，不能
+  再发起注册 collective。扩大 arena 后仍要分别验证初始化内存峰值和至少两轮模型数值。
+- `npu-smi info` 在 950 上可能显示每卡无进程，但 HCCL Test 或其他加速器工作负载仍在
+  使用设备。多机 idle gate 必须同时检查设备节点 owner 和已知加速器进程；发现外部作业
+  只判 busy，不终止。连续模型启动后的 `EI0007/halSqCqAllocate` 表示驱动 stream/SQ-CQ
+  资源尚未回收，应等待并用最小 stream 探针确认恢复，不能改算子代码规避。
 
 ## Combine V2 共享 scratch 的 completion 约束
 
diff --git a/integrations/moonep_torch/tilexr_moonep/runtime.py b/integrations/moonep_torch/tilexr_moonep/runtime.py
index f6430d6..22f34c2 100644
--- a/integrations/moonep_torch/tilexr_moonep/runtime.py
+++ b/integrations/moonep_torch/tilexr_moonep/runtime.py
@@ -592,7 +592,14 @@ def _activate_udma_region(
             return None
         if pointer <= 0 or size <= 0:
             raise ValueError("UDMA registration requires a valid pointer and positive size")
-        if self._active_udma_pointer == pointer and self._active_udma_bytes == size:
+        active_end = self._active_udma_pointer + self._active_udma_bytes
+        requested_end = pointer + size
+        if (
+            self._active_udma_pointer > 0
+            and pointer >= self._active_udma_pointer
+            and requested_end >= pointer
+            and requested_end <= active_end
+        ):
             self._active_udma_owner = owner
             return self._active_udma_handle
         handle = ctypes.c_uint32()
diff --git a/integrations/moonep_torch/tilexr_moonep/torch_api.py b/integrations/moonep_torch/tilexr_moonep/torch_api.py
index afe4946..c2b84b2 100644
--- a/integrations/moonep_torch/tilexr_moonep/torch_api.py
+++ b/integrations/moonep_torch/tilexr_moonep/torch_api.py
@@ -124,6 +124,8 @@ def from_local_weights(
         down,
         *,
         slot_fill_value: float = 0.0,
+        registration_backing=None,
+        registration_backing_factory=None,
         torch_module=None,
     ) -> "ProjectionBuffers":
         torch_module = torch_module or _torch()
@@ -165,20 +167,47 @@ def from_local_weights(
             (cursor_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1) //
             _UDMA_REGISTRATION_ALIGNMENT * _UDMA_REGISTRATION_ALIGNMENT
         )
-        allocation_elements = (
-            registered_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1 + element_bytes - 1
-        ) // element_bytes
-        allocation = torch_module.empty(
-            (allocation_elements,), dtype=gate.dtype,
-            device=f"npu:{context.device_index}"
-        )
-        allocation_ptr = int(allocation.data_ptr())
-        offset_bytes = (-allocation_ptr) % _UDMA_REGISTRATION_ALIGNMENT
-        if offset_bytes % element_bytes:
-            raise RuntimeError("projection registration offset is not element-aligned")
-        backing = allocation.narrow(
-            0, offset_bytes // element_bytes, registered_bytes // element_bytes
-        )
+        if registration_backing is not None and registration_backing_factory is not None:
+            raise ValueError(
+                "registration_backing and registration_backing_factory are mutually exclusive"
+            )
+        if registration_backing_factory is not None:
+            registration_backing = registration_backing_factory(
+                registered_bytes, gate.dtype
+            )
+        if registration_backing is None:
+            allocation_elements = (
+                registered_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1 + element_bytes - 1
+            ) // element_bytes
+            allocation = torch_module.empty(
+                (allocation_elements,), dtype=gate.dtype,
+                device=f"npu:{context.device_index}"
+            )
+            allocation_ptr = int(allocation.data_ptr())
+            offset_bytes = (-allocation_ptr) % _UDMA_REGISTRATION_ALIGNMENT
+            if offset_bytes % element_bytes:
+                raise RuntimeError("projection registration offset is not element-aligned")
+            backing = allocation.narrow(
+                0, offset_bytes // element_bytes, registered_bytes // element_bytes
+            )
+        else:
+            _validate_tensor(
+                torch_module,
+                registration_backing,
+                "registration_backing",
+                dtype=gate.dtype,
+                device_index=context.device_index,
+                allow_storage_offset=True,
+            )
+            if int(registration_backing.data_ptr()) % _UDMA_REGISTRATION_ALIGNMENT:
+                raise ValueError("registration_backing must be 2-MiB aligned")
+            if int(registration_backing.numel()) * element_bytes < registered_bytes:
+                raise ValueError(
+                    "registration_backing is smaller than the projection layout"
+                )
+            backing = registration_backing.narrow(
+                0, 0, registered_bytes // element_bytes
+            )
         if (int(backing.data_ptr()) % _UDMA_REGISTRATION_ALIGNMENT or
                 int(backing.numel()) * element_bytes != registered_bytes):
             raise RuntimeError("projection backing is not 2-MiB registration-aligned")
@@ -288,6 +317,7 @@ class TileXRMoonEPContext:
     _dispatch_workspace_ptr: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_bytes: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_alignment: int = field(init=False, default=0, repr=False)
+    _dispatch_registration_bytes: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_handle: int | None = field(init=False, default=None, repr=False)
     _buffer_owner: Any = field(init=False, default=None, repr=False)
     _bound_stream_ptr: int | None = field(init=False, default=None, repr=False)
@@ -410,6 +440,7 @@ def close(self) -> None:
         self._dispatch_workspace_handle = None
         self._dispatch_workspace_ptr = 0
         self._dispatch_workspace_bytes = 0
+        self._dispatch_registration_bytes = 0
         self._dispatch_workspace_owner = None
         self.runtime.close()
         self._closed = True
@@ -454,8 +485,9 @@ def ensure_dispatch_workspace(self, torch_module) -> None:
                 "native Dispatch returned invalid workspace contract "
                 f"bytes={workspace_bytes} alignment={alignment}"
             )
+        registration_bytes = workspace_bytes
         raw = torch_module.empty(
-            (workspace_bytes + alignment - 1,),
+            (registration_bytes + alignment - 1,),
             dtype=torch_module.uint8,
             device=f"npu:{self.device_index}",
         )
@@ -464,19 +496,77 @@ def ensure_dispatch_workspace(self, torch_module) -> None:
         aligned_ptr = ((raw_ptr + alignment - 1) // alignment) * alignment
         if aligned_ptr + workspace_bytes > raw_ptr + int(raw.numel()):
             raise RuntimeError("aligned Dispatch workspace exceeds its raw allocation")
-        handle = self.runtime.register_dispatch_workspace(aligned_ptr, workspace_bytes)
+        handle = self.runtime.register_dispatch_workspace(aligned_ptr, registration_bytes)
         self._dispatch_workspace_owner = raw
         self._dispatch_workspace_ptr = aligned_ptr
         self._dispatch_workspace_bytes = workspace_bytes
         self._dispatch_workspace_alignment = alignment
+        self._dispatch_registration_bytes = registration_bytes
         self._dispatch_workspace_handle = handle
 
     def activate_dispatch_workspace(self) -> None:
         if self._dispatch_workspace_owner is None:
             raise RuntimeError("Dispatch workspace is not initialized")
         self._dispatch_workspace_handle = self.runtime.register_dispatch_workspace(
-            self._dispatch_workspace_ptr, self._dispatch_workspace_bytes
+            self._dispatch_workspace_ptr, self._dispatch_registration_bytes
+        )
+
+    def promote_projection_arena(self, torch_module, dtype, required_bytes: int):
+        if required_bytes <= 0 or required_bytes % self._dispatch_workspace_alignment:
+            raise ValueError(
+                "projection arena size must be a positive multiple of the UDMA alignment"
+            )
+        reserve_text = os.environ.get("TILEXR_MOONEP_UDMA_ARENA_RESERVE_BYTES", "0")
+        try:
+            reserve_bytes = int(reserve_text, 0)
+        except ValueError as exc:
+            raise ValueError(
+                "TILEXR_MOONEP_UDMA_ARENA_RESERVE_BYTES must be an integer"
+            ) from exc
+        if reserve_bytes <= 0:
+            return None
+        if required_bytes > reserve_bytes:
+            raise RuntimeError(
+                "projection backing exceeds the configured UDMA arena reserve: "
+                f"required={required_bytes} reserve={reserve_bytes}"
+            )
+        if self._dispatch_registration_bytes == self._dispatch_workspace_bytes:
+            registration_bytes = self._dispatch_workspace_bytes + required_bytes
+            alignment = self._dispatch_workspace_alignment
+            raw = torch_module.empty(
+                (registration_bytes + alignment - 1,),
+                dtype=torch_module.uint8,
+                device=f"npu:{self.device_index}",
+            )
+            raw_ptr = int(raw.data_ptr())
+            aligned_ptr = ((raw_ptr + alignment - 1) // alignment) * alignment
+            old_offset = self._dispatch_workspace_ptr - int(
+                self._dispatch_workspace_owner.data_ptr()
+            )
+            new_offset = aligned_ptr - raw_ptr
+            raw.narrow(0, new_offset, self._dispatch_workspace_bytes).copy_(
+                self._dispatch_workspace_owner.narrow(
+                    0, old_offset, self._dispatch_workspace_bytes
+                )
+            )
+            torch_module.npu.synchronize(device=self.device_index)
+            handle = self.runtime.register_dispatch_workspace(
+                aligned_ptr, registration_bytes
+            )
+            self._dispatch_workspace_owner = raw
+            self._dispatch_workspace_ptr = aligned_ptr
+            self._dispatch_registration_bytes = registration_bytes
+            self._dispatch_workspace_handle = handle
+        available = self._dispatch_registration_bytes - self._dispatch_workspace_bytes
+        if required_bytes > available:
+            raise RuntimeError("projection arena was already promoted with a smaller layout")
+        raw_offset = self._dispatch_workspace_ptr - int(
+            self._dispatch_workspace_owner.data_ptr()
+        )
+        byte_view = self._dispatch_workspace_owner.narrow(
+            0, raw_offset + self._dispatch_workspace_bytes, required_bytes
         )
+        return byte_view.view(dtype)
 
     @property
     def dispatch_workspace(self) -> tuple[int, int]:
diff --git a/tests/moonep/python/fakes.py b/tests/moonep/python/fakes.py
index 01314a3..a90052d 100644
--- a/tests/moonep/python/fakes.py
+++ b/tests/moonep/python/fakes.py
@@ -87,6 +87,29 @@ def reshape(self, *shape):
         result.zero_calls = self.zero_calls
         return result
 
+    def view(self, *args):
+        if len(args) == 1 and isinstance(args[0], str):
+            dtype = args[0]
+            source_bytes = self.numel() * self.element_size()
+            target_bytes = {
+                "uint8": 1,
+                "float16": 2,
+                "bfloat16": 2,
+                "int32": 4,
+                "float32": 4,
+                "int64": 8,
+            }[dtype]
+            if source_bytes % target_bytes:
+                raise ValueError("FakeTensor.view dtype has incompatible byte size")
+            result = FakeTensor(
+                (source_bytes // target_bytes,), dtype, self.device,
+                storage_offset=self._storage_offset,
+            )
+            result._ptr = self._ptr
+            result._base = getattr(self, "_base", self)
+            return result
+        return self.reshape(*args)
+
     def narrow(self, dim, start, length):
         dim = int(dim)
         start = int(start)
diff --git a/tests/moonep/python/test_ffi_unittest.py b/tests/moonep/python/test_ffi_unittest.py
index bdc6a3e..95522f0 100644
--- a/tests/moonep/python/test_ffi_unittest.py
+++ b/tests/moonep/python/test_ffi_unittest.py
@@ -425,6 +425,31 @@ def tensor(shape, dtype):
 
 
 class FfiAbiTests(unittest.TestCase):
+    def test_udma_arena_subregion_reuses_active_registration(self):
+        loader = FakeCDLLLoader()
+        runtime = TileXRMoonEPRuntime(
+            rank=0,
+            world_size=2,
+            library_paths={
+                "comm": "libtile-comm.so",
+                "planner": "libtilexr-moonep-planner.so",
+                "combine_v2": "libtilexr-moonep-combine-v2.so.2",
+                "moonep": "libtilexr-moonep.so.1",
+            },
+            cdll_loader=loader,
+        )
+
+        handle = runtime._activate_udma_region(
+            0x200000, 2 * 1024 * 1024, "dispatch", "arena"
+        )
+        reused = runtime._activate_udma_region(
+            0x300000, 1024 * 1024, "projection", "subregion"
+        )
+
+        self.assertEqual(reused, handle)
+        self.assertEqual(loader.register_calls, [(0x200000, 2 * 1024 * 1024)])
+        runtime.close()
+
     def test_reduce_grad_registration_uses_storage_without_expanding_source(self):
         backing = FakeTensor((2 * 1024 * 1024 // 4,), "float32")
         backing._ptr = 0x200000
diff --git a/tests/moonep/python/test_mindspeed_model_runner.py b/tests/moonep/python/test_mindspeed_model_runner.py
index ac9fa98..dc7ed37 100644
--- a/tests/moonep/python/test_mindspeed_model_runner.py
+++ b/tests/moonep/python/test_mindspeed_model_runner.py
@@ -11,6 +11,10 @@
 ROOT = Path(__file__).resolve().parents[3]
 CONTROLLER = ROOT / "tools" / "moonep" / "mindspeed" / "run_model.sh"
 NODE_RUNNER = ROOT / "tools" / "moonep" / "mindspeed" / "run_model_node.sh"
+STAGE_BARRIER = (
+    ROOT / "tools" / "moonep" / "mindspeed" / "mindspeed_stage_barrier.py"
+)
+IDLE_PROBE = ROOT / "tools" / "moonep" / "mindspeed" / "probe_idle.sh"
 GIT_BASH = Path(r"C:\Program Files\Git\bin\bash.exe")
 
 
@@ -67,6 +71,9 @@ def test_runner_scripts_expose_the_supported_interface_and_validated_shape() ->
         "--config",
         "--dry-run",
         "--idle-wait",
+        "--hccl-inter-hccs-disable",
+        "--rank-table-file",
+        "--udma-rootinfo-path",
     ):
         assert option in controller
     for option in (
@@ -82,12 +89,19 @@ def test_runner_scripts_expose_the_supported_interface_and_validated_shape() ->
         "--num-layers 4",
         "--seq-length 4096",
         "--hidden-size 7168",
-        "--num-experts 32",
         "--moe-router-topk 8",
         "--moonep-token-padding 1",
         "--train-iters 8",
     ):
         assert argument in node
+    assert "ep_size=${world_size}" in node
+    assert "base_expert_count=32" in node
+    assert "expert_count=$((((base_expert_count + ep_size - 1) / ep_size) * ep_size))" in node
+    assert '--num-experts "${expert_count}"' in node
+    assert "ep_size=${devices_per_node}" not in node
+    assert '"${node_count}" -gt 1' in node
+    assert "ip -o -4 addr show dev data0.3001" in node
+    assert "interface=data0.3001" in node
     assert "TILEXR_MOONEP_DISPATCH_PEER_MODE=group" in node
     assert "TILEXR_MOONEP_DISPATCH_GROUP_WIDTH=16" in node
     assert "TILEXR_MOONEP_COMBINE_VERSION=2" in node
@@ -96,6 +110,98 @@ def test_runner_scripts_expose_the_supported_interface_and_validated_shape() ->
         "HCCL_NPU_SOCKET_PORT_RANGE=${MODEL_RUNNER_HCCL_NPU_SOCKET_PORT_RANGE:-47000-47100}"
         in node
     )
+    assert 'export HCCL_INTER_HCCS_DISABLE=${hccl_inter_hccs_disable}' in node
+    assert 'export RANK_TABLE_FILE=${rank_table_file}' in node
+    assert 'rank_table_sha256=$(sha256sum "${rank_table_file}"' in node
+    assert 'export TILEXR_UDMA_ROOTINFO_PATH=${udma_rootinfo_path}' in node
+    assert 'udma_rootinfo_sha256=$(sha256sum "${udma_rootinfo_path}"' in node
+    barrier = STAGE_BARRIER.read_text(encoding="utf-8")
+    assert "class MindSpeedBarrierMoonEPBuffer(upstream_moonep.Buffer)" in barrier
+    assert barrier.count("optional_stage_barrier(torch)") == 2
+    assert '"MOONEP_MINDSPEED_STAGE_BARRIER"' in barrier
+    assert "distributed.barrier()" in barrier
+    assert "create_native_barrier_backend.__mindspeed_capabilities__" in barrier
+    assert "MOONEP_MINDSPEED_STAGE_BARRIER=${stage_barrier}" in node
+    assert "mindspeed_stage_barrier:create_native_barrier_backend" in node
+    assert 'idle_probe=${tilexr_home}/tools/moonep/mindspeed/probe_idle.sh' in node
+    assert 'bash "${idle_probe}" --devices "${devices_per_node}"' in node
+    idle_probe = IDLE_PROBE.read_text(encoding="utf-8")
+    assert "timeout 15s npu-smi info" in idle_probe
+    assert "checking live ownership" in idle_probe
+    assert "timeout 5s npu-smi info -l" in idle_probe
+    assert "fuser /dev/davinci[0-9]*" in idle_probe
+    for process_name in (
+        "pretrain_gpt.py",
+        "torch.distributed.launch",
+        "hccl_test/bin/",
+        "alltoallv_test",
+        "tilexr_udma_dem",
+    ):
+        assert process_name in idle_probe
+    assert "accelerator_processes" in idle_probe
+    assert "remote_idle_probe" in controller
+    assert '"${profile_done}" -ne "${devices_per_node}"' in node
+    assert 'set +u\n# Vendor and conda environment scripts' in node
+    assert 'source "${native_env}"\nset -u' in node
+
+
+def test_stage_barrier_is_supported_for_native_and_tilexr_dry_runs(tmp_path: Path) -> None:
+    config = tmp_path / "runner.env"
+    configured = run_controller(
+        "--mode", "multi", "--backend", "native", "--stage-barrier",
+        "--config", str(config), "--dry-run", stdin=answers()
+    )
+    assert configured.returncode == 0, configured.stderr
+    assert configured.stdout.count("--stage-barrier") == 2
+
+    tilexr = run_controller(
+        "--mode", "multi", "--backend", "tilexr", "--stage-barrier",
+        "--config", str(config), "--dry-run"
+    )
+    assert tilexr.returncode == 0, tilexr.stderr
+    assert tilexr.stdout.count("--stage-barrier") == 2
+
+
+def test_inter_hccs_disable_is_validated_and_forwarded(tmp_path: Path) -> None:
+    config = tmp_path / "runner.env"
+    configured = run_controller(
+        "--mode", "multi", "--backend", "native",
+        "--hccl-inter-hccs-disable", "true",
+        "--config", str(config), "--dry-run", stdin=answers()
+    )
+    assert configured.returncode == 0, configured.stderr
+    assert configured.stdout.count("--hccl-inter-hccs-disable true") == 2
+
+    invalid = run_controller(
+        "--mode", "multi", "--backend", "native",
+        "--hccl-inter-hccs-disable", "1",
+        "--config", str(config), "--dry-run"
+    )
+    assert invalid.returncode != 0
+
+
+def test_rank_table_path_is_forwarded_to_every_node(tmp_path: Path) -> None:
+    config = tmp_path / "runner.env"
+    configured = run_controller(
+        "--mode", "multi", "--backend", "native",
+        "--rank-table-file", "/srv/rank table.json",
+        "--config", str(config), "--dry-run", stdin=answers()
+    )
+    assert configured.returncode == 0, configured.stderr
+    assert configured.stdout.count("--rank-table-file /srv/rank\\ table.json") == 2
+
+
+def test_udma_rootinfo_path_is_forwarded_to_every_node(tmp_path: Path) -> None:
+    config = tmp_path / "runner.env"
+    configured = run_controller(
+        "--mode", "multi", "--backend", "tilexr",
+        "--udma-rootinfo-path", "/etc/hccl rootinfo.json.bak",
+        "--config", str(config), "--dry-run", stdin=answers()
+    )
+    assert configured.returncode == 0, configured.stderr
+    assert configured.stdout.count(
+        "--udma-rootinfo-path /etc/hccl\\ rootinfo.json.bak"
+    ) == 2
 
 
 def test_first_run_prompts_and_subsequent_dry_run_reuses_cached_answers(tmp_path: Path) -> None:
@@ -195,6 +301,8 @@ def test_scripts_do_not_invoke_file_transfer_tools_and_define_failure_cleanup()
         assert "rsync " not in script
     assert "cleanup_remote_runs" in controller
     assert "wait_for_stable_idle" in controller
+    assert 'probe_pids+=("$!")' in controller
+    assert 'wait "${probe_pids[${index}]}"' in controller
     assert 'consecutive=$((consecutive + 1))' in controller
     assert 'wait "${cleanup_pid}" || true' in controller
     assert "trap 'handle_signal" in controller
diff --git a/tests/moonep/python/test_mindspeed_rank_table.py b/tests/moonep/python/test_mindspeed_rank_table.py
new file mode 100644
index 0000000..00b9663
--- /dev/null
+++ b/tests/moonep/python/test_mindspeed_rank_table.py
@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[3]
+GENERATOR_PATH = (
+    ROOT / "tools" / "moonep" / "mindspeed" / "generate_rank_table.py"
+)
+
+
+def load_generator():
+    spec = importlib.util.spec_from_file_location("generate_rank_table", GENERATOR_PATH)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def root_info(*primary_eids: str) -> dict[str, object]:
+    rank_list = []
+    for device_id, primary_eid in enumerate(primary_eids):
+        rank_list.append(
+            {
+                "device_id": device_id,
+                "local_id": device_id,
+                "level_list": [
+                    {
+                        "net_layer": 0,
+                        "rank_addr_list": [
+                            {
+                                "addr_type": "EID",
+                                "addr": "f" * 32,
+                                "ports": ["0/0"],
+                            }
+                        ],
+                    },
+                    {
+                        "net_layer": 1,
+                        "rank_addr_list": [
+                            {
+                                "addr_type": "EID",
+                                "addr": "00000000007f02000010000000000001",
+                                "ports": ["1/0", "1/1"],
+                            },
+                            {
+                                "addr_type": "EID",
+                                "addr": primary_eid,
+                                "ports": ["0/1", "0/2"],
+                            },
+                        ],
+                    },
+                ],
+            }
+        )
+    return {"version": "2.0", "rank_count": len(rank_list), "rank_list": rank_list}
+
+
+def interface_info(address: str) -> list[dict[str, object]]:
+    return [
+        {
+            "ifname": "data0.3001",
+            "addr_info": [
+                {
+                    "family": "inet",
+                    "local": address,
+                    "scope": "global",
+                }
+            ],
+        }
+    ]
+
+
+def test_builds_hccl_v12_table_in_host_and_device_order() -> None:
+    generator = load_generator()
+    hosts = ["node-a", "node-b"]
+    snapshots = {
+        "node-a": (
+            interface_info("192.0.2.10"),
+            root_info(
+                "20010db8000000000000000000000001",
+                "20010db8000000000000000000000002",
+            ),
+        ),
+        "node-b": (
+            interface_info("198.51.100.20"),
+            root_info(
+                "20010db8000000000000000000000101",
+                "20010db8000000000000000000000102",
+            ),
+        ),
+    }
+
+    table = generator.build_rank_table(
+        hosts=hosts,
+        super_pod_ids=[5, 15],
+        servers_per_super_pod=1,
+        devices_per_server=2,
+        collect=lambda host: snapshots[host],
+    )
+
+    assert table["version"] == "1.2"
+    assert table["server_count"] == "2"
+    assert table["status"] == "completed"
+    assert [server["server_id"] for server in table["server_list"]] == [
+        "192.0.2.10",
+        "198.51.100.20",
+    ]
+    devices = [
+        device
+        for server in table["server_list"]
+        for device in server["device"]
+    ]
+    assert [device["rank_id"] for device in devices] == ["0", "1", "2", "3"]
+    assert devices[0] == {
+        "device_id": "0",
+        "rank_id": "0",
+        "super_device_id": "20971520",
+        "device_ip": "2001:db8::1",
+    }
+    assert devices[3]["super_device_id"] == "63176705"
+    assert table["super_pod_list"] == [
+        {
+            "super_pod_id": "5",
+            "server_list": [{"server_id": "192.0.2.10"}],
+        },
+        {
+            "super_pod_id": "15",
+            "server_list": [{"server_id": "198.51.100.20"}],
+        },
+    ]
+    generator.validate_rank_table(table)
+
+
+def test_host_inventory_accepts_comments_and_discards_credentials(tmp_path: Path) -> None:
+    generator = load_generator()
+    hosts_file = tmp_path / "hosts"
+    hosts_file.write_text(
+        "# pod 5\nnode-a:do-not-retain\n\nnode-b\n",
+        encoding="utf-8",
+    )
+
+    assert generator.read_hosts(hosts_file) == ["node-a", "node-b"]
+
+
+def test_system_ssh_collector_queries_only_interface_and_rootinfo(monkeypatch) -> None:
+    generator = load_generator()
+    captured: list[list[str]] = []
+
+    def fake_run(command, **_kwargs):
+        captured.append(command)
+        stdout = json.dumps(interface_info("192.0.2.10"))
+        stdout += json.dumps(root_info("20010db8000000000000000000000001"))
+        return generator.subprocess.CompletedProcess(command, 0, stdout, "")
+
+    monkeypatch.setattr(generator.subprocess, "run", fake_run)
+    interface, root = generator.collect_host_snapshot(
+        "node-a",
+        ssh_user="root",
+        interface="data0.3001",
+        rootinfo_path="/etc/hccl_rootinfo.json.bak",
+        timeout=30,
+        ssh_options=["StrictHostKeyChecking=yes"],
+    )
+
+    assert interface == interface_info("192.0.2.10")
+    assert root == root_info("20010db8000000000000000000000001")
+    assert captured[0][-2] == "root@node-a"
+    assert captured[0][-1] == (
+        "ip -j -4 addr show dev data0.3001 && cat /etc/hccl_rootinfo.json.bak"
+    )
+
+
+def test_rejects_an_ambiguous_primary_eid() -> None:
+    generator = load_generator()
+    info = root_info("20010db8000000000000000000000001")
+    level = info["rank_list"][0]["level_list"][1]
+    level["rank_addr_list"].append(
+        {
+            "addr_type": "EID",
+            "addr": "20010db8000000000000000000000002",
+            "ports": ["0/3"],
+        }
+    )
+
+    with pytest.raises(generator.RankTableError, match="unique 0/ EID"):
+        generator.extract_devices(info, devices_per_server=1, host="node-a")
+
+
+def test_rejects_duplicate_eids_across_servers() -> None:
+    generator = load_generator()
+    duplicate = "20010db8000000000000000000000001"
+    snapshots = {
+        "node-a": (interface_info("192.0.2.10"), root_info(duplicate)),
+        "node-b": (interface_info("198.51.100.20"), root_info(duplicate)),
+    }
+
+    with pytest.raises(generator.RankTableError, match="duplicate device_ip"):
+        generator.build_rank_table(
+            hosts=["node-a", "node-b"],
+            super_pod_ids=[5, 15],
+            servers_per_super_pod=1,
+            devices_per_server=1,
+            collect=lambda host: snapshots[host],
+        )
+
+
+def test_rendered_table_is_stable_and_offline_check_rejects_gaps(tmp_path: Path) -> None:
+    generator = load_generator()
+    table = generator.build_rank_table(
+        hosts=["node-a"],
+        super_pod_ids=[5],
+        servers_per_super_pod=1,
+        devices_per_server=1,
+        collect=lambda _host: (
+            interface_info("192.0.2.10"),
+            root_info("20010db8000000000000000000000001"),
+        ),
+    )
+    output = tmp_path / "rank_table.json"
+    generator.write_rank_table(table, output)
+    assert output.read_text(encoding="utf-8") == json.dumps(table, indent=2) + "\n"
+    assert output.read_bytes() == (json.dumps(table, indent=2) + "\n").encode("utf-8")
+
+    table["server_list"][0]["device"][0]["rank_id"] = "1"
+    with pytest.raises(generator.RankTableError, match="contiguous"):
+        generator.validate_rank_table(table)
diff --git a/tools/moonep/mindspeed/README.md b/tools/moonep/mindspeed/README.md
index 703b055..04083e9 100644
--- a/tools/moonep/mindspeed/README.md
+++ b/tools/moonep/mindspeed/README.md
@@ -5,6 +5,8 @@ backend with MindSpeed on Ascend 950 nodes.
 
 - `tilexr_mindspeed_adapter.py`: MindSpeed backend adapter that keeps communication
   buffers owned by TileXR.
+- `mindspeed_stage_barrier.py`: shared optional world barrier immediately before
+  native or TileXR Dispatch and Combine calls, including backward calls.
 - `mindspeed_external_comm_owner.patch`: MindSpeed patch that lets an external backend
   own the communication runtime and buffer.
 - `preflight_adapter.sh`: validates the adapter and the patched MindSpeed ownership
@@ -17,12 +19,21 @@ backend with MindSpeed on Ascend 950 nodes.
   multi-node SSH orchestration.
 - `run_model_node.sh`: non-interactive per-node runner based on the validated
   4K/8P model command.
+- `probe_idle.sh`: bounded NPU idle gate with a conservative device-node fallback
+  for hosts where a concurrent management query blocks full `npu-smi info`.
 
 The runners use the current repository by default and write results below
 `run/moonep/mindspeed`. Override paths with `TILEXR_HOME`,
 `TILEXR_INSTALL_PREFIX`, `TILEXR_CANN_ENV`, `TILEXR_CONDA_SH`,
 `TILEXR_CONDA_ENV`, and `TILEXR_MOONEP_NATIVE_ENV`.
 
+Every multi-node run uses one global expert-parallel group spanning all ranks for
+both backends. The model keeps at least 32 experts and rounds that count up to a
+multiple of the global rank count; for example, 16 nodes x 8 devices runs with
+EP128 and 128 experts, while a single-node run remains EP8 with 32 experts.
+Multi-node runs prefer `data0.3001` for HCCL and Gloo on the validated cluster;
+set `MODEL_RUNNER_SOCKET_IFNAME` to override that interface on other deployments.
+
 Before using the adapter, apply the MindSpeed ownership patch and run its
 preflight:
 
@@ -36,6 +47,53 @@ MINDSPEED_HOME="${MINDSPEED_HOME}" \
 The preflight installs `tilexr_mindspeed_adapter.py` into the checkout named by
 `MINDSPEED_HOME`; use a disposable or task-owned MindSpeed checkout.
 
+## HCCL rank-table generator
+
+`generate_rank_table.py` builds the HCCL v1.2 table used by the validated
+Ascend 950 multi-node topology. It queries only the selected interface with
+`ip -j` and reads each node's HCCL RootInfo JSON; it does not start an NPU
+process or run a model or HCCL Test. The validated 16-node cluster has
+`/etc/hccl_rootinfo.json.bak` but no `/etc/hccl_rootinfo.json`; deployments
+with the canonical file should pass `--rootinfo-path /etc/hccl_rootinfo.json`.
+
+For the 16-node, 128-rank cluster:
+
+```bash
+python3 tools/moonep/mindspeed/generate_rank_table.py \
+    --hosts /path/to/hosts.txt \
+    --super-pod-ids 5,15 \
+    --servers-per-super-pod 8 \
+    --devices-per-server 8 \
+    --interface data0.3001 \
+    --rootinfo-path /etc/hccl_rootinfo.json.bak \
+    --output run/moonep/mindspeed/config/rank_table_128.json
+```
+
+Host order defines rank order. Blank lines and lines beginning with `#` are
+ignored. An inventory entry may be either `HOST` or `HOST:CREDENTIAL`; the
+credential suffix is discarded and never stored or printed. Authentication is
+owned by the system SSH client, so configure its agent, askpass, or terminal
+prompt outside this tool. Add system SSH options with repeated
+`--ssh-option KEY=VALUE` arguments.
+
+The generator requires one unambiguous `0/*` EID from `net_layer=1` for every
+device. The `super_device_id` values use the encoding measured on this B131
+Ascend 950 deployment:
+
+```text
+super_device_id = super_pod_id * 4194304 + device_id * 262145
+```
+
+Confirm that encoding before using the tool on another hardware generation.
+Generated rank tables use deterministic UTF-8/LF formatting so hashes match
+across Linux and Windows, and belong under the ignored `run/` tree. Validate
+an existing file without connecting to any host:
+
+```bash
+python3 tools/moonep/mindspeed/generate_rank_table.py \
+    --check run/moonep/mindspeed/config/rank_table_128.json
+```
+
 ## Full model runner
 
 The first invocation asks for the node list and deployed paths, then writes the
@@ -75,7 +133,8 @@ bash tools/moonep/mindspeed/run_model.sh --mode multi --dry-run
 # Replace the cached answers.
 bash tools/moonep/mindspeed/run_model.sh --configure --mode multi --dry-run
 
-# Enable one profiler window. The diagnostic stage barrier is off by default.
+# Enable one profiler window. The stage barrier is off by default and applies to
+# both native MoonEP and TileXR MoonEP when requested.
 bash tools/moonep/mindspeed/run_model.sh --mode multi --profile
 bash tools/moonep/mindspeed/run_model.sh --mode multi --stage-barrier
 ```
diff --git a/tools/moonep/mindspeed/generate_rank_table.py b/tools/moonep/mindspeed/generate_rank_table.py
new file mode 100644
index 0000000..8b4f344
--- /dev/null
+++ b/tools/moonep/mindspeed/generate_rank_table.py
@@ -0,0 +1,499 @@
+#!/usr/bin/env python3
+"""Generate an HCCL v1.2 rank table from Ascend 950 RootInfo files."""
+
+from __future__ import annotations
+
+import argparse
+import concurrent.futures
+import ipaddress
+import json
+import re
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+from typing import Callable, Sequence
+
+
+SUPER_POD_STRIDE = 4_194_304
+DEVICE_STRIDE = 262_145
+HOST_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
+HEX_EID_PATTERN = re.compile(r"^[0-9A-Fa-f]{32}$")
+
+
+class RankTableError(ValueError):
+    """Raised when discovery data cannot form an unambiguous rank table."""
+
+
+def _integer(value: object, label: str) -> int:
+    if isinstance(value, bool):
+        raise RankTableError(f"{label} must be an integer")
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError) as exc:
+        raise RankTableError(f"{label} must be an integer") from exc
+    if parsed < 0:
+        raise RankTableError(f"{label} must not be negative")
+    return parsed
+
+
+def read_hosts(path: Path) -> list[str]:
+    """Read ordered hosts, ignoring comments and optional credential suffixes."""
+    hosts: list[str] = []
+    seen: set[str] = set()
+    try:
+        lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError as exc:
+        raise RankTableError(f"failed to read host inventory {path}: {exc}") from exc
+
+    for line_number, raw_line in enumerate(lines, start=1):
+        line = raw_line.strip()
+        if not line or line.startswith("#"):
+            continue
+        token = line.split(maxsplit=1)[0]
+        host = token.split(":", maxsplit=1)[0]
+        if not HOST_PATTERN.fullmatch(host):
+            raise RankTableError(f"invalid host at {path}:{line_number}")
+        if host in seen:
+            raise RankTableError(f"duplicate host in {path}: {host}")
+        seen.add(host)
+        hosts.append(host)
+
+    if not hosts:
+        raise RankTableError(f"host inventory is empty: {path}")
+    return hosts
+
+
+def _decode_json_documents(text: str, host: str) -> tuple[object, object]:
+    decoder = json.JSONDecoder()
+    documents: list[object] = []
+    position = 0
+    while position < len(text):
+        while position < len(text) and text[position].isspace():
+            position += 1
+        if position == len(text):
+            break
+        try:
+            document, position = decoder.raw_decode(text, position)
+        except json.JSONDecodeError as exc:
+            raise RankTableError(f"{host}: SSH output is not valid JSON: {exc}") from exc
+        documents.append(document)
+    if len(documents) != 2:
+        raise RankTableError(
+            f"{host}: expected interface JSON and RootInfo JSON, got {len(documents)} documents"
+        )
+    return documents[0], documents[1]
+
+
+def collect_host_snapshot(
+    host: str,
+    *,
+    ssh_user: str,
+    interface: str,
+    rootinfo_path: str,
+    timeout: int,
+    ssh_options: Sequence[str],
+) -> tuple[object, object]:
+    if not HOST_PATTERN.fullmatch(ssh_user):
+        raise RankTableError(f"invalid SSH user: {ssh_user}")
+    for option in ssh_options:
+        if "\n" in option or "\r" in option or "\0" in option:
+            raise RankTableError("SSH options must be single-line values")
+
+    remote_command = "ip -j -4 addr show dev {} && cat {}".format(
+        shlex.quote(interface), shlex.quote(rootinfo_path)
+    )
+    command = [
+        "ssh",
+        "-T",
+        "-o",
+        f"ConnectTimeout={min(timeout, 15)}",
+        "-o",
+        "ServerAliveInterval=15",
+        "-o",
+        "ServerAliveCountMax=2",
+    ]
+    for option in ssh_options:
+        command.extend(("-o", option))
+    command.extend((f"{ssh_user}@{host}", remote_command))
+
+    try:
+        completed = subprocess.run(
+            command,
+            text=True,
+            capture_output=True,
+            timeout=timeout,
+            check=False,
+        )
+    except FileNotFoundError as exc:
+        raise RankTableError("system ssh command was not found") from exc
+    except subprocess.TimeoutExpired as exc:
+        raise RankTableError(f"{host}: SSH discovery timed out after {timeout}s") from exc
+    if completed.returncode != 0:
+        detail = completed.stderr.strip() or f"exit code {completed.returncode}"
+        raise RankTableError(f"{host}: SSH discovery failed: {detail}")
+    return _decode_json_documents(completed.stdout, host)
+
+
+def collect_snapshots(
+    hosts: Sequence[str],
+    *,
+    ssh_user: str,
+    interface: str,
+    rootinfo_path: str,
+    timeout: int,
+    ssh_options: Sequence[str],
+    jobs: int,
+) -> dict[str, tuple[object, object]]:
+    if jobs < 1:
+        raise RankTableError("jobs must be at least 1")
+    snapshots: dict[str, tuple[object, object]] = {}
+    with concurrent.futures.ThreadPoolExecutor(max_workers=min(jobs, len(hosts))) as pool:
+        futures = {
+            pool.submit(
+                collect_host_snapshot,
+                host,
+                ssh_user=ssh_user,
+                interface=interface,
+                rootinfo_path=rootinfo_path,
+                timeout=timeout,
+                ssh_options=ssh_options,
+            ): host
+            for host in hosts
+        }
+        for future in concurrent.futures.as_completed(futures):
+            host = futures[future]
+            snapshots[host] = future.result()
+    return snapshots
+
+
+def extract_server_id(interface_info: object, host: str) -> str:
+    if not isinstance(interface_info, list):
+        raise RankTableError(f"{host}: interface query must return a JSON array")
+    addresses: list[str] = []
+    for interface in interface_info:
+        if not isinstance(interface, dict):
+            continue
+        addr_info = interface.get("addr_info", [])
+        if not isinstance(addr_info, list):
+            continue
+        for address in addr_info:
+            if not isinstance(address, dict):
+                continue
+            if address.get("family") == "inet" and address.get("scope") == "global":
+                local = address.get("local")
+                try:
+                    parsed = ipaddress.IPv4Address(str(local))
+                except ipaddress.AddressValueError as exc:
+                    raise RankTableError(f"{host}: malformed interface IPv4 address") from exc
+                addresses.append(str(parsed))
+    addresses = list(dict.fromkeys(addresses))
+    if len(addresses) != 1:
+        raise RankTableError(f"{host}: expected one global interface IPv4 address")
+    return addresses[0]
+
+
+def _primary_eid(device: dict[str, object], host: str, device_id: int) -> str:
+    level_list = device.get("level_list")
+    if not isinstance(level_list, list):
+        raise RankTableError(f"{host}: device {device_id} has no level_list")
+    network_levels = [
+        level
+        for level in level_list
+        if isinstance(level, dict) and _integer(level.get("net_layer"), "net_layer") == 1
+    ]
+    if len(network_levels) != 1:
+        raise RankTableError(f"{host}: device {device_id} must have one net_layer=1 entry")
+
+    rank_addresses = network_levels[0].get("rank_addr_list")
+    if not isinstance(rank_addresses, list):
+        raise RankTableError(f"{host}: device {device_id} has no rank_addr_list")
+    candidates: list[str] = []
+    for address in rank_addresses:
+        if not isinstance(address, dict) or address.get("addr_type") != "EID":
+            continue
+        ports = address.get("ports")
+        if not isinstance(ports, list) or not ports:
+            continue
+        if all(isinstance(port, str) and port.startswith("0/") for port in ports):
+            candidates.append(str(address.get("addr", "")))
+    if len(candidates) != 1:
+        raise RankTableError(f"{host}: device {device_id} requires a unique 0/ EID")
+
+    raw_eid = candidates[0]
+    if not HEX_EID_PATTERN.fullmatch(raw_eid):
+        raise RankTableError(f"{host}: device {device_id} has a malformed EID")
+    return str(ipaddress.IPv6Address(bytes.fromhex(raw_eid)))
+
+
+def extract_devices(
+    root_info: object, *, devices_per_server: int, host: str
+) -> list[tuple[int, str]]:
+    if not isinstance(root_info, dict):
+        raise RankTableError(f"{host}: RootInfo must be a JSON object")
+    if str(root_info.get("version")) != "2.0":
+        raise RankTableError(f"{host}: expected RootInfo version 2.0")
+    if _integer(root_info.get("rank_count"), "rank_count") != devices_per_server:
+        raise RankTableError(
+            f"{host}: RootInfo rank_count does not match {devices_per_server} devices"
+        )
+    rank_list = root_info.get("rank_list")
+    if not isinstance(rank_list, list) or len(rank_list) != devices_per_server:
+        raise RankTableError(f"{host}: RootInfo rank_list is incomplete")
+
+    by_device: dict[int, dict[str, object]] = {}
+    for device in rank_list:
+        if not isinstance(device, dict):
+            raise RankTableError(f"{host}: RootInfo device entry must be an object")
+        device_id = _integer(device.get("device_id"), "device_id")
+        if device_id in by_device:
+            raise RankTableError(f"{host}: duplicate device_id {device_id}")
+        by_device[device_id] = device
+    expected_ids = set(range(devices_per_server))
+    if set(by_device) != expected_ids:
+        raise RankTableError(f"{host}: device IDs must be 0..{devices_per_server - 1}")
+
+    return [
+        (device_id, _primary_eid(by_device[device_id], host, device_id))
+        for device_id in range(devices_per_server)
+    ]
+
+
+def build_rank_table(
+    *,
+    hosts: Sequence[str],
+    super_pod_ids: Sequence[int],
+    servers_per_super_pod: int,
+    devices_per_server: int,
+    collect: Callable[[str], tuple[object, object]],
+) -> dict[str, object]:
+    if servers_per_super_pod < 1 or devices_per_server < 1:
+        raise RankTableError("server and device counts must be at least 1")
+    if len(set(super_pod_ids)) != len(super_pod_ids):
+        raise RankTableError("super-pod IDs must be unique")
+    if any(super_pod_id < 0 for super_pod_id in super_pod_ids):
+        raise RankTableError("super-pod IDs must not be negative")
+    expected_servers = len(super_pod_ids) * servers_per_super_pod
+    if len(hosts) != expected_servers:
+        raise RankTableError(
+            f"host count {len(hosts)} does not match {expected_servers} super-pod servers"
+        )
+
+    server_list: list[dict[str, object]] = []
+    for server_index, host in enumerate(hosts):
+        interface_info, root_info = collect(host)
+        server_id = extract_server_id(interface_info, host)
+        super_pod_id = super_pod_ids[server_index // servers_per_super_pod]
+        devices = []
+        for device_id, device_ip in extract_devices(
+            root_info, devices_per_server=devices_per_server, host=host
+        ):
+            rank_id = server_index * devices_per_server + device_id
+            super_device_id = super_pod_id * SUPER_POD_STRIDE + device_id * DEVICE_STRIDE
+            devices.append(
+                {
+                    "device_id": str(device_id),
+                    "rank_id": str(rank_id),
+                    "super_device_id": str(super_device_id),
+                    "device_ip": device_ip,
+                }
+            )
+        server_list.append({"server_id": server_id, "device": devices})
+
+    super_pod_list = []
+    for pod_index, super_pod_id in enumerate(super_pod_ids):
+        begin = pod_index * servers_per_super_pod
+        pod_servers = server_list[begin : begin + servers_per_super_pod]
+        super_pod_list.append(
+            {
+                "super_pod_id": str(super_pod_id),
+                "server_list": [
+                    {"server_id": server["server_id"]} for server in pod_servers
+                ],
+            }
+        )
+
+    table: dict[str, object] = {
+        "version": "1.2",
+        "server_count": str(len(server_list)),
+        "server_list": server_list,
+        "super_pod_list": super_pod_list,
+        "status": "completed",
+    }
+    validate_rank_table(table)
+    return table
+
+
+def validate_rank_table(table: object) -> None:
+    if not isinstance(table, dict):
+        raise RankTableError("rank table must be a JSON object")
+    if table.get("version") != "1.2" or table.get("status") != "completed":
+        raise RankTableError("rank table requires version 1.2 and completed status")
+    server_list = table.get("server_list")
+    super_pod_list = table.get("super_pod_list")
+    if not isinstance(server_list, list) or not server_list:
+        raise RankTableError("rank table server_list must not be empty")
+    if not isinstance(super_pod_list, list) or not super_pod_list:
+        raise RankTableError("rank table super_pod_list must not be empty")
+    if _integer(table.get("server_count"), "server_count") != len(server_list):
+        raise RankTableError("server_count does not match server_list")
+
+    server_ids: list[str] = []
+    rank_ids: list[int] = []
+    device_ips: list[str] = []
+    device_count: int | None = None
+    devices_by_server: dict[str, list[dict[str, object]]] = {}
+    for server in server_list:
+        if not isinstance(server, dict) or not isinstance(server.get("device"), list):
+            raise RankTableError("each server requires a device list")
+        try:
+            server_id = str(ipaddress.IPv4Address(str(server.get("server_id"))))
+        except ipaddress.AddressValueError as exc:
+            raise RankTableError("server_id must be an IPv4 address") from exc
+        if server_id in devices_by_server:
+            raise RankTableError(f"duplicate server_id: {server_id}")
+        devices = server["device"]
+        if not devices:
+            raise RankTableError(f"{server_id}: device list must not be empty")
+        if device_count is None:
+            device_count = len(devices)
+        elif len(devices) != device_count:
+            raise RankTableError("all servers must have the same device count")
+        devices_by_server[server_id] = devices
+        server_ids.append(server_id)
+        for device in devices:
+            if not isinstance(device, dict):
+                raise RankTableError("device entries must be objects")
+            rank_ids.append(_integer(device.get("rank_id"), "rank_id"))
+            device_ip = str(device.get("device_ip"))
+            try:
+                ipaddress.IPv6Address(device_ip)
+            except ipaddress.AddressValueError as exc:
+                raise RankTableError(f"invalid device_ip: {device_ip}") from exc
+            device_ips.append(device_ip)
+    if rank_ids != list(range(len(rank_ids))):
+        raise RankTableError("rank IDs must be contiguous in server/device order")
+    if len(set(device_ips)) != len(device_ips):
+        raise RankTableError("rank table contains duplicate device_ip values")
+
+    referenced_servers: list[str] = []
+    seen_pods: set[int] = set()
+    assert device_count is not None
+    for pod in super_pod_list:
+        if not isinstance(pod, dict) or not isinstance(pod.get("server_list"), list):
+            raise RankTableError("each super-pod requires a server_list")
+        super_pod_id = _integer(pod.get("super_pod_id"), "super_pod_id")
+        if super_pod_id in seen_pods:
+            raise RankTableError(f"duplicate super_pod_id: {super_pod_id}")
+        seen_pods.add(super_pod_id)
+        for server_reference in pod["server_list"]:
+            if not isinstance(server_reference, dict):
+                raise RankTableError("super-pod server references must be objects")
+            server_id = str(server_reference.get("server_id"))
+            if server_id not in devices_by_server:
+                raise RankTableError(f"unknown super-pod server_id: {server_id}")
+            referenced_servers.append(server_id)
+            devices = devices_by_server[server_id]
+            device_ids = [_integer(device.get("device_id"), "device_id") for device in devices]
+            if device_ids != list(range(device_count)):
+                raise RankTableError(f"{server_id}: device IDs must be contiguous")
+            for device_id, device in zip(device_ids, devices):
+                expected = super_pod_id * SUPER_POD_STRIDE + device_id * DEVICE_STRIDE
+                actual = _integer(device.get("super_device_id"), "super_device_id")
+                if actual != expected:
+                    raise RankTableError(
+                        f"{server_id}: super_device_id does not match super-pod/device encoding"
+                    )
+    if referenced_servers != server_ids:
+        raise RankTableError("super_pod_list must reference every server once and in order")
+
+
+def write_rank_table(table: dict[str, object], output: Path) -> None:
+    validate_rank_table(table)
+    try:
+        output.parent.mkdir(parents=True, exist_ok=True)
+        payload = (json.dumps(table, indent=2) + "\n").encode("utf-8")
+        output.write_bytes(payload)
+    except OSError as exc:
+        raise RankTableError(f"failed to write rank table {output}: {exc}") from exc
+
+
+def _parse_super_pod_ids(value: str) -> list[int]:
+    try:
+        values = [_integer(part.strip(), "super_pod_id") for part in value.split(",")]
+    except RankTableError as exc:
+        raise argparse.ArgumentTypeError(str(exc)) from exc
+    if not values or any(not part.strip() for part in value.split(",")):
+        raise argparse.ArgumentTypeError("super-pod IDs must be a comma-separated list")
+    return values
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    source = parser.add_mutually_exclusive_group(required=True)
+    source.add_argument("--hosts", type=Path, help="Ordered host inventory")
+    source.add_argument("--check", type=Path, help="Validate an existing rank table")
+    parser.add_argument("--super-pod-ids", type=_parse_super_pod_ids)
+    parser.add_argument("--servers-per-super-pod", type=int, default=8)
+    parser.add_argument("--devices-per-server", type=int, default=8)
+    parser.add_argument("--interface", default="data0.3001")
+    parser.add_argument("--rootinfo-path", default="/etc/hccl_rootinfo.json.bak")
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--ssh-user", default="root")
+    parser.add_argument(
+        "--ssh-option",
+        action="append",
+        default=[],
+        metavar="KEY=VALUE",
+        help="Additional system SSH -o option; repeat as needed",
+    )
+    parser.add_argument("--timeout", type=int, default=30, help="Per-host SSH timeout")
+    parser.add_argument("--jobs", type=int, default=16, help="Parallel SSH queries")
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.check is not None:
+            table = json.loads(args.check.read_text(encoding="utf-8"))
+            validate_rank_table(table)
+            rank_count = sum(len(server["device"]) for server in table["server_list"])
+            print(f"valid HCCL v1.2 rank table: servers={table['server_count']} ranks={rank_count}")
+            return 0
+        if args.super_pod_ids is None:
+            parser.error("--super-pod-ids is required with --hosts")
+        if args.output is None:
+            parser.error("--output is required with --hosts")
+        if args.timeout < 1:
+            parser.error("--timeout must be at least 1")
+
+        hosts = read_hosts(args.hosts)
+        snapshots = collect_snapshots(
+            hosts,
+            ssh_user=args.ssh_user,
+            interface=args.interface,
+            rootinfo_path=args.rootinfo_path,
+            timeout=args.timeout,
+            ssh_options=args.ssh_option,
+            jobs=args.jobs,
+        )
+        table = build_rank_table(
+            hosts=hosts,
+            super_pod_ids=args.super_pod_ids,
+            servers_per_super_pod=args.servers_per_super_pod,
+            devices_per_server=args.devices_per_server,
+            collect=lambda host: snapshots[host],
+        )
+        write_rank_table(table, args.output)
+        rank_count = len(hosts) * args.devices_per_server
+        print(f"wrote {args.output}: servers={len(hosts)} ranks={rank_count}")
+        return 0
+    except (OSError, json.JSONDecodeError, RankTableError) as exc:
+        print(f"error: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/tools/moonep/mindspeed/mindspeed_stage_barrier.py b/tools/moonep/mindspeed/mindspeed_stage_barrier.py
new file mode 100644
index 0000000..4306b9c
--- /dev/null
+++ b/tools/moonep/mindspeed/mindspeed_stage_barrier.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import os
+
+import moonep as upstream_moonep
+
+from .moonep_backend import MOONEP_NATIVE_NPU_CAPABILITY, MoonEPBufferFlexBackend
+
+
+_STAGE_BARRIER_ENV = "MOONEP_MINDSPEED_STAGE_BARRIER"
+
+
+def optional_stage_barrier(torch_module) -> None:
+    if os.environ.get(_STAGE_BARRIER_ENV, "0") != "1":
+        return
+    distributed = torch_module.distributed
+    if not distributed.is_available() or not distributed.is_initialized():
+        raise RuntimeError(
+            "MoonEP stage barrier requires an initialized process group"
+        )
+    distributed.barrier()
+
+
+class MindSpeedBarrierMoonEPBuffer(upstream_moonep.Buffer):
+    """Native MoonEP Buffer with opt-in pre-Dispatch/Combine synchronization."""
+
+    def dispatch(self, *args, **kwargs):
+        import torch
+
+        optional_stage_barrier(torch)
+        return super().dispatch(*args, **kwargs)
+
+    def combine(self, *args, **kwargs):
+        import torch
+
+        optional_stage_barrier(torch)
+        return super().combine(*args, **kwargs)
+
+
+def create_native_barrier_backend(**kwargs):
+    kwargs["buffer_cls"] = MindSpeedBarrierMoonEPBuffer
+    return MoonEPBufferFlexBackend(**kwargs)
+
+
+create_native_barrier_backend.__mindspeed_capabilities__ = {
+    MOONEP_NATIVE_NPU_CAPABILITY
+}
diff --git a/tools/moonep/mindspeed/preflight_adapter.sh b/tools/moonep/mindspeed/preflight_adapter.sh
index b6be6a5..8162873 100644
--- a/tools/moonep/mindspeed/preflight_adapter.sh
+++ b/tools/moonep/mindspeed/preflight_adapter.sh
@@ -7,6 +7,7 @@ tilexr_home=${TILEXR_HOME:-$(cd "${script_dir}/../../.." && pwd)}
 mindspeed_home=${MINDSPEED_HOME:?MINDSPEED_HOME must point to the MindSpeed checkout}
 adapter_dir=${mindspeed_home}/mindspeed/core/transformer/moe
 adapter=${adapter_dir}/tilexr_mindspeed_adapter.py
+stage_barrier=${adapter_dir}/mindspeed_stage_barrier.py
 install_prefix=${TILEXR_INSTALL_PREFIX:-${tilexr_home}/install}
 
 if [[ ! -d "${adapter_dir}" ]]; then
@@ -15,6 +16,7 @@ if [[ ! -d "${adapter_dir}" ]]; then
 fi
 
 install -m 0644 "${script_dir}/tilexr_mindspeed_adapter.py" "${adapter}"
+install -m 0644 "${script_dir}/mindspeed_stage_barrier.py" "${stage_barrier}"
 if grep -Eq 'from moonep import Buffer|aclshmem' "${adapter}"; then
     echo "forbidden SHMEM Buffer dependency in TileXR adapter" >&2
     exit 1
@@ -30,9 +32,11 @@ from types import SimpleNamespace
 import torch
 
 from mindspeed.core.transformer.moe import tilexr_mindspeed_adapter as adapter
+from mindspeed.core.transformer.moe import mindspeed_stage_barrier
 from mindspeed.core.transformer.moe.moonep_model_arena import MoonEPModelArenaLayout
 
 assert adapter.MindSpeedTileXRBuffer.__mro__[1].__module__ == "tilexr_moonep.compat"
+assert callable(mindspeed_stage_barrier.create_native_barrier_backend)
 assert "hidden_buffer" in inspect.signature(adapter.MindSpeedTileXRBuffer.dispatch).parameters
 assert "hidden_buffer" in inspect.signature(adapter.MindSpeedTileXRBuffer.combine).parameters
 assert adapter.MOONEP_NATIVE_NPU_CAPABILITY in (
diff --git a/tools/moonep/mindspeed/probe_idle.sh b/tools/moonep/mindspeed/probe_idle.sh
new file mode 100644
index 0000000..a037b93
--- /dev/null
+++ b/tools/moonep/mindspeed/probe_idle.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+devices=
+log=/dev/null
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --devices) devices=${2:?--devices requires a value}; shift 2 ;;
+        --log) log=${2:?--log requires a value}; shift 2 ;;
+        *) printf 'Unknown argument: %s\n' "$1" >&2; exit 2 ;;
+    esac
+done
+if [[ ! "${devices}" =~ ^[1-9][0-9]*$ ]]; then
+    echo "--devices must be a positive integer" >&2
+    exit 2
+fi
+
+mkdir -p "$(dirname "${log}")"
+accelerator_process_pattern='pretrain_gpt.py|torch.distributed.launch|hccl_test/bin/|all_reduce_test|alltoallv_test|tilexr_udma_dem'
+accelerator_processes=$(pgrep -fc "${accelerator_process_pattern}" || true)
+if timeout 15s npu-smi info >"${log}" 2>&1; then
+    idle=$(grep -c 'No running processes found in NPU' "${log}" || true)
+    if [[ "${idle}" -eq "${devices}" && "${accelerator_processes}" -eq 0 ]]; then
+        echo "${idle}"
+        exit 0
+    fi
+    printf 'probe=full reported_idle=%s expected=%s accelerator_processes=%s; checking live ownership\n' \
+        "${idle}" "${devices}" "${accelerator_processes}" >>"${log}"
+fi
+
+list_log=${log}.list
+if ! timeout 5s npu-smi info -l >"${list_log}" 2>&1; then
+    printf 'probe=fallback list=failed\n' >>"${log}"
+    echo 0
+    exit 0
+fi
+listed=$(sed -n 's/.*Total Count[[:space:]]*:[[:space:]]*\([0-9][0-9]*\).*/\1/p' \
+    "${list_log}" | head -1)
+device_nodes=$(find /dev -maxdepth 1 -type c -name 'davinci[0-9]*' 2>/dev/null | wc -l)
+device_users=0
+if command -v fuser >/dev/null 2>&1; then
+    device_users=$(fuser /dev/davinci[0-9]* 2>/dev/null | wc -w || true)
+else
+    device_users=-1
+fi
+printf 'probe=fallback listed=%s device_nodes=%s device_users=%s accelerator_processes=%s\n' \
+    "${listed:-unknown}" "${device_nodes}" "${device_users}" "${accelerator_processes}" \
+    >>"${log}"
+if [[ "${listed:-0}" -eq "${devices}" && "${device_nodes}" -eq "${devices}" && \
+      "${device_users}" -eq 0 && "${accelerator_processes}" -eq 0 ]]; then
+    echo "${devices}"
+else
+    echo 0
+fi
diff --git a/tools/moonep/mindspeed/run_model.sh b/tools/moonep/mindspeed/run_model.sh
index fae0170..b8c2538 100755
--- a/tools/moonep/mindspeed/run_model.sh
+++ b/tools/moonep/mindspeed/run_model.sh
@@ -14,6 +14,10 @@ Options:
   --backend tilexr|native   Select the MoonEP backend (default: tilexr).
   --profile                 Enable the one-iteration NPU profiler window.
   --stage-barrier           Add diagnostic world barriers before Dispatch/Combine.
+  --hccl-inter-hccs-disable true|false
+                            Select HCCL inter-HCCS behavior explicitly.
+  --rank-table-file PATH    Use the same explicit HCCL rank table on every node.
+  --udma-rootinfo-path PATH Use each node's TileXR UDMA RootInfo file.
   --configure               Replace the cached configuration interactively.
   --config PATH             Use an alternate cached configuration file.
   --master-port PORT        Distributed launcher port (default: 29501).
@@ -33,6 +37,9 @@ mode=single
 backend=tilexr
 profile=0
 stage_barrier=0
+hccl_inter_hccs_disable=""
+rank_table_file=""
+udma_rootinfo_path=""
 configure=0
 dry_run=0
 config=${TILEXR_MODEL_RUNNER_CONFIG:-${default_config}}
@@ -47,6 +54,15 @@ while [[ $# -gt 0 ]]; do
         --backend) backend=${2:?--backend requires a value}; shift 2 ;;
         --profile) profile=1; shift ;;
         --stage-barrier) stage_barrier=1; shift ;;
+        --hccl-inter-hccs-disable)
+            hccl_inter_hccs_disable=${2:?--hccl-inter-hccs-disable requires a value}
+            shift 2
+            ;;
+        --rank-table-file) rank_table_file=${2:?--rank-table-file requires a value}; shift 2 ;;
+        --udma-rootinfo-path)
+            udma_rootinfo_path=${2:?--udma-rootinfo-path requires a value}
+            shift 2
+            ;;
         --configure) configure=1; shift ;;
         --config) config=${2:?--config requires a value}; shift 2 ;;
         --master-port) master_port=${2:?--master-port requires a value}; shift 2 ;;
@@ -61,10 +77,9 @@ done
 
 case "${mode}" in single|multi) ;; *) printf 'Invalid mode: %s\n' "${mode}" >&2; exit 2 ;; esac
 case "${backend}" in tilexr|native) ;; *) printf 'Invalid backend: %s\n' "${backend}" >&2; exit 2 ;; esac
-if [[ "${stage_barrier}" -eq 1 && "${backend}" != tilexr ]]; then
-    echo "--stage-barrier is supported only by the TileXR backend" >&2
-    exit 2
-fi
+case "${hccl_inter_hccs_disable}" in ""|true|false) ;;
+    *) printf 'Invalid HCCL inter-HCCS value: %s\n' "${hccl_inter_hccs_disable}" >&2; exit 2 ;;
+esac
 if [[ ! "${master_port}" =~ ^[0-9]+$ ]] || (( master_port < 1 || master_port > 65535 )); then
     printf 'Invalid master port: %s\n' "${master_port}" >&2
     exit 2
@@ -214,10 +229,16 @@ node_arguments() {
     )
     [[ "${profile}" -eq 1 ]] && args+=(--profile)
     [[ "${stage_barrier}" -eq 1 ]] && args+=(--stage-barrier)
+    [[ -n "${hccl_inter_hccs_disable}" ]] && \
+        args+=(--hccl-inter-hccs-disable "${hccl_inter_hccs_disable}")
+    [[ -n "${rank_table_file}" ]] && args+=(--rank-table-file "${rank_table_file}")
+    [[ -n "${udma_rootinfo_path}" ]] && \
+        args+=(--udma-rootinfo-path "${udma_rootinfo_path}")
     quote_command "${args[@]}"
 }
 
 remote_runner=${MODEL_RUNNER_TILEXR_HOME}/tools/moonep/mindspeed/run_model_node.sh
+remote_idle_probe=${MODEL_RUNNER_TILEXR_HOME}/tools/moonep/mindspeed/probe_idle.sh
 local_runner=${script_dir}/run_model_node.sh
 ssh_options=(-o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=3)
 
@@ -265,12 +286,26 @@ done
 wait_for_stable_idle() {
     local deadline=$((SECONDS + idle_wait_sec))
     local consecutive=0
-    local index idle all_idle
+    local index idle all_idle probe_status
+    local probe_dir=${controller_dir}/idle_probe
+    local -a probe_pids=()
+    mkdir -p "${probe_dir}"
     while true; do
         all_idle=1
+        probe_pids=()
         for index in "${!targets[@]}"; do
-            idle=$(ssh "${ssh_options[@]}" "${targets[${index}]}" \
-                "npu-smi info | grep -c 'No running processes found in NPU' || true")
+            ssh "${ssh_options[@]}" "${targets[${index}]}" \
+                "timeout 25s bash ${remote_idle_probe} --devices ${MODEL_RUNNER_DEVICES_PER_NODE}" \
+                >"${probe_dir}/${index}.out" 2>"${probe_dir}/${index}.err" &
+            probe_pids+=("$!")
+        done
+        for index in "${!targets[@]}"; do
+            probe_status=0
+            wait "${probe_pids[${index}]}" || probe_status=$?
+            idle=$(tail -n 1 "${probe_dir}/${index}.out" 2>/dev/null || true)
+            if [[ "${probe_status}" -ne 0 || ! "${idle}" =~ ^[0-9]+$ ]]; then
+                idle=0
+            fi
             if [[ "${idle}" -ne "${MODEL_RUNNER_DEVICES_PER_NODE}" ]]; then
                 all_idle=0
             fi
diff --git a/tools/moonep/mindspeed/run_model_node.sh b/tools/moonep/mindspeed/run_model_node.sh
index f7784ee..e2b02fc 100755
--- a/tools/moonep/mindspeed/run_model_node.sh
+++ b/tools/moonep/mindspeed/run_model_node.sh
@@ -14,7 +14,9 @@ Required options:
   --install-prefix PATH --cann-env PATH --conda-sh PATH --conda-env NAME
   --native-env PATH --tokenizer-path PATH --data-path PATH --run-tag TAG
 
-Optional: --profile --stage-barrier --timeout SECONDS
+Optional: --profile --stage-barrier --hccl-inter-hccs-disable true|false
+          --rank-table-file PATH --timeout SECONDS
+          --udma-rootinfo-path PATH
 Internal cleanup: --stop --backend BACKEND --node-rank RANK
                   --tilexr-home PATH --run-tag TAG
 EOF
@@ -39,6 +41,9 @@ run_tag=
 timeout_sec=900
 profile=0
 stage_barrier=0
+hccl_inter_hccs_disable=""
+rank_table_file=""
+udma_rootinfo_path=""
 stop=0
 
 while [[ $# -gt 0 ]]; do
@@ -62,12 +67,42 @@ while [[ $# -gt 0 ]]; do
         --timeout) timeout_sec=${2:?--timeout requires a value}; shift 2 ;;
         --profile) profile=1; shift ;;
         --stage-barrier) stage_barrier=1; shift ;;
+        --hccl-inter-hccs-disable)
+            hccl_inter_hccs_disable=${2:?--hccl-inter-hccs-disable requires a value}
+            shift 2
+            ;;
+        --rank-table-file) rank_table_file=${2:?--rank-table-file requires a value}; shift 2 ;;
+        --udma-rootinfo-path)
+            udma_rootinfo_path=${2:?--udma-rootinfo-path requires a value}
+            shift 2
+            ;;
         --stop) stop=1; shift ;;
         -h|--help) usage; exit 0 ;;
         *) printf 'Unknown argument: %s\n' "$1" >&2; usage >&2; exit 2 ;;
     esac
 done
 
+case "${hccl_inter_hccs_disable}" in
+    "") unset HCCL_INTER_HCCS_DISABLE ;;
+    true|false) export HCCL_INTER_HCCS_DISABLE=${hccl_inter_hccs_disable} ;;
+    *) printf 'Invalid HCCL inter-HCCS value: %s\n' "${hccl_inter_hccs_disable}" >&2; exit 2 ;;
+esac
+if [[ -n "${rank_table_file}" ]]; then
+    [[ -f "${rank_table_file}" ]] || { echo "Rank table not found: ${rank_table_file}" >&2; exit 2; }
+    export RANK_TABLE_FILE=${rank_table_file}
+else
+    unset RANK_TABLE_FILE
+fi
+if [[ -n "${udma_rootinfo_path}" ]]; then
+    [[ -f "${udma_rootinfo_path}" ]] || {
+        echo "UDMA RootInfo not found: ${udma_rootinfo_path}" >&2
+        exit 2
+    }
+    export TILEXR_UDMA_ROOTINFO_PATH=${udma_rootinfo_path}
+else
+    unset TILEXR_UDMA_ROOTINFO_PATH
+fi
+
 case "${backend}" in tilexr|native) ;; *) printf 'Invalid backend: %s\n' "${backend}" >&2; exit 2 ;; esac
 if [[ -z "${tilexr_home}" || -z "${run_tag}" || -z "${node_rank}" ]]; then
     echo "--tilexr-home, --run-tag, and --node-rank are required" >&2
@@ -122,11 +157,6 @@ if (( node_count < 1 || node_rank >= node_count || devices_per_node < 1 || \
     echo "invalid distributed launcher dimensions" >&2
     exit 2
 fi
-if [[ "${stage_barrier}" -eq 1 && "${backend}" != tilexr ]]; then
-    echo "stage barriers require the TileXR backend" >&2
-    exit 2
-fi
-
 for path in "${cann_env}" "${conda_sh}" "${native_env}"; do
     [[ -f "${path}" ]] || { printf 'Required file not found: %s\n' "${path}" >&2; exit 1; }
 done
@@ -156,6 +186,8 @@ trap 'exit 130' INT
 trap 'exit 143' TERM HUP
 exec > >(tee "${output}/controller.log") 2>&1
 
+set +u
+# Vendor and conda environment scripts commonly append optional variables.
 # shellcheck disable=SC1090
 source "${cann_env}"
 # shellcheck disable=SC1090
@@ -163,10 +195,13 @@ source "${conda_sh}"
 conda activate "${conda_env}"
 # shellcheck disable=SC1090
 source "${native_env}"
+set -u
 
+idle_probe=${tilexr_home}/tools/moonep/mindspeed/probe_idle.sh
+[[ -f "${idle_probe}" ]] || { echo "Idle probe not found: ${idle_probe}" >&2; exit 1; }
 for gate in 1 2; do
-    npu-smi info >"${output}/npu_gate_${gate}.log"
-    idle=$(grep -c 'No running processes found in NPU' "${output}/npu_gate_${gate}.log" || true)
+    idle=$(bash "${idle_probe}" --devices "${devices_per_node}" \
+        --log "${output}/npu_gate_${gate}.log")
     if [[ "${idle}" -ne "${devices_per_node}" ]]; then
         printf 'exit_code=90\nreason=npu_busy\ngate=%s\nidle=%s\n' "${gate}" "${idle}" \
             | tee "${output}/result.txt"
@@ -184,6 +219,10 @@ export LD_LIBRARY_PATH="${shmem_backend}:${LD_LIBRARY_PATH:-}"
 [[ -f /usr/lib64/libstdc++.so.6 ]] && export LD_PRELOAD=/usr/lib64/libstdc++.so.6
 
 interface=${MODEL_RUNNER_SOCKET_IFNAME:-}
+if [[ -z "${interface}" && "${node_count}" -gt 1 ]] && \
+    ip -o -4 addr show dev data0.3001 2>/dev/null | grep -q .; then
+    interface=data0.3001
+fi
 if [[ -z "${interface}" ]]; then
     interface=$(ip route get "${master_addr}" 2>/dev/null | awk '{for (i=1; i<=NF; ++i) if ($i == "dev") {print $(i+1); exit}}')
 fi
@@ -215,13 +254,21 @@ unset TILEXR_MOONEP_DEBUG_SYNC_COMBINE TILEXR_MOONEP_DEBUG_SYNC_DISPATCH_STATUS
 unset TILEXR_MOONEP_DEBUG_PREFETCH_UDMA TILEXR_MOONEP_DUMP_DFX_ON_ERROR
 unset TILEXR_MOONEP_FLAG_DUMP_DIR TILEXR_MOONEP_FLAG_DUMP_MODE
 unset TILEXR_MINDSPEED_TRACE TILEXR_MINDSPEED_PLAN_DUMP_DIR
+unset MOONEP_MINDSPEED_STAGE_BARRIER TILEXR_MINDSPEED_STAGE_BARRIER
 unset TILEXR_MINDSPEED_FORCE_DUMMY_UDMA ASCEND_LAUNCH_BLOCKING
 unset PROFILING_MODE PROFILING_OPTIONS ASCEND_MOONEP_DISPATCH_ENABLE_DFX
 unset ASCEND_MOONEP_DISPATCH_ENABLE_TRACE ASCEND_MOONEP_DISPATCH_TRACE
+unset TILEXR_MINDSPEED_FINITE_CHECK
 
 world_size=$((node_count * devices_per_node))
 global_batch_size=${world_size}
+ep_size=${world_size}
+base_expert_count=32
+expert_count=$((((base_expert_count + ep_size - 1) / ep_size) * ep_size))
 backend_args=()
+install -m 0644 "${tilexr_home}/tools/moonep/mindspeed/mindspeed_stage_barrier.py" \
+    "${mindspeed_home}/mindspeed/core/transformer/moe/mindspeed_stage_barrier.py"
+export MOONEP_MINDSPEED_STAGE_BARRIER=${stage_barrier}
 if [[ "${backend}" == tilexr ]]; then
     MINDSPEED_HOME=${mindspeed_home} TILEXR_HOME=${tilexr_home} \
         TILEXR_INSTALL_PREFIX=${install_prefix} \
@@ -234,17 +281,21 @@ if [[ "${backend}" == tilexr ]]; then
     export TILEXR_MOONEP_DISPATCH_PEER_MODE=group
     export TILEXR_MOONEP_DISPATCH_GROUP_WIDTH=16
     export TILEXR_MOONEP_COMBINE_VERSION=2
-    export TILEXR_MINDSPEED_STAGE_BARRIER=${stage_barrier}
+    export TILEXR_MOONEP_UDMA_ARENA_RESERVE_BYTES=$((192 * 1024 * 1024))
     unset TILEXR_MOONEP_DISPATCH_TRANSPORT
     export TILEXR_COMM_ID="${master_addr}:$((master_port + 10000))"
-    ep_size=${world_size}
     backend_args+=(
         --moonep-token-padding 1
         --moonep-backend-factory
         mindspeed.core.transformer.moe.tilexr_mindspeed_adapter:create_tilexr_moonep_backend
     )
 else
-    ep_size=${devices_per_node}
+    if [[ "${stage_barrier}" -eq 1 ]]; then
+        backend_args+=(
+            --moonep-backend-factory
+            mindspeed.core.transformer.moe.mindspeed_stage_barrier:create_native_barrier_backend
+        )
+    fi
 fi
 
 profile_output=${output}/profiling
@@ -258,13 +309,28 @@ if [[ "${profile}" -eq 1 ]]; then
     )
 fi
 
+rank_table_sha256=disabled
+if [[ -n "${rank_table_file}" ]]; then
+    rank_table_sha256=$(sha256sum "${rank_table_file}" | awk '{print $1}')
+fi
+udma_rootinfo_sha256=disabled
+if [[ -n "${udma_rootinfo_path}" ]]; then
+    udma_rootinfo_sha256=$(sha256sum "${udma_rootinfo_path}" | awk '{print $1}')
+fi
+
 {
     printf 'backend=%s\nrun_tag=%s\nnode_rank=%s\nworld_size=%s\nrank_per_dev=1\n' \
         "${backend}" "${run_tag}" "${node_rank}" "${world_size}"
     printf 'ep_size=%s\nprofile=%s\nstage_barrier=%s\ninterface=%s\n' \
         "${ep_size}" "${profile}" "${stage_barrier}" "${interface}"
-    printf 'shape=4K/8P layers=4 hidden=7168 experts=32 token_padding=1 iterations=8\n'
-    printf 'debug=disabled flag_dump=disabled plan_dump=disabled\n'
+    printf 'rank_table_file=%s\nrank_table_sha256=%s\n' \
+        "${rank_table_file:-disabled}" "${rank_table_sha256}"
+    printf 'udma_rootinfo_path=%s\nudma_rootinfo_sha256=%s\n' \
+        "${udma_rootinfo_path:-disabled}" "${udma_rootinfo_sha256}"
+    printf 'shape=4K/8P layers=4 hidden=7168 experts=%s token_padding=1 iterations=8\n' \
+        "${expert_count}"
+    printf 'finite_check=%s flag_dump=disabled plan_dump=disabled\n' \
+        "${TILEXR_MINDSPEED_FINITE_CHECK:-disabled}"
     env | grep -E '^(TILEXR_|MOONEP_|ASCEND_MOONEP_|HCCL_|GLOO_SOCKET)' | sort
 } >"${output}/provenance.log"
 
@@ -326,7 +392,7 @@ python -m torch.distributed.launch \
     --qk-layernorm --mla-mm-split --mla-fa-without-pad \
     --moe-grouped-gemm --moe-token-dispatcher-type flex \
     --first-k-dense-replace 0 --moe-enable-moonep --moe-layer-freq 1 \
-    --moe-shared-expert-intermediate-size 2048 --num-experts 32 \
+    --moe-shared-expert-intermediate-size 2048 --num-experts "${expert_count}" \
     --moe-router-topk 8 --moe-ffn-hidden-size 2048 \
     --moe-router-load-balancing-type seq_aux_loss \
     --moe-router-num-groups 8 --moe-router-group-topk 4 \
@@ -361,8 +427,8 @@ finite_final_loss=$(grep -Ec \
     'iteration[[:space:]]+8/[[:space:]]*8.*lm loss:[[:space:]]*[0-9]' \
     "${output}/controller.log" || true)
 profile_done=$(find "${profile_output}" -type f -name analyse.done 2>/dev/null | wc -l || true)
-npu-smi info >"${output}/npu_after.log" || true
-post_idle=$(grep -c 'No running processes found in NPU' "${output}/npu_after.log" || true)
+post_idle=$(bash "${idle_probe}" --devices "${devices_per_node}" \
+    --log "${output}/npu_after.log")
 if [[ "${status}" -eq 0 && ( "${skipped}" -ne 0 || "${nan}" -ne 0 ) ]]; then
     status=92
 fi
@@ -370,6 +436,10 @@ if [[ "${status}" -eq 0 && "${node_count}" -eq 1 && \
       ( "${nonfinite_grad}" -ne 0 || "${finite_final_loss}" -eq 0 ) ]]; then
     status=93
 fi
+if [[ "${status}" -eq 0 && "${profile}" -eq 1 && \
+      "${profile_done}" -ne "${devices_per_node}" ]]; then
+    status=94
+fi
 printf 'exit_code=%s\niterations=%s\nlast_iteration=%s\nskipped_nonzero=%s\nnan_nonzero=%s\nnonfinite_grad=%s\nfinite_final_loss=%s\nprofile_done=%s\npost_idle=%s\ncompleted=%s\n' \
     "${status}" "${iterations}" "${last_iteration}" "${skipped}" "${nan}" \
     "${nonfinite_grad}" "${finite_final_loss}" "${profile_done}" "${post_idle}" \
diff --git a/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py b/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py
index 04e4ccf..b55ad46 100644
--- a/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py
+++ b/tools/moonep/mindspeed/tilexr_mindspeed_adapter.py
@@ -10,6 +10,7 @@
     MOONEP_NATIVE_NPU_CAPABILITY,
     MoonEPBufferFlexBackend,
 )
+from .mindspeed_stage_barrier import optional_stage_barrier
 
 
 _UDMA_COMPAT_REGISTRATION_BYTES = 2 * 1024 * 1024
@@ -64,6 +65,7 @@ def __init__(self, *args, token_buffer_count=1, **kwargs):
         self._tilexr_remote_prefetches = 0
         self._plan_owner_token = object()
         self._dispatch_generation = 0
+        self._finite_check_sequence = 0
         self._ctx = self._require_ctx()
         if os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1":
             print(
@@ -143,8 +145,33 @@ def _dump_native_plan_once(self, plan):
             path,
         )
 
+    def _require_finite(self, stage, plan=None, **tensors):
+        if os.environ.get("TILEXR_MINDSPEED_FINITE_CHECK", "0") != "1":
+            return
+        self._finite_check_sequence += 1
+        epoch = "none" if plan is None else str(plan._require_native().epoch)
+        for name, tensor in tensors.items():
+            if tensor is None:
+                continue
+            checksum = tensor.sum(dtype=self._torch.float32)
+            if bool(self._torch.isfinite(checksum).item()):
+                continue
+            print(
+                "TILEXR_MINDSPEED_FIRST_NONFINITE "
+                f"rank={self._context.planner_group_rank} "
+                f"sequence={self._finite_check_sequence} stage={stage} "
+                f"epoch={epoch} tensor={name} checksum={float(checksum.item())} "
+                f"shape={tuple(tensor.shape)} dtype={tensor.dtype}",
+                flush=True,
+            )
+            raise RuntimeError(
+                f"TileXR MindSpeed detected a non-finite checksum at {stage}.{name}"
+            )
+
     def dispatch(self, *args, hidden_buffer=None, **kwargs):
         self._optional_stage_barrier()
+        hidden_input = args[0] if args else kwargs.get("hidden_sh")
+        self._require_finite("dispatch.input", kwargs.get("plan"), hidden=hidden_input)
         async_finish = bool(kwargs.pop("async_finish", False))
         zero_copy = bool(kwargs.pop("zero_copy", False))
         result = super().dispatch(
@@ -154,6 +181,9 @@ def dispatch(self, *args, hidden_buffer=None, **kwargs):
             **kwargs,
         )
         hidden, route_weights, cu_seqlens, plan = result
+        self._require_finite(
+            "dispatch.output", plan, hidden=hidden, route_weights=route_weights
+        )
         if os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1":
             native_plan = plan._require_native()
             print(
@@ -212,6 +242,18 @@ def _stage_route_weights(self, route_weights, *, hidden_buffer):
 
     def combine(self, *args, hidden_buffer=None, **kwargs):
         self._optional_stage_barrier()
+        plan = kwargs.get("plan") if kwargs.get("plan") is not None else (
+            args[0] if args else None
+        )
+        hidden_input = kwargs.get("hidden_nvsh")
+        route_input = kwargs.get("route_weights_nvs")
+        if hidden_input is None and len(args) > 1:
+            hidden_input = args[1]
+        if route_input is None and len(args) > 2:
+            route_input = args[2]
+        self._require_finite(
+            "combine.input", plan, hidden=hidden_input, route_weights=route_input
+        )
         zero_copy = bool(kwargs.pop("zero_copy", False))
         if hidden_buffer is not None:
             self._validate_boundary(hidden_buffer)
@@ -240,6 +282,12 @@ def combine(self, *args, hidden_buffer=None, **kwargs):
                 )
         try:
             result = super().combine(*args, zero_copy=False, **kwargs)
+            self._require_finite(
+                "combine.output",
+                plan,
+                hidden=result[0],
+                route_weights=result[1],
+            )
             if os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1":
                 try:
                     self._native_buffer.synchronize()
@@ -277,6 +325,11 @@ def _ensure_packed_projections(self, local_fc1, local_fc2):
                 local_fc1,
                 dummy,
                 local_fc2,
+                registration_backing_factory=lambda required_bytes, dtype: (
+                    self._context.promote_projection_arena(
+                        self._torch, dtype, required_bytes
+                    )
+                ),
                 torch_module=self._torch,
             )
             self._native_buffer.register_projection_buffers(projections)
@@ -303,6 +356,9 @@ def _prefetch_weight_packed_batch2(
         del local_slots, source_vas
         full_fc1, full_fc2 = full_weights
         local_fc1, local_fc2 = local_experts
+        self._require_finite(
+            "prefetch.input", plan, local_fc1=local_fc1, local_fc2=local_fc2
+        )
         projections = self._ensure_packed_projections(local_fc1, local_fc2)
         native_plan = plan._require_native()
         trace = os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1"
@@ -344,6 +400,9 @@ def _prefetch_weight_packed_batch2(
             full_fc2[self.E + slot].copy_(projections.down[local + slot])
             if expert // local != rank:
                 active_remote += 1
+        self._require_finite(
+            "prefetch.output", plan, full_fc1=full_fc1, full_fc2=full_fc2
+        )
         self._tilexr_remote_prefetches += active_remote
         if os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1":
             print(
@@ -354,14 +413,7 @@ def _prefetch_weight_packed_batch2(
         return self._record_event() if async_finish else None
 
     def _optional_stage_barrier(self):
-        if os.environ.get("TILEXR_MINDSPEED_STAGE_BARRIER", "0") != "1":
-            return
-        distributed = self._torch.distributed
-        if not distributed.is_available() or not distributed.is_initialized():
-            raise RuntimeError(
-                "TileXR stage barrier requires an initialized process group"
-            )
-        distributed.barrier()
+        optional_stage_barrier(self._torch)
 
     def _ensure_reduce_dummy(self, full_fc1, reduce_fc1):
         dummy_width = 16
@@ -407,6 +459,14 @@ def _reduce_grad_packed_batch2(
         full_fc1, full_fc2 = full_grads
         reduce_fc1, reduce_fc2 = reduce_buffers
         local_fc1, local_fc2 = local_grads
+        self._require_finite(
+            "reduce_grad.input",
+            plan,
+            full_fc1=full_fc1,
+            full_fc2=full_fc2,
+            reduce_fc1=reduce_fc1,
+            reduce_fc2=reduce_fc2,
+        )
         dummy, dummy_reduce = self._ensure_reduce_dummy(full_fc1, reduce_fc1)
         trace = os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1"
         native_plan = plan._require_native()
@@ -445,6 +505,14 @@ def _reduce_grad_packed_batch2(
         begin = self._context.planner_group_rank * local
         local_fc1.copy_(full_fc1.narrow(0, begin, local))
         local_fc2.copy_(full_fc2.narrow(0, begin, local))
+        self._require_finite(
+            "reduce_grad.output",
+            plan,
+            full_fc1=full_fc1,
+            full_fc2=full_fc2,
+            local_fc1=local_fc1,
+            local_fc2=local_fc2,
+        )
         if os.environ.get("TILEXR_MINDSPEED_TRACE", "0") == "1":
             print(
                 f"[TileXR MindSpeed rank {self._context.planner_group_rank}] "