feat(moonep): support 128-rank MindSpeed runs

docs/moonep/MINDSPEED_DEBUGGING_EXPERIENCE.md

@@ -189,9 +189,25 @@ reset 等单一变量。一次同时修改 Kernel、Host、timeout 和路由，
 - 不要用 toy shape 证明生产规模的 UB/SQ 容量安全。
 - 不要依赖已消费 SQE 的内容推导 CQ completion。
 - 不要在调试运行和性能运行之间复用未审计的环境变量。
+- 多机 HCCL rank table 与 TileXR UDMA RootInfo 是两种不同配置：前者描述全局
+  rank/topology，后者描述每台主机的本地 UDMA EID/port。不能用一个文件替代另一个；
+  应分别保存路径、哈希和 parser/init 证据。
+- 多机 launcher 全部退出 0 只证明进程完成，不证明模型数值正确。至少同时门禁完整迭代数、
+  最终有限 loss、全程有限 gradient norm、skip/NaN 计数、profiler artifact 数和退出后 idle。
+- 每次实机 A/B 必须同时保存源码 commit/diff 与实际加载 `.so` 路径和哈希。源码同步成功
+  不代表远端安装树已重编译；如果重新构建后行为变化而代码修复尚未被单变量证明，应明确
+  记录为 binary-provenance 问题，不能把诊断插桩本身误报成根因修复。
 - 不要把小 Tensor view 的逻辑字节数等同于 HCCP MR 范围；扩大 backing storage 时保持逻辑 shape 不变，否则会污染算子工作量和性能数据。
 - 不要因某次补丁通过完整模型就跳过最小 reproducer；最小 reproducer 才能证明因果。
 - 不要删除被推翻的假设记录。保留否定证据可以防止后续重复猜测。
+- 多阶段共享普通 UDMA 注册时，不能在热路径中按 stage 反复切换单一 active MR；不同
+  rank 的 stage 到达顺序可能不同，使无标签注册 collective 串轮。可预测的工作区应合并
+  到一个持久注册 arena，算子继续使用各自逻辑子区间；子区间激活必须命中已有 MR，不能
+  再发起注册 collective。扩大 arena 后仍要分别验证初始化内存峰值和至少两轮模型数值。
+- `npu-smi info` 在 950 上可能显示每卡无进程，但 HCCL Test 或其他加速器工作负载仍在
+  使用设备。多机 idle gate 必须同时检查设备节点 owner 和已知加速器进程；发现外部作业
+  只判 busy，不终止。连续模型启动后的 `EI0007/halSqCqAllocate` 表示驱动 stream/SQ-CQ
+  资源尚未回收，应等待并用最小 stream 探针确认恢复，不能改算子代码规避。
 
 ## Combine V2 共享 scratch 的 completion 约束
 

integrations/moonep_torch/tilexr_moonep/runtime.py

@@ -592,7 +592,14 @@ def _activate_udma_region(
             return None
         if pointer <= 0 or size <= 0:
             raise ValueError("UDMA registration requires a valid pointer and positive size")
-        if self._active_udma_pointer == pointer and self._active_udma_bytes == size:
+        active_end = self._active_udma_pointer + self._active_udma_bytes
+        requested_end = pointer + size
+        if (
+            self._active_udma_pointer > 0
+            and pointer >= self._active_udma_pointer
+            and requested_end >= pointer
+            and requested_end <= active_end
+        ):
             self._active_udma_owner = owner
             return self._active_udma_handle
         handle = ctypes.c_uint32()

integrations/moonep_torch/tilexr_moonep/torch_api.py

@@ -124,6 +124,8 @@ def from_local_weights(
         down,
         *,
         slot_fill_value: float = 0.0,
+        registration_backing=None,
+        registration_backing_factory=None,
         torch_module=None,
     ) -> "ProjectionBuffers":
         torch_module = torch_module or _torch()
@@ -165,20 +167,47 @@ def from_local_weights(
             (cursor_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1) //
             _UDMA_REGISTRATION_ALIGNMENT * _UDMA_REGISTRATION_ALIGNMENT
         )
-        allocation_elements = (
-            registered_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1 + element_bytes - 1
-        ) // element_bytes
-        allocation = torch_module.empty(
-            (allocation_elements,), dtype=gate.dtype,
-            device=f"npu:{context.device_index}"
-        )
-        allocation_ptr = int(allocation.data_ptr())
-        offset_bytes = (-allocation_ptr) % _UDMA_REGISTRATION_ALIGNMENT
-        if offset_bytes % element_bytes:
-            raise RuntimeError("projection registration offset is not element-aligned")
-        backing = allocation.narrow(
-            0, offset_bytes // element_bytes, registered_bytes // element_bytes
-        )
+        if registration_backing is not None and registration_backing_factory is not None:
+            raise ValueError(
+                "registration_backing and registration_backing_factory are mutually exclusive"
+            )
+        if registration_backing_factory is not None:
+            registration_backing = registration_backing_factory(
+                registered_bytes, gate.dtype
+            )
+        if registration_backing is None:
+            allocation_elements = (
+                registered_bytes + _UDMA_REGISTRATION_ALIGNMENT - 1 + element_bytes - 1
+            ) // element_bytes
+            allocation = torch_module.empty(
+                (allocation_elements,), dtype=gate.dtype,
+                device=f"npu:{context.device_index}"
+            )
+            allocation_ptr = int(allocation.data_ptr())
+            offset_bytes = (-allocation_ptr) % _UDMA_REGISTRATION_ALIGNMENT
+            if offset_bytes % element_bytes:
+                raise RuntimeError("projection registration offset is not element-aligned")
+            backing = allocation.narrow(
+                0, offset_bytes // element_bytes, registered_bytes // element_bytes
+            )
+        else:
+            _validate_tensor(
+                torch_module,
+                registration_backing,
+                "registration_backing",
+                dtype=gate.dtype,
+                device_index=context.device_index,
+                allow_storage_offset=True,
+            )
+            if int(registration_backing.data_ptr()) % _UDMA_REGISTRATION_ALIGNMENT:
+                raise ValueError("registration_backing must be 2-MiB aligned")
+            if int(registration_backing.numel()) * element_bytes < registered_bytes:
+                raise ValueError(
+                    "registration_backing is smaller than the projection layout"
+                )
+            backing = registration_backing.narrow(
+                0, 0, registered_bytes // element_bytes
+            )
         if (int(backing.data_ptr()) % _UDMA_REGISTRATION_ALIGNMENT or
                 int(backing.numel()) * element_bytes != registered_bytes):
             raise RuntimeError("projection backing is not 2-MiB registration-aligned")
@@ -288,6 +317,7 @@ class TileXRMoonEPContext:
     _dispatch_workspace_ptr: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_bytes: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_alignment: int = field(init=False, default=0, repr=False)
+    _dispatch_registration_bytes: int = field(init=False, default=0, repr=False)
     _dispatch_workspace_handle: int | None = field(init=False, default=None, repr=False)
     _buffer_owner: Any = field(init=False, default=None, repr=False)
     _bound_stream_ptr: int | None = field(init=False, default=None, repr=False)
@@ -410,6 +440,7 @@ def close(self) -> None:
         self._dispatch_workspace_handle = None
         self._dispatch_workspace_ptr = 0
         self._dispatch_workspace_bytes = 0
+        self._dispatch_registration_bytes = 0
         self._dispatch_workspace_owner = None
         self.runtime.close()
         self._closed = True
@@ -454,8 +485,9 @@ def ensure_dispatch_workspace(self, torch_module) -> None:
                 "native Dispatch returned invalid workspace contract "
                 f"bytes={workspace_bytes} alignment={alignment}"
             )
+        registration_bytes = workspace_bytes
         raw = torch_module.empty(
-            (workspace_bytes + alignment - 1,),
+            (registration_bytes + alignment - 1,),
             dtype=torch_module.uint8,
             device=f"npu:{self.device_index}",
         )
@@ -464,19 +496,77 @@ def ensure_dispatch_workspace(self, torch_module) -> None:
         aligned_ptr = ((raw_ptr + alignment - 1) // alignment) * alignment
         if aligned_ptr + workspace_bytes > raw_ptr + int(raw.numel()):
             raise RuntimeError("aligned Dispatch workspace exceeds its raw allocation")
-        handle = self.runtime.register_dispatch_workspace(aligned_ptr, workspace_bytes)
+        handle = self.runtime.register_dispatch_workspace(aligned_ptr, registration_bytes)
         self._dispatch_workspace_owner = raw
         self._dispatch_workspace_ptr = aligned_ptr
         self._dispatch_workspace_bytes = workspace_bytes
         self._dispatch_workspace_alignment = alignment
+        self._dispatch_registration_bytes = registration_bytes
         self._dispatch_workspace_handle = handle
 
     def activate_dispatch_workspace(self) -> None:
         if self._dispatch_workspace_owner is None:
             raise RuntimeError("Dispatch workspace is not initialized")
         self._dispatch_workspace_handle = self.runtime.register_dispatch_workspace(
-            self._dispatch_workspace_ptr, self._dispatch_workspace_bytes
+            self._dispatch_workspace_ptr, self._dispatch_registration_bytes
+        )
+
+    def promote_projection_arena(self, torch_module, dtype, required_bytes: int):
+        if required_bytes <= 0 or required_bytes % self._dispatch_workspace_alignment:
+            raise ValueError(
+                "projection arena size must be a positive multiple of the UDMA alignment"
+            )
+        reserve_text = os.environ.get("TILEXR_MOONEP_UDMA_ARENA_RESERVE_BYTES", "0")
+        try:
+            reserve_bytes = int(reserve_text, 0)
+        except ValueError as exc:
+            raise ValueError(
+                "TILEXR_MOONEP_UDMA_ARENA_RESERVE_BYTES must be an integer"
+            ) from exc
+        if reserve_bytes <= 0:
+            return None
+        if required_bytes > reserve_bytes:
+            raise RuntimeError(
+                "projection backing exceeds the configured UDMA arena reserve: "
+                f"required={required_bytes} reserve={reserve_bytes}"
+            )
+        if self._dispatch_registration_bytes == self._dispatch_workspace_bytes:
+            registration_bytes = self._dispatch_workspace_bytes + required_bytes
+            alignment = self._dispatch_workspace_alignment
+            raw = torch_module.empty(
+                (registration_bytes + alignment - 1,),
+                dtype=torch_module.uint8,
+                device=f"npu:{self.device_index}",
+            )
+            raw_ptr = int(raw.data_ptr())
+            aligned_ptr = ((raw_ptr + alignment - 1) // alignment) * alignment
+            old_offset = self._dispatch_workspace_ptr - int(
+                self._dispatch_workspace_owner.data_ptr()
+            )
+            new_offset = aligned_ptr - raw_ptr
+            raw.narrow(0, new_offset, self._dispatch_workspace_bytes).copy_(
+                self._dispatch_workspace_owner.narrow(
+                    0, old_offset, self._dispatch_workspace_bytes
+                )
+            )
+            torch_module.npu.synchronize(device=self.device_index)
+            handle = self.runtime.register_dispatch_workspace(
+                aligned_ptr, registration_bytes
+            )
+            self._dispatch_workspace_owner = raw
+            self._dispatch_workspace_ptr = aligned_ptr
+            self._dispatch_registration_bytes = registration_bytes
+            self._dispatch_workspace_handle = handle
+        available = self._dispatch_registration_bytes - self._dispatch_workspace_bytes
+        if required_bytes > available:
+            raise RuntimeError("projection arena was already promoted with a smaller layout")
+        raw_offset = self._dispatch_workspace_ptr - int(
+            self._dispatch_workspace_owner.data_ptr()
+        )
+        byte_view = self._dispatch_workspace_owner.narrow(
+            0, raw_offset + self._dispatch_workspace_bytes, required_bytes
         )
+        return byte_view.view(dtype)
 
     @property
     def dispatch_workspace(self) -> tuple[int, int]:

tests/moonep/python/fakes.py

@@ -87,6 +87,29 @@ def reshape(self, *shape):
         result.zero_calls = self.zero_calls
         return result
 
+    def view(self, *args):
+        if len(args) == 1 and isinstance(args[0], str):
+            dtype = args[0]
+            source_bytes = self.numel() * self.element_size()
+            target_bytes = {
+                "uint8": 1,
+                "float16": 2,
+                "bfloat16": 2,
+                "int32": 4,
+                "float32": 4,
+                "int64": 8,
+            }[dtype]
+            if source_bytes % target_bytes:
+                raise ValueError("FakeTensor.view dtype has incompatible byte size")
+            result = FakeTensor(
+                (source_bytes // target_bytes,), dtype, self.device,
+                storage_offset=self._storage_offset,
+            )
+            result._ptr = self._ptr
+            result._base = getattr(self, "_base", self)
+            return result
+        return self.reshape(*args)
+
     def narrow(self, dim, start, length):
         dim = int(dim)
         start = int(start)

tests/moonep/python/test_ffi_unittest.py

@@ -425,6 +425,31 @@ def tensor(shape, dtype):
 
 
 class FfiAbiTests(unittest.TestCase):
+    def test_udma_arena_subregion_reuses_active_registration(self):
+        loader = FakeCDLLLoader()
+        runtime = TileXRMoonEPRuntime(
+            rank=0,
+            world_size=2,
+            library_paths={
+                "comm": "libtile-comm.so",
+                "planner": "libtilexr-moonep-planner.so",
+                "combine_v2": "libtilexr-moonep-combine-v2.so.2",
+                "moonep": "libtilexr-moonep.so.1",
+            },
+            cdll_loader=loader,
+        )
+
+        handle = runtime._activate_udma_region(
+            0x200000, 2 * 1024 * 1024, "dispatch", "arena"
+        )
+        reused = runtime._activate_udma_region(
+            0x300000, 1024 * 1024, "projection", "subregion"
+        )
+
+        self.assertEqual(reused, handle)
+        self.assertEqual(loader.register_calls, [(0x200000, 2 * 1024 * 1024)])
+        runtime.close()
+
     def test_reduce_grad_registration_uses_storage_without_expanding_source(self):
         backing = FakeTensor((2 * 1024 * 1024 // 4,), "float32")
         backing._ptr = 0x200000