From 217dbddb85c1c97adad7494b07c5b6e1b227eb02 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Mon, 6 Jul 2026 16:17:36 +0800 Subject: [PATCH 01/41] feat: add direct CCU runtime support --- CMakeLists.txt | 1 - docs/CCU_ASCENDC_HANDOFF.md | 222 + src/comm/CMakeLists.txt | 44 + src/comm/ccu/tilexr_ccu_abi_constants.h | 18 + src/comm/ccu/tilexr_ccu_barrier_program.cpp | 273 ++ src/comm/ccu/tilexr_ccu_barrier_program.h | 56 + .../ccu/tilexr_ccu_direct_orchestrator.cpp | 1199 ++++++ src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 137 + src/comm/ccu/tilexr_ccu_direct_runtime.cpp | 1529 +++++++ src/comm/ccu/tilexr_ccu_direct_runtime.h | 152 + src/comm/ccu/tilexr_ccu_driver_adapter.cpp | 524 +++ src/comm/ccu/tilexr_ccu_driver_adapter.h | 147 + src/comm/ccu/tilexr_ccu_hccp_loader.cpp | 581 +++ src/comm/ccu/tilexr_ccu_hccp_loader.h | 109 + src/comm/ccu/tilexr_ccu_hccp_types.h | 584 +++ src/comm/ccu/tilexr_ccu_install_provider.cpp | 1534 +++++++ src/comm/ccu/tilexr_ccu_install_provider.h | 183 + src/comm/ccu/tilexr_ccu_launch_package.cpp | 270 ++ src/comm/ccu/tilexr_ccu_launch_package.h | 55 + .../ccu/tilexr_ccu_lower_layer_payloads.cpp | 172 + .../ccu/tilexr_ccu_lower_layer_payloads.h | 67 + .../tilexr_ccu_lower_layer_plan_builder.cpp | 671 +++ .../ccu/tilexr_ccu_lower_layer_plan_builder.h | 182 + src/comm/ccu/tilexr_ccu_memory_program.cpp | 203 + src/comm/ccu/tilexr_ccu_memory_program.h | 59 + src/comm/ccu/tilexr_ccu_microcode.cpp | 298 ++ src/comm/ccu/tilexr_ccu_microcode.h | 100 + src/comm/ccu/tilexr_ccu_producer_plan.cpp | 456 ++ src/comm/ccu/tilexr_ccu_producer_plan.h | 99 + src/comm/ccu/tilexr_ccu_provider.cpp | 401 ++ src/comm/ccu/tilexr_ccu_provider.h | 87 + .../tilexr_ccu_ra_custom_channel_provider.cpp | 112 + .../tilexr_ccu_ra_custom_channel_provider.h | 82 + src/comm/ccu/tilexr_ccu_repository.cpp | 894 ++++ src/comm/ccu/tilexr_ccu_repository.h | 132 + .../ccu/tilexr_ccu_resource_allocator.cpp | 377 ++ src/comm/ccu/tilexr_ccu_resource_allocator.h | 130 + src/comm/ccu/tilexr_ccu_runtime.cpp | 97 + src/comm/ccu/tilexr_ccu_runtime.h | 47 + src/comm/ccu/tilexr_ccu_specs.cpp | 178 + src/comm/ccu/tilexr_ccu_specs.h | 91 + src/comm/comm_wrap.cpp | 448 ++ src/comm/tilexr_comm.cpp | 1057 ++++- src/comm/tilexr_comm.h | 81 + src/comm/tilexr_internal.cpp | 27 +- src/include/tilexr_api.h | 150 +- tests/ccu/CMakeLists.txt | 10 + .../ccu_lower_layer_payload_hcomm_oracle.cpp | 348 ++ tests/ccu/ccu_npu_smi_busy_guard.py | 136 + .../ccu/ccu_public_direct_api_compile_probe.c | 59 + tests/ccu/ccu_tilexr_basic_info_probe.cpp | 124 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 1261 ++++++ tests/ccu/check_tile_comm_no_hcomm_deps.sh | 50 + .../run_tilexr_ccu_direct_evening_smoke.sh | 538 +++ tests/ccu/run_tilexr_ccu_direct_smoke.sh | 609 +++ tests/ccu/test_tilexr_ccu_barrier_program.py | 649 +++ tests/ccu/test_tilexr_ccu_basic_info_probe.py | 138 + .../test_tilexr_ccu_direct_orchestrator.py | 1724 ++++++++ .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 596 +++ .../test_tilexr_ccu_direct_smoke_runner.py | 1399 ++++++ tests/ccu/test_tilexr_ccu_driver_adapter.py | 992 +++++ tests/ccu/test_tilexr_ccu_install_provider.py | 3334 +++++++++++++++ tests/ccu/test_tilexr_ccu_launch_package.py | 501 +++ .../test_tilexr_ccu_lower_layer_payloads.py | 304 ++ ...r_ccu_lower_layer_payloads_hcomm_oracle.py | 73 + ...est_tilexr_ccu_lower_layer_plan_builder.py | 3770 +++++++++++++++++ tests/ccu/test_tilexr_ccu_memory_program.py | 315 ++ tests/ccu/test_tilexr_ccu_microcode.py | 572 +++ tests/ccu/test_tilexr_ccu_producer_plan.py | 461 ++ tests/ccu/test_tilexr_ccu_provider.py | 1618 +++++++ ...est_tilexr_ccu_public_api_compile_probe.py | 93 + tests/ccu/test_tilexr_ccu_public_comm_api.py | 169 + ...est_tilexr_ccu_ra_custom_channel_loader.py | 2110 +++++++++ ...t_tilexr_ccu_ra_custom_channel_provider.py | 263 ++ tests/ccu/test_tilexr_ccu_repository.py | 1018 +++++ .../ccu/test_tilexr_ccu_resource_allocator.py | 917 ++++ tests/ccu/test_tilexr_ccu_runtime_boundary.py | 114 + .../test_tilexr_ccu_runtime_fake_launch.py | 679 +++ .../test_tilexr_ccu_source_guard_coverage.py | 125 + tests/ccu/test_tilexr_ccu_specs.py | 224 + tests/comm/unit/test_tilexr_source_guards.cpp | 765 +++- 81 files changed, 40362 insertions(+), 12 deletions(-) create mode 100644 docs/CCU_ASCENDC_HANDOFF.md create mode 100644 src/comm/ccu/tilexr_ccu_abi_constants.h create mode 100644 src/comm/ccu/tilexr_ccu_barrier_program.cpp create mode 100644 src/comm/ccu/tilexr_ccu_barrier_program.h create mode 100644 src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp create mode 100644 src/comm/ccu/tilexr_ccu_direct_orchestrator.h create mode 100644 src/comm/ccu/tilexr_ccu_direct_runtime.cpp create mode 100644 src/comm/ccu/tilexr_ccu_direct_runtime.h create mode 100644 src/comm/ccu/tilexr_ccu_driver_adapter.cpp create mode 100644 src/comm/ccu/tilexr_ccu_driver_adapter.h create mode 100644 src/comm/ccu/tilexr_ccu_hccp_loader.cpp create mode 100644 src/comm/ccu/tilexr_ccu_hccp_loader.h create mode 100644 src/comm/ccu/tilexr_ccu_hccp_types.h create mode 100644 src/comm/ccu/tilexr_ccu_install_provider.cpp create mode 100644 src/comm/ccu/tilexr_ccu_install_provider.h create mode 100644 src/comm/ccu/tilexr_ccu_launch_package.cpp create mode 100644 src/comm/ccu/tilexr_ccu_launch_package.h create mode 100644 src/comm/ccu/tilexr_ccu_lower_layer_payloads.cpp create mode 100644 src/comm/ccu/tilexr_ccu_lower_layer_payloads.h create mode 100644 src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp create mode 100644 src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h create mode 100644 src/comm/ccu/tilexr_ccu_memory_program.cpp create mode 100644 src/comm/ccu/tilexr_ccu_memory_program.h create mode 100644 src/comm/ccu/tilexr_ccu_microcode.cpp create mode 100644 src/comm/ccu/tilexr_ccu_microcode.h create mode 100644 src/comm/ccu/tilexr_ccu_producer_plan.cpp create mode 100644 src/comm/ccu/tilexr_ccu_producer_plan.h create mode 100644 src/comm/ccu/tilexr_ccu_provider.cpp create mode 100644 src/comm/ccu/tilexr_ccu_provider.h create mode 100644 src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp create mode 100644 src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.h create mode 100644 src/comm/ccu/tilexr_ccu_repository.cpp create mode 100644 src/comm/ccu/tilexr_ccu_repository.h create mode 100644 src/comm/ccu/tilexr_ccu_resource_allocator.cpp create mode 100644 src/comm/ccu/tilexr_ccu_resource_allocator.h create mode 100644 src/comm/ccu/tilexr_ccu_runtime.cpp create mode 100644 src/comm/ccu/tilexr_ccu_runtime.h create mode 100644 src/comm/ccu/tilexr_ccu_specs.cpp create mode 100644 src/comm/ccu/tilexr_ccu_specs.h create mode 100644 tests/ccu/CMakeLists.txt create mode 100644 tests/ccu/ccu_lower_layer_payload_hcomm_oracle.cpp create mode 100644 tests/ccu/ccu_npu_smi_busy_guard.py create mode 100644 tests/ccu/ccu_public_direct_api_compile_probe.c create mode 100644 tests/ccu/ccu_tilexr_basic_info_probe.cpp create mode 100644 tests/ccu/ccu_tilexr_direct_smoke_probe.cpp create mode 100644 tests/ccu/check_tile_comm_no_hcomm_deps.sh create mode 100644 tests/ccu/run_tilexr_ccu_direct_evening_smoke.sh create mode 100644 tests/ccu/run_tilexr_ccu_direct_smoke.sh create mode 100644 tests/ccu/test_tilexr_ccu_barrier_program.py create mode 100644 tests/ccu/test_tilexr_ccu_basic_info_probe.py create mode 100644 tests/ccu/test_tilexr_ccu_direct_orchestrator.py create mode 100644 tests/ccu/test_tilexr_ccu_direct_smoke_probe.py create mode 100644 tests/ccu/test_tilexr_ccu_direct_smoke_runner.py create mode 100644 tests/ccu/test_tilexr_ccu_driver_adapter.py create mode 100644 tests/ccu/test_tilexr_ccu_install_provider.py create mode 100644 tests/ccu/test_tilexr_ccu_launch_package.py create mode 100644 tests/ccu/test_tilexr_ccu_lower_layer_payloads.py create mode 100644 tests/ccu/test_tilexr_ccu_lower_layer_payloads_hcomm_oracle.py create mode 100644 tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py create mode 100644 tests/ccu/test_tilexr_ccu_memory_program.py create mode 100644 tests/ccu/test_tilexr_ccu_microcode.py create mode 100644 tests/ccu/test_tilexr_ccu_producer_plan.py create mode 100644 tests/ccu/test_tilexr_ccu_provider.py create mode 100644 tests/ccu/test_tilexr_ccu_public_api_compile_probe.py create mode 100644 tests/ccu/test_tilexr_ccu_public_comm_api.py create mode 100644 tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py create mode 100644 tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py create mode 100644 tests/ccu/test_tilexr_ccu_repository.py create mode 100644 tests/ccu/test_tilexr_ccu_resource_allocator.py create mode 100644 tests/ccu/test_tilexr_ccu_runtime_boundary.py create mode 100644 tests/ccu/test_tilexr_ccu_runtime_fake_launch.py create mode 100644 tests/ccu/test_tilexr_ccu_source_guard_coverage.py create mode 100644 tests/ccu/test_tilexr_ccu_specs.py diff --git a/CMakeLists.txt b/CMakeLists.txt index d8e33b5e..7606a98e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,7 +55,6 @@ include_directories( ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/profiling/ ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/runtime/ ${ASCEND_HOME_PATH}/${ARCH}-linux/include/ - ${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl/ ${ASCEND_HOME_PATH}/${ARCH}-linux/include/experiment ${ASCEND_HOME_PATH}/${ARCH}-linux/include/experiment/runtime/ ${ASCEND_HOME_PATH}/${ARCH}-linux/include/experiment/msprof/ diff --git a/docs/CCU_ASCENDC_HANDOFF.md b/docs/CCU_ASCENDC_HANDOFF.md new file mode 100644 index 00000000..b38f699f --- /dev/null +++ b/docs/CCU_ASCENDC_HANDOFF.md @@ -0,0 +1,222 @@ +# TileXR Direct CCU Handoff + +This document is the maintained handoff for TileXR-owned direct CCU support. +It replaces the earlier investigation log. Keep it factual and update it only +when the production path, public API, or validation gate changes. + +## Current Status + +TileXR can prepare, install, submit, and validate a two-rank direct CCU flow +without a production dependency on hcomm, HCCL private CCU producers, or +`libmc2_client.so`. + +Validated path: + +- Direct CCU repository and mission installation. +- Mission/key/taskInfo generation for `rtCCULaunch`. +- XN, CKE, channel, PFE, and jetty lower-layer resource installation. +- Barrier smoke through CCU synchronization instructions. +- P2P data-plane validation through CCU memory-copy microcode. + +Completion evidence from the 950 validation server: + +```text +tilexr_ccu_direct_evening_smoke finalStatus prepare=pass submit=pass barrier=pass p2p=pass +tilexr_ccu_direct_smoke p2pCcuCopy ... mismatches=0 ... passed=1 +TileXRDirectCcuTrace program.sync[5] decoded=TransRmtMemToLocMem ... +TileXR CCU dependency guard passed: no hcomm/HCCL private CCU dependency or symbol reference +``` + +## Production Boundary + +`src/comm` must not link or include hcomm or HCCL private CCU producer APIs. +The allowed production dependency surface is CANN runtime/ACL plus TileXR-owned +CCU code under `src/comm/ccu`. + +Keep these out of production code: + +- `libhcomm.so`, `libhccl_v2.so`, `libhccl_fwk.so`, `libmc2_client.so` +- `HcclGetCcuTaskInfo`, `HcomGetCcuTaskInfo` +- hcomm private resource allocators, repositories, and channel abstractions +- AscendC kernel-side `Hccl` integration until TileXR has + a stable TileXR-owned context producer for that exact ABI + +Run the dependency guard after every CCU production edit: + +```bash +source scripts/common_env.sh +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh build/src/comm/libtile-comm.so +``` + +## Architecture + +Main modules: + +- `tilexr_ccu_direct_runtime.*`: owns direct runtime interaction and basic CCU + information discovery. +- `tilexr_ccu_driver_adapter.*`: wraps the low-level custom-channel driver + operations used for CCU resource and repository installation. +- `tilexr_ccu_hccp_loader.*`: resolves TileXR-owned HCCP/RA entry points. +- `tilexr_ccu_install_provider.*`: installs lower-layer CCU resources, + repository images, and missions. +- `tilexr_ccu_repository.*`: builds and uploads the instruction repository. +- `tilexr_ccu_resource_allocator.*`: reserves mission, instruction, XN, CKE, + GSA, and channel ranges. +- `tilexr_ccu_producer_plan.*`: builds the direct CCU program and submit tasks. +- `tilexr_ccu_barrier_program.*`: emits synchronization microcode. +- `tilexr_ccu_memory_program.*`: emits CCU memory-copy microcode. +- `tilexr_ccu_direct_orchestrator.*`: joins allocation, lower-layer planning, + repository installation, mission installation, and launch-package creation. + +Host integration: + +- `TileXRComm::PrepareDirectCcuInstallAttempt(...)` prepares generic direct CCU + tasks. +- `TileXRComm::PrepareDirectCcuMemoryCopyInstallAttempt(...)` prepares P2P CCU + copy tasks. +- `TileXRCommPrepareDirectCcu(...)` and + `TileXRCommPrepareDirectCcuMemoryCopy(...)` expose the public C API. + +Runtime launch: + +- The final launch path uses `TileXRDirectCcuSubmitPrepared(...)` or + `TileXRDirectCcuSubmitPreparedTask(...)`. +- `rtCCULaunch` is reached through `libruntime.so`; no hcomm launch wrapper is + required. + +## Public API + +`src/include/tilexr_api.h` exposes: + +```c +int TileXRCommPrepareDirectCcu( + TileXRCommPtr comm, + const TileXRDirectCcuPrepareOptions* options, + TileXRDirectCcuPreparedTasksPtr* prepared, + TileXRDirectCcuPrepareReport* report); + +int TileXRCommPrepareDirectCcuMemoryCopy( + TileXRCommPtr comm, + const TileXRDirectCcuMemoryCopyPrepareOptions* options, + TileXRDirectCcuPreparedTasksPtr* prepared, + TileXRDirectCcuPrepareReport* report); + +int TileXRDirectCcuSubmitPrepared( + TileXRDirectCcuPreparedTasksPtr prepared, + aclrtStream stream, + TileXRDirectCcuSubmitReport* report); + +int TileXRDirectCcuSubmitPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t taskIndex, + aclrtStream stream, + TileXRDirectCcuSubmitReport* report); + +int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared); +``` + +Memory-copy direction constants: + +```c +TILEXR_DIRECT_CCU_MEMORY_COPY_REMOTE_TO_LOCAL +TILEXR_DIRECT_CCU_MEMORY_COPY_LOCAL_TO_REMOTE +``` + +The direct P2P smoke currently validates remote-to-local transfer on both ranks. + +## Key Lessons + +- The AscendC CCU `HcclCombineOpParam` route is not a validated TileXR + integration path. Public MC2 allocation probes returned AICPU/MC2 context + shapes, not the CCU context consumed by `HCCL_SERVER_TYPE_CCU`. +- `rtCcuTaskInfo_t` and `rtCCULaunch` are reachable through CANN runtime, but + the hard part is producing the repository, mission key, lower-layer resources, + and task fields correctly. +- Repository installation requires the RA context resource-window route. A zero + or synthetic resource-window token can let lower-layer calls appear partially + valid while `SET_INSTRUCTION` still fails. +- The passing lower-layer route uses hcomm-compatible semantics without linking + hcomm: reverse endpoint EID and imported peer TPN are important. +- P2P validation must use CCU copy microcode. The old marker/IPC harness did not + prove the direct CCU data plane and has been removed from the maintained smoke. +- Process bring-up overrides for runtime task fields, submit-task arguments, + peer remote-XN/CKE bindings, and hcomm-trace resource remapping were removed + from maintained code. The formal path relies on exchanged TileXR resource + evidence. +- CCU direct work is independent of UDMA. Do not use UDMA code as a reference + for CCU resource installation or data movement. +- CANN 9.1 runtime declares `rtUbDevQueryInfo` in + `runtime/rts/rts_device.h`; the `tile-comm` target must include + `${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/runtime/runtime/`. + +## Maintained Validation + +Local source/unit checks: + +```bash +python -m unittest \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_public_comm_api \ + tests.ccu.test_tilexr_ccu_direct_smoke_runner \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_memory_program \ + tests.ccu.test_tilexr_ccu_microcode +``` + +Broader CCU checks: + +```bash +python -m unittest discover tests/ccu +``` + +Remote build and no-hcomm guard: + +```bash +source scripts/common_env.sh +cmake --build build --target tile-comm -j2 +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh build/src/comm/libtile-comm.so +``` + +Hardware smoke: + +```bash +source scripts/common_env.sh +env \ + TILEXR_CCU_SMOKE_DEVICES=0,1 \ + TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU=1 \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=ra_ctx \ + TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES=prepare,submit,barrier,p2p \ + TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES='ra_ctx_full:acl:full_repository:instruction_bytes:lower_layer_first' \ + TILEXR_CCU_EVENING_TOTAL_TIMEOUT=180 \ + bash tests/ccu/run_tilexr_ccu_direct_evening_smoke.sh +``` + +Expected final line: + +```text +tilexr_ccu_direct_evening_smoke finalStatus prepare=pass submit=pass barrier=pass p2p=pass +``` + +Expected P2P evidence in rank logs: + +```text +tilexr_ccu_direct_smoke p2pCcuCopy ... mismatches=0 ... passed=1 +decoded=TransRmtMemToLocMem +``` + +## Development Rules + +- Keep direct CCU code under `src/comm/ccu` and public API declarations under + `src/include/tilexr_api.h`. +- Keep hardware smoke scripts default-safe; they must require explicit env + opt-in before touching NPU devices. +- Do not reintroduce marker-based P2P success criteria. It does not prove CCU + transfer. +- Do not add runtime env overrides that mutate prepared task fields, prepared + task arguments, or peer binding proof. They make the smoke hard to interpret + and are not a production integration path. +- Keep verbose CCU instruction/resource traces behind env gates. They are useful + for hardware failures, but should not be required for normal validation. +- Prefer small unit tests for microcode, repository layout, resource allocation, + and submit-task packaging. Use hardware smoke only for the final device-plane + proof. diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index fe13647a..91c436fd 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -105,6 +105,44 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp tilexr_internal.cpp tilexr_internal.h tilexr_log.h + ccu/tilexr_ccu_abi_constants.h + ccu/tilexr_ccu_barrier_program.h + ccu/tilexr_ccu_barrier_program.cpp + ccu/tilexr_ccu_direct_orchestrator.h + ccu/tilexr_ccu_direct_orchestrator.cpp + ccu/tilexr_ccu_direct_runtime.h + ccu/tilexr_ccu_direct_runtime.cpp + ccu/tilexr_ccu_driver_adapter.h + ccu/tilexr_ccu_driver_adapter.cpp + ccu/tilexr_ccu_hccp_types.h + ccu/tilexr_ccu_hccp_loader.h + ccu/tilexr_ccu_hccp_loader.cpp + ccu/tilexr_ccu_launch_package.h + ccu/tilexr_ccu_launch_package.cpp + ccu/tilexr_ccu_install_provider.h + ccu/tilexr_ccu_install_provider.cpp + ccu/tilexr_ccu_lower_layer_plan_builder.h + ccu/tilexr_ccu_lower_layer_plan_builder.cpp + ccu/tilexr_ccu_lower_layer_payloads.h + ccu/tilexr_ccu_lower_layer_payloads.cpp + ccu/tilexr_ccu_memory_program.h + ccu/tilexr_ccu_memory_program.cpp + ccu/tilexr_ccu_microcode.h + ccu/tilexr_ccu_microcode.cpp + ccu/tilexr_ccu_producer_plan.h + ccu/tilexr_ccu_producer_plan.cpp + ccu/tilexr_ccu_provider.h + ccu/tilexr_ccu_provider.cpp + ccu/tilexr_ccu_ra_custom_channel_provider.h + ccu/tilexr_ccu_ra_custom_channel_provider.cpp + ccu/tilexr_ccu_repository.h + ccu/tilexr_ccu_repository.cpp + ccu/tilexr_ccu_resource_allocator.h + ccu/tilexr_ccu_resource_allocator.cpp + ccu/tilexr_ccu_runtime.h + ccu/tilexr_ccu_runtime.cpp + ccu/tilexr_ccu_specs.h + ccu/tilexr_ccu_specs.cpp tools/socket/tilexr_sock_exchange.h tools/socket/tilexr_sock_exchange.cpp udma/tilexr_hccp_defs.h @@ -119,6 +157,11 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ) add_library(tile-comm SHARED ${TILEXR_SOURCE_FILE}) +set_target_properties(tile-comm PROPERTIES + BUILD_WITH_INSTALL_RPATH FALSE + SKIP_BUILD_RPATH TRUE + INSTALL_RPATH "" + SKIP_INSTALL_RPATH TRUE) if(TILEXR_HAVE_PTO_SDMA) target_compile_definitions(tile-comm PRIVATE TILEXR_HAVE_PTO_SDMA=1) @@ -134,6 +177,7 @@ target_include_directories(tile-comm ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/ ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/profiling/ ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/runtime/ + ${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/runtime/runtime/ ${ASCEND_HOME_PATH}/${ARCH}-linux/include/ PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} diff --git a/src/comm/ccu/tilexr_ccu_abi_constants.h b/src/comm/ccu/tilexr_ccu_abi_constants.h new file mode 100644 index 00000000..10b1692d --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_abi_constants.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_ABI_CONSTANTS_H +#define TILEXR_CCU_ABI_CONSTANTS_H + +#include + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_EID_BYTES = 16; +constexpr uint32_t TILEXR_CCU_REMOTE_CCU_VA_SHIFT = 23; + +} // namespace TileXR + +#endif // TILEXR_CCU_ABI_CONSTANTS_H diff --git a/src/comm/ccu/tilexr_ccu_barrier_program.cpp b/src/comm/ccu/tilexr_ccu_barrier_program.cpp new file mode 100644 index 00000000..319caeea --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_barrier_program.cpp @@ -0,0 +1,273 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_barrier_program.h" + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuBarrierProgramReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuBarrierProgramReport{}; + } +} + +int Fail( + std::vector* program, + TileXRCcuBarrierProgramReport* report, + const std::string& message) +{ + if (program != nullptr) { + program->clear(); + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool MissingPostResource(const TileXRCcuBarrierSyncSpec& spec) +{ + return spec.remoteXn == 0 || spec.localXn == 0 || spec.channelId == 0 || + spec.remoteNotifyCke == 0 || spec.remoteNotifyMask == 0; +} + +bool MissingSyncCkePostResource(const TileXRCcuBarrierSyncSpec& spec) +{ + return spec.channelId == 0 || spec.remoteNotifyCke == 0 || spec.remoteNotifyMask == 0 || + spec.sourceCke == 0 || spec.sourceCkeMask == 0; +} + +bool MissingWaitResource(const TileXRCcuBarrierSyncSpec& spec) +{ + return spec.localWaitCke == 0 || spec.localWaitMask == 0; +} + +void FillReport( + size_t specCount, + size_t totalInstructionCount, + TileXRCcuBarrierProgramReport* report, + bool hasWaitInstructions = true) +{ + if (report == nullptr) { + return; + } + report->postInstructionCount = static_cast(specCount); + report->waitInstructionCount = hasWaitInstructions ? static_cast(specCount) : 0U; + report->totalInstructionCount = static_cast(totalInstructionCount); + report->message = "ok"; +} + +bool LoadBeforePostOnly(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +bool PostOnly(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly || + mode == TileXRCcuBarrierMode::SyncCkePostOnly || + mode == TileXRCcuBarrierMode::LocalCkePostOnly; +} + +bool SyncCkeMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncCke || + mode == TileXRCcuBarrierMode::SyncCkeSetWait || + mode == TileXRCcuBarrierMode::SyncCkePostOnly; +} + +bool SyncCkeSetWaitMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncCkeSetWait; +} + +void AddSourceCkeInitInstructions( + const std::vector& specs, + std::vector* program) +{ + std::vector initialized; + for (const auto& spec : specs) { + bool seen = false; + for (uint16_t cke : initialized) { + if (cke == spec.sourceCke) { + seen = true; + break; + } + } + if (seen) { + continue; + } + TileXRCcuCkeSpec init; + init.ckeId = spec.sourceCke; + init.mask = spec.sourceCkeMask; + init.clearWait = true; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSetCke(init, &instr) == TILEXR_SUCCESS) { + program->push_back(instr); + initialized.push_back(spec.sourceCke); + } + } +} + +int AddLocalCkeDiagnosticInstructions( + const std::vector& specs, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + for (const auto& spec : specs) { + TileXRCcuCkeSpec post; + post.ckeId = spec.localWaitCke; + post.mask = spec.localWaitMask; + post.clearWait = false; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSetCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode local CKE diagnostic set instruction"); + } + program->push_back(instr); + } + + for (const auto& spec : specs) { + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.localWaitCke; + wait.waitMask = spec.localWaitMask; + wait.clearWait = spec.clearLocalWait; + TileXRCcuInstr instr; + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode local CKE diagnostic wait instruction"); + } + program->push_back(instr); + } + + return TILEXR_SUCCESS; +} + +int AddLocalCkePostOnlyDiagnosticInstructions( + const std::vector& specs, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + for (const auto& spec : specs) { + TileXRCcuCkeSpec post; + post.ckeId = spec.localWaitCke; + post.mask = spec.localWaitMask; + post.clearWait = false; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSetCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode local CKE post-only diagnostic set instruction"); + } + program->push_back(instr); + } + return TILEXR_SUCCESS; +} + +} // namespace + +int TileXRCcuBuildBarrierProgram( + const std::vector& specs, + std::vector* program, + TileXRCcuBarrierProgramReport* report, + TileXRCcuBarrierMode mode) +{ + ResetReport(report); + if (program == nullptr) { + return Fail(program, report, "missing output CCU barrier program"); + } + program->clear(); + if (specs.empty()) { + return Fail(program, report, "missing CCU barrier sync specs"); + } + + for (const auto& spec : specs) { + if (mode != TileXRCcuBarrierMode::LocalCke && + mode != TileXRCcuBarrierMode::LocalCkePostOnly && + (SyncCkeMode(mode) ? MissingSyncCkePostResource(spec) : + MissingPostResource(spec))) { + return Fail(program, report, "missing remote XN post resource for CCU barrier program"); + } + if (!PostOnly(mode) && MissingWaitResource(spec)) { + return Fail(program, report, "missing local wait CKE resource for CCU barrier program"); + } + } + + program->reserve(specs.size() * (SyncCkeMode(mode) ? 3U : 2U)); + if (mode == TileXRCcuBarrierMode::LocalCke) { + const int ret = AddLocalCkeDiagnosticInstructions(specs, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + FillReport(specs.size(), program->size(), report); + return TILEXR_SUCCESS; + } + if (mode == TileXRCcuBarrierMode::LocalCkePostOnly) { + const int ret = AddLocalCkePostOnlyDiagnosticInstructions(specs, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + FillReport(specs.size(), program->size(), report, false); + return TILEXR_SUCCESS; + } + if (SyncCkeMode(mode)) { + AddSourceCkeInitInstructions(specs, program); + } + + for (const auto& spec : specs) { + TileXRCcuInstr instr; + if (SyncCkeMode(mode)) { + TileXRCcuSyncCkeSpec post; + post.remoteCke = spec.remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = spec.remoteNotifyMask; + post.channelId = spec.channelId; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU barrier SyncCKE post instruction"); + } + } else { + if (LoadBeforePostOnly(mode)) { + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 1U, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU barrier local XN load instruction"); + } + program->push_back(instr); + } + TileXRCcuSyncXnSpec post; + post.remoteXn = spec.remoteXn; + post.localXn = spec.localXn; + post.channelId = spec.channelId; + post.notifyCke = spec.remoteNotifyCke; + post.notifyMask = spec.remoteNotifyMask; + if (TileXRCcuEncodeSyncXn(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU barrier post instruction"); + } + } + program->push_back(instr); + } + + if (PostOnly(mode)) { + FillReport(specs.size(), program->size(), report, false); + return TILEXR_SUCCESS; + } + + for (const auto& spec : specs) { + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.localWaitCke; + wait.waitMask = spec.localWaitMask; + wait.clearWait = spec.clearLocalWait; + TileXRCcuInstr instr; + const int ret = SyncCkeMode(mode) && !SyncCkeSetWaitMode(mode) ? + TileXRCcuEncodeClearCke(wait, &instr) : + TileXRCcuEncodeSetCke(wait, &instr); + if (ret != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU barrier wait/clear instruction"); + } + program->push_back(instr); + } + + FillReport(specs.size(), program->size(), report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_barrier_program.h b/src/comm/ccu/tilexr_ccu_barrier_program.h new file mode 100644 index 00000000..394b4680 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_barrier_program.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_BARRIER_PROGRAM_H +#define TILEXR_CCU_BARRIER_PROGRAM_H + +#include "ccu/tilexr_ccu_microcode.h" + +#include +#include +#include + +namespace TileXR { + +enum class TileXRCcuBarrierMode { + SyncXn = 0, + SyncCke = 1, + LocalCke = 2, + SyncXnPostOnly = 3, + SyncXnLoadPostOnly = 4, + SyncCkePostOnly = 5, + LocalCkePostOnly = 6, + SyncCkeSetWait = 7, +}; + +struct TileXRCcuBarrierSyncSpec { + uint16_t remoteXn = 0; + uint16_t localXn = 0; + uint16_t channelId = 0; + uint16_t remoteNotifyCke = 0; + uint16_t remoteNotifyMask = 0; + uint16_t localWaitCke = 0; + uint16_t localWaitMask = 0; + uint16_t sourceCke = 0; + uint16_t sourceCkeMask = 0; + bool clearLocalWait = true; +}; + +struct TileXRCcuBarrierProgramReport { + uint32_t postInstructionCount = 0; + uint32_t waitInstructionCount = 0; + uint32_t totalInstructionCount = 0; + std::string message; +}; + +int TileXRCcuBuildBarrierProgram( + const std::vector& specs, + std::vector* program, + TileXRCcuBarrierProgramReport* report, + TileXRCcuBarrierMode mode = TileXRCcuBarrierMode::SyncXn); + +} // namespace TileXR + +#endif // TILEXR_CCU_BARRIER_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp new file mode 100644 index 00000000..f3d8aef5 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -0,0 +1,1199 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_direct_orchestrator.h" + +#include "ccu/tilexr_ccu_runtime.h" + +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +constexpr const char* TILEXR_CCU_DIRECT_KNOWN_MISSING_INSTALL_SURFACES = + "remote XN install provider is missing"; +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_SQE_ARGS_TO_X_HEADER = 0x0001U; +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_IMD_TO_GSA_HEADER = 0x0002U; +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER = 0x0003U; +constexpr uint16_t TILEXR_CCU_TRACE_SET_CKE_HEADER = 0x0802U; +constexpr uint16_t TILEXR_CCU_TRACE_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; +constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALID_SHIFT = 52ULL; +constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_ID_SHIFT = 32ULL; +constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_ID_MASK = 0xfffffULL; +constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALUE_MASK = 0xffffffffULL; +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; + +void ResetReport(TileXRCcuDirectInstallReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + } +} + +void ClearAttempt(TileXRCcuDirectInstallAttempt* attempt) +{ + if (attempt != nullptr) { + *attempt = TileXRCcuDirectInstallAttempt{}; + } +} + +int Fail(TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report, const std::string& message) +{ + ClearAttempt(attempt); + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool HasRepositoryInstallInputs(const TileXRCcuDirectInstallOptions& options) +{ + return options.driverAdapter != nullptr && + options.repositoryMemoryOps.alloc != nullptr && + options.repositoryMemoryOps.copyHostToDevice != nullptr && + options.repositoryMemoryOps.free != nullptr; +} + +bool SyncXnMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXn || + mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +bool HasNonZeroArgs(const TileXRCcuTaskWindow& window) +{ + for (uint64_t arg : window.args) { + if (arg != 0) { + return true; + } + } + return false; +} + +bool ContainsRange(uint16_t outerStart, uint16_t outerCount, uint16_t innerStart, uint32_t innerCount) +{ + if (outerCount == 0 || innerCount == 0) { + return false; + } + const uint32_t outerBegin = outerStart; + const uint32_t outerEnd = outerBegin + outerCount; + const uint32_t innerBegin = innerStart; + const uint32_t innerEnd = innerBegin + innerCount; + return innerBegin >= outerBegin && innerEnd <= outerEnd; +} + +bool RangesOverlap(uint16_t firstStart, uint32_t firstCount, uint16_t secondStart, uint32_t secondCount) +{ + const uint32_t firstEnd = static_cast(firstStart) + firstCount; + const uint32_t secondEnd = static_cast(secondStart) + secondCount; + return static_cast(firstStart) < secondEnd && static_cast(secondStart) < firstEnd; +} + +TileXRCcuRange MakeRange(uint8_t dieId, uint16_t startId, uint16_t count) +{ + TileXRCcuRange range; + range.dieId = dieId; + range.startId = startId; + range.num = count; + return range; +} + +uint64_t PackCcuSqeToken(uint32_t tokenId, uint32_t tokenValue, bool valid) +{ + const uint64_t validBits = valid ? 1ULL : 0ULL; + return (validBits << TILEXR_CCU_PACKED_TOKEN_VALID_SHIFT) | + ((static_cast(tokenId) & TILEXR_CCU_PACKED_TOKEN_ID_MASK) << TILEXR_CCU_PACKED_TOKEN_ID_SHIFT) | + (static_cast(tokenValue) & TILEXR_CCU_PACKED_TOKEN_VALUE_MASK); +} + +uint16_t ReadLe16(const uint8_t* raw, uint32_t offset) +{ + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1U]) << 8U); +} + +uint64_t DecodeChannelRemoteCcuVa(const TileXRCcuChannelCtxDataV1& ctx) +{ + const uint16_t word28 = ReadLe16(ctx.raw, 28); + const uint16_t word34 = ReadLe16(ctx.raw, 34); + const uint64_t dstVa = + ((static_cast(word28) >> 8U) & 0xffULL) | + (static_cast(ReadLe16(ctx.raw, 30)) << 8U) | + (static_cast(ReadLe16(ctx.raw, 32)) << 24U) | + ((static_cast(word34) & 0x1ULL) << 40U); + return dstVa << TILEXR_CCU_REMOTE_CCU_VA_SHIFT; +} + +int PopulateHcommStyleSqeTaskArgs( + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.taskWindows.size() < 2 || + !SyncXnMode(attempt->plan.barrierMode)) { + return TILEXR_SUCCESS; + } + + TileXRCcuTaskWindow& sqeLoadTask = attempt->plan.taskWindows[0]; + if (sqeLoadTask.argSize != TILEXR_CCU_SQE_ARGS_LEN || + sqeLoadTask.instCnt == 0 || + attempt->specInfo.resourceAddr == 0) { + return TILEXR_SUCCESS; + } + if (HasNonZeroArgs(sqeLoadTask)) { + return TILEXR_SUCCESS; + } + + sqeLoadTask.args.assign(TILEXR_CCU_SQE_ARGS_LEN, 0); + sqeLoadTask.args[0] = attempt->specInfo.resourceAddr; + sqeLoadTask.args[1] = attempt->specInfo.resourceAddr; + + if (!attempt->preparedLowerLayerPlan.msidTokens.empty()) { + const auto& token = attempt->preparedLowerLayerPlan.msidTokens[0]; + sqeLoadTask.args[2] = PackCcuSqeToken(token.tokenId, token.tokenValue, true); + } + for (const auto& channel : attempt->preparedLowerLayerPlan.channels) { + const uint64_t remoteCcuVa = DecodeChannelRemoteCcuVa(channel.ctx); + if (remoteCcuVa != 0) { + sqeLoadTask.args[3] = remoteCcuVa; + break; + } + } + + if (report != nullptr) { + report->message.clear(); + } + return TILEXR_SUCCESS; +} + +bool DirectTraceEnabled() +{ + const char* value = std::getenv("TILEXR_CCU_DIRECT_TRACE"); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +uint16_t TraceSlot(uint64_t word, uint32_t slot) +{ + return static_cast((word >> (slot * 16U)) & 0xffffU); +} + +uint16_t TraceRead16(const uint8_t* raw, uint32_t offset) +{ + return ReadLe16(raw, offset); +} + +uint64_t TraceReadDoorbellVa(const TileXRCcuLocalJettyCtxData& ctx) +{ + uint64_t value = 0; + for (uint32_t word = 0; word < 4U; ++word) { + value |= static_cast(TraceRead16(ctx.raw, word * 2U)) << (word * 16U); + } + return value; +} + +uint64_t TraceLoadImmediate(const TileXRCcuInstr& instr) +{ + return (instr.words[0] >> 32U) | ((instr.words[1] & 0xffffffffULL) << 32U); +} + +void TraceDecodedPfeCtx(size_t index, const TileXRCcuPfeInstall& pfe) +{ + const uint16_t word = TraceRead16(pfe.ctx.raw, 2); + const uint16_t jettyCountMinusOne = word & 0x7fU; + std::cerr << "TileXRDirectCcuTrace lowerLayerPfe[" << index << "]" + << " decoded=PfeCtx" + << " dieId=" << static_cast(pfe.dieId) + << " pfeOffset=" << pfe.pfeOffset + << " startTaJettyId=" << TraceRead16(pfe.ctx.raw, 0) + << " jettyCount=" << static_cast(jettyCountMinusOne) + 1U + << " jettyCountMinusOne=" << jettyCountMinusOne + << " startLocalJettyCtxId=" << ((word >> 7U) & 0x7fU) + << "\n"; +} + +void TraceDecodedLocalJettyCtx(size_t jettyIndex, size_t ctxIndex, const TileXRCcuLocalJettyCtxData& ctx) +{ + const uint16_t word8 = TraceRead16(ctx.raw, 8); + const uint16_t word10 = TraceRead16(ctx.raw, 10); + const uint16_t word14 = TraceRead16(ctx.raw, 14); + const uint16_t word22 = TraceRead16(ctx.raw, 22); + const uint16_t word24 = TraceRead16(ctx.raw, 24); + const uint32_t tokenId = + ((word8 >> 8U) & 0xffU) | + ((static_cast(word10) & 0xfffU) << 8U); + const uint32_t tokenValue = + ((word10 >> 12U) & 0xfU) | + (static_cast(TraceRead16(ctx.raw, 12)) << 4U) | + ((static_cast(word14) & 0xfffU) << 20U); + const uint32_t wqeBasicBlockShift = (word14 >> 12U) & 0xfU; + const uint32_t wqeBasicBlockCount = 1U << wqeBasicBlockShift; + const uint32_t wqeBasicBlockStartId = + ((word22 >> 12U) & 0xfU) | + ((static_cast(word24) & 0xffU) << 4U); + + std::cerr << "TileXRDirectCcuTrace lowerLayerJettyCtx[" << jettyIndex << "," << ctxIndex << "]" + << " decoded=LocalJettyCtx" + << " doorbellVa=" << std::hex << std::showbase << TraceReadDoorbellVa(ctx) + << " doorbellTokenId=" << tokenId + << " doorbellTokenValue=" << tokenValue + << std::dec << std::noshowbase + << " pfeId=" << (word8 & 0xfU) + << " ioDieId=" << ((word8 >> 4U) & 0x1U) + << " doorbellAddrType=" << ((word8 >> 5U) & 0x1U) + << " tokenValueValid=" << ((word8 >> 6U) & 0x1U) + << " sqeBasicBlockLeftShifts=" << wqeBasicBlockShift + << " wqeBasicBlockCount=" << wqeBasicBlockCount + << " inferredSqDepth=" << (wqeBasicBlockCount / 4U) + << " wqeBasicBlockStartId=" << wqeBasicBlockStartId + << " pi=" << TraceRead16(ctx.raw, 16) + << " ci=" << TraceRead16(ctx.raw, 18) + << " maxCi=" << TraceRead16(ctx.raw, 20) + << " oooCqeCnt=" << (word22 & 0xfffU) + << " doorbellSendState=" << ((word24 >> 8U) & 0x3U) + << "\n"; +} + +void TraceDecodedChannelCtxV1(size_t index, const TileXRCcuChannelInstall& channel) +{ + const uint16_t word16 = TraceRead16(channel.ctx.raw, 16); + const uint16_t word18 = TraceRead16(channel.ctx.raw, 18); + const uint16_t word20 = TraceRead16(channel.ctx.raw, 20); + const uint16_t word22 = TraceRead16(channel.ctx.raw, 22); + const uint16_t word24 = TraceRead16(channel.ctx.raw, 24); + const uint16_t word28 = TraceRead16(channel.ctx.raw, 28); + const uint16_t word34 = TraceRead16(channel.ctx.raw, 34); + + const uint32_t tpn = word16 | ((static_cast(word18) & 0xffU) << 16U); + const uint16_t startJettyId = + ((word18 >> 12U) & 0xfU) | + static_cast((word20 & 0xfffU) << 4U); + const uint16_t jettyCountMinusOne = + ((word20 >> 12U) & 0xfU) | + static_cast((word22 & 0x7U) << 4U); + const uint32_t tokenId = + ((static_cast(word22) >> 4U) & 0xfffU) | + ((static_cast(word24) & 0xffU) << 12U); + const uint32_t tokenValue = + ((static_cast(word24) >> 8U) & 0xffU) | + (static_cast(TraceRead16(channel.ctx.raw, 26)) << 8U) | + ((static_cast(word28) & 0xffU) << 24U); + const uint64_t dstVa = + ((static_cast(word28) >> 8U) & 0xffULL) | + (static_cast(TraceRead16(channel.ctx.raw, 30)) << 8U) | + (static_cast(TraceRead16(channel.ctx.raw, 32)) << 24U) | + ((static_cast(word34) & 0x1ULL) << 40U); + const uint64_t remoteCcuVa = dstVa << TILEXR_CCU_REMOTE_CCU_VA_SHIFT; + + std::cerr << "TileXRDirectCcuTrace lowerLayerChannel[" << index << "]" + << " decoded=ChannelCtxV1" + << " dieId=" << static_cast(channel.dieId) + << " channelId=" << channel.channelId + << " tpn=" << std::hex << std::showbase << tpn + << " memoryTokenId=" << tokenId + << " memoryTokenValue=" << tokenValue + << " dstVaShifted=" << dstVa + << " remoteCcuVa=" << remoteCcuVa + << std::dec << std::noshowbase + << " sourcePfeId=" << ((word18 >> 8U) & 0xfU) + << " startTaJettyId=" << startJettyId + << " jettyCount=" << static_cast(jettyCountMinusOne) + 1U + << " jettyCountMinusOne=" << jettyCountMinusOne + << " ioDieId=" << ((word22 >> 3U) & 0x1U) + << " tokenValueValid=" << ((word34 >> 1U) & 0x1U) + << " remoteEid="; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + std::cerr << std::hex << std::setw(2) << std::setfill('0') << std::noshowbase + << static_cast(channel.ctx.raw[i]); + } + std::cerr << std::dec << std::setfill(' ') << "\n"; +} + +void TraceDecodedInstr(const char* label, size_t index, const TileXRCcuInstr& instr) +{ + const uint16_t opcode = TraceSlot(instr.words[0], 0); + std::cerr << "TileXRDirectCcuTrace " << label << "[" << index << "] "; + switch (opcode) { + case TILEXR_CCU_TRACE_LOAD_SQE_ARGS_TO_X_HEADER: + std::cerr << "decoded=LoadSqeArgsToX" + << " xnId=" << TraceSlot(instr.words[0], 1) + << " sqeArgId=" << TraceSlot(instr.words[0], 2); + break; + case TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER: + std::cerr << "decoded=LoadImdToXn" + << " xnId=" << TraceSlot(instr.words[0], 1) + << " immediate=" << std::hex << std::showbase << TraceLoadImmediate(instr) + << std::dec << std::noshowbase + << " secFlag=" << TraceSlot(instr.words[1], 2); + break; + case TILEXR_CCU_TRACE_LOAD_IMD_TO_GSA_HEADER: + std::cerr << "decoded=LoadImdToGSA" + << " gsaId=" << TraceSlot(instr.words[0], 1) + << " immediate=" << std::hex << std::showbase << TraceLoadImmediate(instr) + << std::dec << std::noshowbase; + break; + case TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER: { + const uint16_t control = TraceSlot(instr.words[1], 3); + const uint16_t flags = TraceSlot(instr.words[2], 3); + std::cerr << "decoded=TransRmtMemToLocMem" + << " remoteGsa=" << TraceSlot(instr.words[0], 3) + << " remoteXn=" << TraceSlot(instr.words[1], 0) + << " localGsa=" << TraceSlot(instr.words[0], 1) + << " localXn=" << TraceSlot(instr.words[0], 2) + << " lengthXn=" << TraceSlot(instr.words[1], 1) + << " channelId=" << TraceSlot(instr.words[1], 2) + << " clearType=" << (flags & 0x1U) + << " lengthEn=" << ((flags >> 1U) & 0x1U) + << " reduceEn=" << ((flags >> 2U) & 0x1U) + << " reduceDataType=" << ((control >> 8U) & 0xfU) + << " reduceOpCode=" << ((control >> 12U) & 0xfU) + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + } + case TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER: { + const uint16_t control = TraceSlot(instr.words[1], 3); + const uint16_t flags = TraceSlot(instr.words[2], 3); + std::cerr << "decoded=TransLocMemToRmtMem" + << " localGsa=" << TraceSlot(instr.words[0], 3) + << " localXn=" << TraceSlot(instr.words[1], 0) + << " remoteGsa=" << TraceSlot(instr.words[0], 1) + << " remoteXn=" << TraceSlot(instr.words[0], 2) + << " lengthXn=" << TraceSlot(instr.words[1], 1) + << " channelId=" << TraceSlot(instr.words[1], 2) + << " clearType=" << (flags & 0x1U) + << " lengthEn=" << ((flags >> 1U) & 0x1U) + << " reduceEn=" << ((flags >> 2U) & 0x1U) + << " reduceDataType=" << ((control >> 8U) & 0xfU) + << " reduceOpCode=" << ((control >> 12U) & 0xfU) + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + } + case TILEXR_CCU_TRACE_SYNC_XN_HEADER: + std::cerr << "decoded=SyncXn" + << " remoteXn=" << TraceSlot(instr.words[0], 1) + << " localXn=" << TraceSlot(instr.words[0], 2) + << " channelId=" << TraceSlot(instr.words[1], 0) + << " notifyCke=" << TraceSlot(instr.words[1], 1) + << " notifyMask=" << TraceSlot(instr.words[1], 2) + << " traceFlag=" << std::hex << std::showbase << instr.words[2] + << std::dec << std::noshowbase + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + case TILEXR_CCU_TRACE_SYNC_CKE_HEADER: + std::cerr << "decoded=SyncCke" + << " remoteCke=" << TraceSlot(instr.words[0], 1) + << " localCke=" << TraceSlot(instr.words[0], 2) + << " localCkeMask=" << TraceSlot(instr.words[0], 3) + << " channelId=" << TraceSlot(instr.words[1], 0) + << " clearType=" << TraceSlot(instr.words[2], 3) + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + case TILEXR_CCU_TRACE_SET_CKE_HEADER: + std::cerr << "decoded=SetCke" + << " clearType=" << TraceSlot(instr.words[0], 1) + << " ckeId=" << TraceSlot(instr.words[0], 2) + << " mask=" << TraceSlot(instr.words[0], 3) + << " waitCkeId=" << TraceSlot(instr.words[1], 0) + << " waitMask=" << TraceSlot(instr.words[1], 1); + break; + case TILEXR_CCU_TRACE_CLEAR_CKE_HEADER: + std::cerr << "decoded=ClearCke" + << " clearType=" << TraceSlot(instr.words[0], 1) + << " ckeId=" << TraceSlot(instr.words[0], 2) + << " mask=" << TraceSlot(instr.words[0], 3) + << " waitCkeId=" << TraceSlot(instr.words[1], 0) + << " waitMask=" << TraceSlot(instr.words[1], 1); + break; + default: + std::cerr << "decoded=Unknown opcode=" << std::hex << std::showbase << opcode + << std::dec << std::noshowbase; + break; + } + std::cerr << "\n"; +} + +void TraceInstr(const char* label, size_t index, const TileXRCcuInstr& instr) +{ + std::cerr << "TileXRDirectCcuTrace " << label << "[" << index << "] words=" + << std::hex << std::showbase + << instr.words[0] << "," << instr.words[1] << "," + << instr.words[2] << "," << instr.words[3] + << std::dec << std::noshowbase << "\n"; + TraceDecodedInstr(label, index, instr); +} + +void TraceDirectInstallAttempt(const TileXRCcuDirectInstallAttempt& attempt) +{ + if (!DirectTraceEnabled()) { + return; + } + + std::cerr << "TileXRDirectCcuTrace begin" + << " missionId=" << static_cast(attempt.plan.mission.missionId) + << " missionKey=" << std::hex << std::showbase << attempt.plan.mission.key + << std::dec << std::noshowbase + << " dieId=" << static_cast(attempt.plan.mission.dieId) + << "\n"; + for (size_t i = 0; i < attempt.plan.syncResources.size(); ++i) { + const auto& resource = attempt.plan.syncResources[i]; + std::cerr << "TileXRDirectCcuTrace syncResource[" << i << "]" + << " dieId=" << static_cast(resource.dieId) + << " channelId=" << resource.channelId + << " localXn=" << resource.localXn + << " remoteXn=" << resource.remoteXn + << " notifyCke=" << resource.notifyCke + << " localWaitCke=" << resource.localWaitCke + << " localWaitMask=" << resource.localWaitMask + << " remoteNotifyMask=" << resource.remoteNotifyMask + << " sourceCke=" << resource.sourceCke + << " sourceCkeMask=" << resource.sourceCkeMask + << "\n"; + } + for (size_t i = 0; i < attempt.preparedLowerLayerPlan.remoteXnBindings.size(); ++i) { + const auto& proof = attempt.preparedLowerLayerPlan.remoteXnBindings[i]; + std::cerr << "TileXRDirectCcuTrace remoteXnBinding[" << i << "]" + << " dieId=" << static_cast(proof.dieId) + << " channelId=" << proof.channelId + << " localXn=" << proof.localXn + << " remoteXn=" << proof.remoteXn + << " notifyCke=" << proof.notifyCke + << " localWaitCke=" << proof.localWaitCke + << " peerRank=" << proof.peerRank + << " peerExchangeObserved=" << (proof.peerExchangeObserved ? 1 : 0) + << " endpointRouteVerified=" << (proof.endpointRouteVerified ? 1 : 0) + << " channelResourceOwnerVerified=" << (proof.channelResourceOwnerVerified ? 1 : 0) + << " transportResourceExchangeVerified=" << (proof.transportResourceExchangeVerified ? 1 : 0) + << "\n"; + } + for (size_t i = 0; i < attempt.preparedLowerLayerPlan.pfes.size(); ++i) { + const auto& pfe = attempt.preparedLowerLayerPlan.pfes[i]; + std::cerr << "TileXRDirectCcuTrace lowerLayerPfe[" << i << "]" + << " dieId=" << static_cast(pfe.dieId) + << " pfeOffset=" << pfe.pfeOffset + << " ctx="; + for (uint8_t byte : pfe.ctx.raw) { + std::cerr << std::hex << std::setw(2) << std::setfill('0') + << static_cast(byte); + } + std::cerr << std::dec << std::setfill(' ') << "\n"; + TraceDecodedPfeCtx(i, pfe); + } + for (size_t i = 0; i < attempt.preparedLowerLayerPlan.channels.size(); ++i) { + const auto& channel = attempt.preparedLowerLayerPlan.channels[i]; + std::cerr << "TileXRDirectCcuTrace lowerLayerChannel[" << i << "]" + << " dieId=" << static_cast(channel.dieId) + << " channelId=" << channel.channelId + << " ctx="; + for (uint8_t byte : channel.ctx.raw) { + std::cerr << std::hex << std::setw(2) << std::setfill('0') + << static_cast(byte); + } + std::cerr << std::dec << std::setfill(' ') << "\n"; + TraceDecodedChannelCtxV1(i, channel); + } + for (size_t i = 0; i < attempt.preparedLowerLayerPlan.jettys.size(); ++i) { + const auto& jetty = attempt.preparedLowerLayerPlan.jettys[i]; + std::cerr << "TileXRDirectCcuTrace lowerLayerJetty[" << i << "]" + << " dieId=" << static_cast(jetty.dieId) + << " startJettyCtxId=" << jetty.startJettyCtxId + << " ctxCount=" << jetty.ctxs.size() + << "\n"; + for (size_t j = 0; j < jetty.ctxs.size(); ++j) { + std::cerr << "TileXRDirectCcuTrace lowerLayerJettyCtx[" << i << "," << j << "] ctx="; + for (uint8_t byte : jetty.ctxs[j].raw) { + std::cerr << std::hex << std::setw(2) << std::setfill('0') + << static_cast(byte); + } + std::cerr << std::dec << std::setfill(' ') << "\n"; + TraceDecodedLocalJettyCtx(i, j, jetty.ctxs[j]); + } + } + for (size_t i = 0; i < attempt.plan.taskWindows.size(); ++i) { + const auto& window = attempt.plan.taskWindows[i]; + std::cerr << "TileXRDirectCcuTrace taskWindow[" << i << "]" + << " dieId=" << static_cast(window.dieId) + << " instStartId=" << window.instStartId + << " instCnt=" << window.instCnt + << " argSize=" << window.argSize + << " args="; + for (size_t arg = 0; arg < window.args.size(); ++arg) { + if (arg != 0) { + std::cerr << ","; + } + std::cerr << std::hex << std::showbase << window.args[arg] + << std::dec << std::noshowbase; + } + std::cerr << "\n"; + } + const auto& tracedTasks = attempt.submitTasks.empty() ? attempt.package.tasks : attempt.submitTasks; + for (size_t i = 0; i < tracedTasks.size(); ++i) { + const auto& task = tracedTasks[i]; + std::cerr << "TileXRDirectCcuTrace task[" << i << "]" + << " dieId=" << static_cast(task.dieId) + << " missionId=" << static_cast(task.missionId) + << " timeout=" << task.timeout + << " instStartId=" << task.instStartId + << " instCnt=" << task.instCnt + << " argSize=" << task.argSize + << " key=" << std::hex << std::showbase << task.key + << std::dec << std::noshowbase + << " args="; + for (uint32_t arg = 0; arg < TILEXR_CCU_SQE_ARGS_LEN; ++arg) { + if (arg != 0) { + std::cerr << ","; + } + std::cerr << std::hex << std::showbase << task.args[arg] + << std::dec << std::noshowbase; + } + std::cerr << "\n"; + } + for (size_t i = 0; i < attempt.package.program.sqeLoad.size(); ++i) { + TraceInstr("program.sqeLoad", i, attempt.package.program.sqeLoad[i]); + } + for (size_t i = 0; i < attempt.package.program.sync.size(); ++i) { + TraceInstr("program.sync", i, attempt.package.program.sync[i]); + } + std::cerr << "TileXRDirectCcuTrace end\n"; +} + +void ApplySplitCkeOptions( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuResourceSpec* resourceSpec) +{ + if (resourceSpec == nullptr) { + return; + } + if (options.localWaitCkeStartId != 0 || options.localWaitCkeCount != 0) { + resourceSpec->localWaitCkeStartId = + options.localWaitCkeStartId == 0 ? options.ckeStartId : options.localWaitCkeStartId; + resourceSpec->localWaitCkeCount = + options.localWaitCkeCount == 0 ? resourceSpec->ckeCount : options.localWaitCkeCount; + } + if (options.remoteNotifyCkeStartId != 0 || options.remoteNotifyCkeCount != 0) { + resourceSpec->remoteNotifyCkeStartId = + options.remoteNotifyCkeStartId == 0 ? options.ckeStartId : options.remoteNotifyCkeStartId; + resourceSpec->remoteNotifyCkeCount = + options.remoteNotifyCkeCount == 0 ? resourceSpec->ckeCount : options.remoteNotifyCkeCount; + } +} + +void ApplyRemoteXnOptions( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuResourceSpec* resourceSpec) +{ + if (resourceSpec == nullptr) { + return; + } + if (options.remoteXnStartId != 0 || options.remoteXnCount != 0) { + resourceSpec->remoteXnStartId = options.remoteXnStartId; + resourceSpec->remoteXnCount = + options.remoteXnCount == 0 ? resourceSpec->xnCount : options.remoteXnCount; + } +} + +int PrepareLowerLayerPlanIfNeeded( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (options.prepareLowerLayerPlan == nullptr) { + return TILEXR_SUCCESS; + } + if (attempt == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport planReport; + const int ret = options.prepareLowerLayerPlan( + attempt->allocation, + &plan, + &planReport, + options.lowerLayerPlanUserData); + attempt->lowerLayerPlanReport = planReport; + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message.empty() ? + "failed to prepare direct CCU lower-layer install plan" : + planReport.message; + } + return ret; + } + attempt->preparedLowerLayerPlan = plan; + return TILEXR_SUCCESS; +} + +int ReconcileProducerPlanWithLowerLayerProof( + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->preparedLowerLayerPlan.remoteXnBindings.empty()) { + return TILEXR_SUCCESS; + } + if (attempt->preparedLowerLayerPlan.remoteXnBindings.size() != attempt->plan.syncResources.size()) { + if (report != nullptr) { + report->message = "lower-layer remote XN proof count does not match producer sync resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + for (auto& resource : attempt->plan.syncResources) { + bool matched = false; + for (const auto& proof : attempt->preparedLowerLayerPlan.remoteXnBindings) { + if (proof.dieId != resource.dieId || + proof.channelId != resource.channelId || + proof.localXn != resource.localXn || + !proof.peerExchangeObserved) { + continue; + } + resource.remoteXn = proof.remoteXn; + resource.notifyCke = proof.notifyCke; + if (proof.localWaitCke != 0) { + resource.localWaitCke = proof.localWaitCke; + } + matched = true; + break; + } + if (!matched) { + if (report != nullptr) { + report->message = "lower-layer remote XN proof does not cover producer sync resource"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; +} + +int ConfigureDirectMemoryCopyResources( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->plan.syncResources.size() != 1 || attempt->plan.taskWindows.size() != 1) { + if (report != nullptr) { + report->message = "memory copy direct CCU plan requires one sync resource and one task"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (options.gsaStartId == 0 || attempt->resourceSpec.gsaCount < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "memory copy direct CCU requires a kernel-local GSA resource window"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint8_t dieId = attempt->specInfo.dieId; + const uint16_t localXnStart = attempt->allocation.localXn.startId; + if (!ContainsRange( + attempt->resourceSpec.xnStartId, + attempt->resourceSpec.xnCount, + localXnStart, + TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)) { + if (report != nullptr) { + report->message = "memory copy direct CCU local XN window is too small"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + uint16_t remoteXnStart = attempt->allocation.remoteXn.startId; + if (RangesOverlap( + localXnStart, + TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT, + remoteXnStart, + 1U)) { + remoteXnStart = static_cast(localXnStart + TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT); + } + if (!ContainsRange(attempt->resourceSpec.xnStartId, attempt->resourceSpec.xnCount, remoteXnStart, 1U)) { + if (report != nullptr) { + report->message = "memory copy direct CCU remote XN window is outside the XN resource range"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->allocation.localXn = + MakeRange(dieId, localXnStart, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)); + attempt->allocation.localGsa = + MakeRange(dieId, options.gsaStartId, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT)); + attempt->allocation.remoteXn = MakeRange(dieId, remoteXnStart, 1U); + attempt->plan.kernelLocalXn = attempt->allocation.localXn; + attempt->plan.kernelLocalGsa = attempt->allocation.localGsa; + attempt->plan.syncResources[0].remoteXn = remoteXnStart; + attempt->plan.taskWindows[0].instCnt = + static_cast(std::max( + attempt->plan.taskWindows[0].instCnt, + TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT)); + return TILEXR_SUCCESS; +} + +int BuildDirectMemoryCopyLaunchPackage( + const TileXRCcuDirectMemoryCopySpec& memoryCopy, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->plan.syncResources.empty()) { + if (report != nullptr) { + report->message = "missing direct CCU memory copy producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; + if (attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT || + attempt->plan.kernelLocalXn.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU memory copy GSA/XN resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuMemoryCopySpec copySpec; + copySpec.direction = memoryCopy.direction; + copySpec.localGsa = attempt->plan.kernelLocalGsa.startId; + copySpec.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + copySpec.localXn = attempt->plan.kernelLocalXn.startId; + copySpec.remoteXn = static_cast(attempt->plan.kernelLocalXn.startId + 1U); + copySpec.lengthXn = static_cast(attempt->plan.kernelLocalXn.startId + 2U); + copySpec.localAddr = memoryCopy.localAddr; + copySpec.localToken = memoryCopy.localToken; + copySpec.remoteAddr = memoryCopy.remoteAddr; + copySpec.remoteToken = memoryCopy.remoteToken; + copySpec.lengthBytes = memoryCopy.lengthBytes; + copySpec.channelId = resource.channelId; + copySpec.completionCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; + copySpec.completionMask = resource.localWaitMask == 0 ? 1U : resource.localWaitMask; + + TileXRCcuProgram program; + TileXRCcuMemoryProgramReport memoryReport; + if (TileXRCcuBuildMemoryCopyProgram(copySpec, &program.sync, &memoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = memoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + +void FillReportFromAttempt(const TileXRCcuDirectInstallAttempt& attempt, TileXRCcuDirectInstallReport* report) +{ + if (report == nullptr) { + return; + } + report->pipelineBuilt = true; + report->installAttempted = attempt.installReport.installAttempted; + report->installSucceeded = attempt.installReport.installSucceeded; + report->submitReady = attempt.providerReport.submitReady; + report->requiredInstallSurfaceCount = attempt.installReport.requiredInstallSurfaceCount; + report->publicVerifiedInstallSurfaceCount = attempt.installReport.publicVerifiedInstallSurfaceCount; + report->missingInstallSurfaceCount = attempt.installReport.missingInstallSurfaceCount; + report->taskCount = static_cast(attempt.package.tasks.size()); + report->submitTaskCount = static_cast(attempt.submitTasks.size()); +} + +void ResetSubmitReport(TileXRCcuDirectSubmitReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuDirectSubmitReport{}; + } +} + +void TraceFinalRuntimeTask(size_t taskIndex, const TileXRCcuTask& task) +{ + if (!DirectTraceEnabled()) { + return; + } + std::cerr << "TileXRDirectCcuTrace finalRuntimeTask[" << taskIndex << "]" + << " dieId=" << static_cast(task.dieId) + << " missionId=" << static_cast(task.missionId) + << " timeout=" << task.timeout + << " instStartId=" << task.instStartId + << " instCnt=" << task.instCnt + << " key=0x" << std::hex << std::nouppercase << task.key + << std::dec + << " argSize=" << task.argSize; + for (uint32_t arg = 0; arg < TILEXR_CCU_SQE_ARGS_LEN; ++arg) { + std::cerr << " args[" << arg << "]=0x" + << std::hex << std::nouppercase << task.args[arg] + << std::dec; + } + std::cerr << "\n"; +} + +std::string FormatSubmitTaskFailure( + size_t taskIndex, + int ret, + const TileXRCcuTask& task, + const TileXRCcuRuntimeSubmitReport* runtimeReport) +{ + const TileXRCcuTask& diagnosticTask = + (runtimeReport != nullptr && runtimeReport->finalTaskCaptured) ? + runtimeReport->finalTask : + task; + std::ostringstream oss; + oss << "direct CCU submit failed task=" << taskIndex + << " ret=" << ret; + if (runtimeReport != nullptr && runtimeReport->runtimeLaunchAttempted) { + oss << " rtRet=" << runtimeReport->runtimeRet; + } + oss << " dieId=" << static_cast(diagnosticTask.dieId) + << " missionId=" << static_cast(diagnosticTask.missionId) + << " timeout=" << diagnosticTask.timeout + << " instStartId=" << diagnosticTask.instStartId + << " instCnt=" << diagnosticTask.instCnt + << " key=0x" << std::hex << std::nouppercase << diagnosticTask.key + << std::dec + << " argSize=" << diagnosticTask.argSize; + for (uint32_t arg = 0; arg < TILEXR_CCU_SQE_ARGS_LEN; ++arg) { + oss << " args[" << arg << "]=0x" + << std::hex << std::nouppercase << diagnosticTask.args[arg] + << std::dec; + } + return oss.str(); +} + +int ReturnWithAttemptStatus( + int ret, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt != nullptr) { + FillReportFromAttempt(*attempt, report); + if (report != nullptr) { + if (attempt->providerReport.submitReady) { + report->message = attempt->providerReport.message; + } else if (!attempt->installReport.message.empty()) { + report->message = attempt->installReport.message; + } else { + report->message = attempt->providerReport.message.empty() ? + TILEXR_CCU_DIRECT_KNOWN_MISSING_INSTALL_SURFACES : + attempt->providerReport.message; + } + } + } + return ret; +} + +} // namespace + +int TileXRCcuSubmitPreparedTasks( + const std::vector& submitTasks, + void* stream, + TileXRCcuTaskSubmitFn submitFn, + void* submitUserData, + TileXRCcuDirectSubmitReport* report) +{ + ResetSubmitReport(report); + if (submitTasks.empty()) { + if (report != nullptr) { + report->message = "missing prepared direct CCU submit tasks"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (stream == nullptr) { + if (report != nullptr) { + report->taskCount = static_cast(submitTasks.size()); + report->message = "missing runtime stream for direct CCU submit"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + uint32_t submitted = 0; + for (size_t i = 0; i < submitTasks.size(); ++i) { + TileXRCcuTask taskForSubmit = submitTasks[i]; + + TileXRCcuRuntimeSubmitReport runtimeReport; + const bool useDefaultSubmit = submitFn == nullptr; + const int ret = useDefaultSubmit ? + TileXRCcuSubmitTaskWithReport(taskForSubmit, stream, &runtimeReport) : + submitFn(taskForSubmit, stream, submitUserData); + if (useDefaultSubmit && runtimeReport.finalTaskCaptured) { + TraceFinalRuntimeTask(i, runtimeReport.finalTask); + } + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + report->taskCount = static_cast(submitTasks.size()); + report->submittedTaskCount = submitted; + report->message = FormatSubmitTaskFailure( + i, + ret, + taskForSubmit, + useDefaultSubmit ? &runtimeReport : nullptr); + } + return ret; + } + ++submitted; + } + + if (report != nullptr) { + report->submitted = true; + report->taskCount = static_cast(submitTasks.size()); + report->submittedTaskCount = submitted; + report->message = "direct CCU prepared tasks submitted"; + } + return TILEXR_SUCCESS; +} + +int RunDirectInstallAttemptImpl( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectMemoryCopySpec* memoryCopy, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + ResetReport(report); + if (attempt == nullptr) { + return Fail(nullptr, report, "missing output direct CCU install attempt"); + } + ClearAttempt(attempt); + + if (options.basicInfo == nullptr) { + return Fail(attempt, report, "missing direct CCU basic info"); + } + if (options.provider.empty()) { + return Fail(attempt, report, "missing direct CCU install provider"); + } + if (!options.offlineOnly && !HasRepositoryInstallInputs(options)) { + return Fail(attempt, report, "missing direct CCU repository install inputs"); + } + attempt->repositoryMemoryOps = options.repositoryMemoryOps; + attempt->repositoryMemoryUserData = options.repositoryMemoryUserData; + + TileXRCcuSpecsReport specsReport; + int ret = TileXRCcuDecodeBasicInfo(*options.basicInfo, &attempt->specInfo, &specsReport); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, specsReport.message); + } + + ret = TileXRCcuBuildResourceSpec( + attempt->specInfo, + options.missionStartId, + options.instructionStartId, + options.xnStartId, + options.ckeStartId, + options.channelStartId, + &attempt->resourceSpec, + &specsReport, + options.gsaStartId); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, specsReport.message); + } + attempt->resourceSpec.missionInstructionStartId = options.missionInstructionStartId; + ApplyRemoteXnOptions(options, &attempt->resourceSpec); + ApplySplitCkeOptions(options, &attempt->resourceSpec); + + attempt->resourceRequest.sqeArgCount = memoryCopy == nullptr ? options.sqeArgCount : 0U; + attempt->resourceRequest.syncResourceCount = memoryCopy == nullptr ? options.syncResourceCount : 1U; + attempt->resourceRequest.syncInstructionCount = memoryCopy == nullptr ? + options.syncInstructionCount : + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); + attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; + attempt->resourceRequest.barrierMode = options.barrierMode; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(attempt->resourceSpec) != TILEXR_SUCCESS) { + return Fail(attempt, report, "failed to initialize direct CCU resource allocator"); + } + + TileXRCcuResourceAllocatorReport allocatorReport; + ret = allocator.Allocate( + attempt->resourceRequest, + &attempt->plan, + &attempt->allocation, + &allocatorReport); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, allocatorReport.message); + } + + if (memoryCopy != nullptr) { + ret = ConfigureDirectMemoryCopyResources(options, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to configure direct CCU memory copy resources" : + report->message); + } + } + + ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to prepare direct CCU lower-layer install plan" : + report->message); + } + + ret = ReconcileProducerPlanWithLowerLayerProof(attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to reconcile direct CCU lower-layer peer resources" : + report->message); + } + if (memoryCopy == nullptr) { + ret = PopulateHcommStyleSqeTaskArgs(attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, "failed to populate direct CCU SQE task arguments"); + } + } + + TileXRCcuLaunchPackageReport packageReport; + ret = memoryCopy == nullptr ? + TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport) : + BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + memoryCopy == nullptr ? packageReport.message : + (report == nullptr || report->message.empty() ? + "failed to build direct CCU memory copy launch package" : + report->message)); + } + + TraceDirectInstallAttempt(*attempt); + + ret = TileXRCcuBindLaunchPackageInstallScope( + &attempt->package, + options.deviceId, + options.rank, + options.provider); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, "failed to bind direct CCU launch install scope"); + } + + TileXRCcuInstallManifestReport manifestReport; + ret = TileXRCcuBuildInstallManifest(attempt->package, &attempt->manifest, &manifestReport); + if (ret != TILEXR_SUCCESS) { + return Fail(attempt, report, manifestReport.message); + } + + TileXRCcuInstallRequest installRequest; + installRequest.package = &attempt->package; + installRequest.manifest = &attempt->manifest; + installRequest.deviceId = options.deviceId; + installRequest.rank = options.rank; + installRequest.provider = options.provider; + installRequest.offlineOnly = options.offlineOnly; + installRequest.driverAdapter = options.driverAdapter; + installRequest.repositoryMemoryOps = options.repositoryMemoryOps; + installRequest.repositoryMemoryUserData = options.repositoryMemoryUserData; + installRequest.repositoryInstallOptions = options.repositoryInstallOptions; + installRequest.repositoryReceipt = &attempt->repositoryReceipt; + installRequest.installOrder = options.installOrder; + installRequest.lowerLayerPlan = + options.prepareLowerLayerPlan == nullptr ? options.lowerLayerPlan : &attempt->preparedLowerLayerPlan; + + const int installRet = TileXRCcuInstallHardware( + installRequest, + &attempt->evidence, + &attempt->installReport); + + const int submitRet = TileXRCcuPrepareSubmitTasks( + attempt->package, + attempt->evidence, + &attempt->submitTasks, + &attempt->providerReport); + if (submitRet == TILEXR_SUCCESS) { + return ReturnWithAttemptStatus(TILEXR_SUCCESS, attempt, report); + } + if (installRet != TILEXR_SUCCESS) { + return ReturnWithAttemptStatus(installRet, attempt, report); + } + return ReturnWithAttemptStatus(submitRet, attempt, report); +} + +int TileXRCcuRunDirectInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + return RunDirectInstallAttemptImpl(options, nullptr, attempt, report); +} + +int TileXRCcuRunDirectMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectMemoryCopySpec& memoryCopy, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (memoryCopy.localAddr == 0 || memoryCopy.localToken == 0 || + memoryCopy.remoteAddr == 0 || memoryCopy.remoteToken == 0 || + memoryCopy.lengthBytes == 0) { + ResetReport(report); + ClearAttempt(attempt); + if (report != nullptr) { + report->message = "invalid direct CCU memory copy address/token inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return RunDirectInstallAttemptImpl(options, &memoryCopy, attempt, report); +} + +int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) +{ + if (attempt.repositoryReceipt.deviceInstructionPtr == nullptr) { + attempt.repositoryReceipt = TileXRCcuRepositoryInstallReceipt{}; + attempt.repositoryReleaseReport = TileXRCcuRepositoryReport{}; + attempt.repositoryReleaseReport.message = "ok"; + return TILEXR_SUCCESS; + } + return TileXRCcuReleaseRepositoryInstallReceipt( + attempt.repositoryReceipt, + attempt.repositoryMemoryOps, + attempt.repositoryMemoryUserData, + &attempt.repositoryReleaseReport); +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h new file mode 100644 index 00000000..60530cf7 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_DIRECT_ORCHESTRATOR_H +#define TILEXR_CCU_DIRECT_ORCHESTRATOR_H + +#include "ccu/tilexr_ccu_install_provider.h" +#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" +#include "ccu/tilexr_ccu_memory_program.h" +#include "ccu/tilexr_ccu_specs.h" + +#include +#include +#include + +namespace TileXR { + +using TileXRCcuLowerLayerPlanPrepareFn = int (*)( + const TileXRCcuResourceAllocation& allocation, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + void* userData); + +struct TileXRCcuDirectInstallOptions { + const TileXRCcuBasicInfo* basicInfo = nullptr; + uint32_t sqeArgCount = 0; + uint32_t syncResourceCount = 0; + uint32_t syncInstructionCount = 0; + uint32_t bindingsPerSyncResource = 1; + uint16_t missionStartId = 0; + uint16_t instructionStartId = 0; + uint16_t missionInstructionStartId = 0; + uint16_t xnStartId = 0; + uint16_t gsaStartId = 0; + uint16_t remoteXnStartId = 0; + uint16_t remoteXnCount = 0; + uint16_t ckeStartId = 0; + uint16_t channelStartId = 0; + uint16_t localWaitCkeStartId = 0; + uint16_t localWaitCkeCount = 0; + uint16_t remoteNotifyCkeStartId = 0; + uint16_t remoteNotifyCkeCount = 0; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; + uint32_t deviceId = 0; + uint32_t rank = 0; + std::string provider; + bool offlineOnly = false; + const TileXRCcuDriverAdapter* driverAdapter = nullptr; + TileXRCcuDeviceMemoryOps repositoryMemoryOps; + void* repositoryMemoryUserData = nullptr; + TileXRCcuRepositoryInstallOptions repositoryInstallOptions; + TileXRCcuRepositoryMemoryAllocMode repositoryMemoryAllocMode = TileXRCcuRepositoryMemoryAllocMode::Acl; + TileXRCcuInstallOrder installOrder = TileXRCcuInstallOrder::InstallLowerLayerFirst; + const TileXRCcuLowerLayerInstallPlan* lowerLayerPlan = nullptr; + TileXRCcuLowerLayerPlanPrepareFn prepareLowerLayerPlan = nullptr; + void* lowerLayerPlanUserData = nullptr; +}; + +struct TileXRCcuDirectMemoryCopySpec { + TileXRCcuMemoryCopyDirection direction = TileXRCcuMemoryCopyDirection::RemoteToLocal; + uint64_t localAddr = 0; + uint64_t localToken = 0; + uint64_t remoteAddr = 0; + uint64_t remoteToken = 0; + uint64_t lengthBytes = 0; +}; + +struct TileXRCcuDirectInstallAttempt { + TileXRCcuSpecInfo specInfo; + TileXRCcuResourceSpec resourceSpec; + TileXRCcuResourceRequest resourceRequest; + TileXRCcuResourceAllocation allocation; + TileXRCcuProducerPlan plan; + TileXRCcuLaunchPackage package; + TileXRCcuInstallManifest manifest; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuProviderReport providerReport; + TileXRCcuRepositoryInstallReceipt repositoryReceipt; + TileXRCcuRepositoryReport repositoryReleaseReport; + TileXRCcuDeviceMemoryOps repositoryMemoryOps; + void* repositoryMemoryUserData = nullptr; + TileXRCcuLowerLayerInstallPlan preparedLowerLayerPlan; + TileXRCcuLowerLayerPlanBuilderReport lowerLayerPlanReport; + std::vector submitTasks; +}; + +struct TileXRCcuDirectInstallReport { + bool pipelineBuilt = false; + bool installAttempted = false; + bool installSucceeded = false; + bool submitReady = false; + uint32_t requiredInstallSurfaceCount = 0; + uint32_t publicVerifiedInstallSurfaceCount = 0; + uint32_t missingInstallSurfaceCount = 0; + uint32_t taskCount = 0; + uint32_t submitTaskCount = 0; + std::string message; +}; + +struct TileXRCcuDirectSubmitReport { + bool submitted = false; + uint32_t taskCount = 0; + uint32_t submittedTaskCount = 0; + std::string message; +}; + +using TileXRCcuTaskSubmitFn = int (*)( + const TileXRCcuTask& task, + void* stream, + void* userData); + +int TileXRCcuRunDirectInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + +int TileXRCcuRunDirectMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectMemoryCopySpec& memoryCopy, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + +int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt); + +int TileXRCcuSubmitPreparedTasks( + const std::vector& submitTasks, + void* stream, + TileXRCcuTaskSubmitFn submitFn, + void* submitUserData, + TileXRCcuDirectSubmitReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_DIRECT_ORCHESTRATOR_H diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp new file mode 100644 index 00000000..c1291d26 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -0,0 +1,1529 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_direct_runtime.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +constexpr uint32_t TILEXR_CCU_DEFAULT_DIRECT_SQ_DEPTH = 8; +constexpr uint32_t TILEXR_CCU_DIRECT_CCUM_SQE_BYTES = 64; +constexpr uint32_t TILEXR_CCU_DIRECT_SQ_EBB_WORDS = 4; +constexpr uint32_t TILEXR_CCU_DIRECT_LOOP_JETTY_ID = 1024; +constexpr uint32_t TILEXR_CCU_DIRECT_LOOP_JETTY_CTX_ID = 0; +constexpr uint64_t TILEXR_CCU_V1_WQE_BASIC_BLOCK_OFFSET = TILEXR_CCU_V1_CCUM_OFFSET + 0x800000ULL; +constexpr uint64_t TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES = 256ULL * 1024ULL; +constexpr uint32_t TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH = 64; +constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_MAX_POLLS = 1000; +constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_SLEEP_US = 1000; +constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS = 8; +constexpr int TILEXR_CCU_DIRECT_MAX_RANK_SIZE = 128; +constexpr int TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL = 2; +constexpr int TILEXR_CCU_HCCP_ASYNC_EAGAIN = 128301; +constexpr const char* TILEXR_CCU_DIRECT_HDC_TYPE_ENV = "TILEXR_CCU_DIRECT_HDC_TYPE"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"; +constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV = + "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE"; +constexpr const char* TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE_ENV = + "TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE"; +constexpr const char* TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE_ENV = + "TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE"; +constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV = + "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE"; +constexpr const char* TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET_ENV = + "TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET"; +constexpr uint64_t TILEXR_CCU_UB_MEM_PAGE_SIZE = 4096ULL; +constexpr uint32_t TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT = 8; + +struct TileXRCcuEndpointTpHandleExchange { + uint64_t tpHandles[TILEXR_CCU_DIRECT_MAX_RANK_SIZE] = {}; + uint32_t psn = 0; +}; + +void ResetReport(TileXRCcuDirectRuntimeReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuDirectRuntimeReport{}; + } +} + +int Fail(TileXRCcuDirectRuntimeReport* report, const std::string& message, int code = TILEXR_ERROR_NOT_FOUND) +{ + if (report != nullptr) { + report->message = message; + } + return code; +} + +bool IsEmptyEid(const std::array& eid) +{ + return std::all_of(eid.begin(), eid.end(), [](uint8_t value) { return value == 0; }); +} + +std::array ReverseEndpointEid( + const std::array& eid) +{ + std::array reversed {}; + std::reverse_copy(eid.begin(), eid.end(), reversed.begin()); + return reversed; +} + +std::string FormatEndpointEid(const std::array& eid) +{ + std::ostringstream text; + text << std::hex << std::setfill('0'); + for (uint8_t byte : eid) { + text << std::setw(2) << static_cast(byte); + } + return text.str(); +} + +bool HasCompleteEndpointRoute(const TileXRCcuLowerLayerTransportRoute& route) +{ + return !IsEmptyEid(route.remoteEid) && + route.doorbellVa != 0 && + route.doorbellTokenId != 0 && + route.sqDepth != 0; +} + +bool UseImportedPeerEndpointRoute() +{ + const char* mode = std::getenv(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV); + return mode == nullptr || mode[0] == '\0' || std::strcmp(mode, "imported_peer") == 0; +} + +uint64_t SelectResourceWindowBytes(const TileXRCcuBasicInfo& basicInfo) +{ + (void)basicInfo; + return TILEXR_CCU_RESOURCE_WINDOW_BYTES; +} + +std::string RankEnvName(const char* base, int rank) +{ + std::ostringstream name; + name << base << "_RANK" << rank; + return name.str(); +} + +const char* SelectRankedEnv(const char* base, int rank) +{ + const std::string rankedName = RankEnvName(base, rank); + const char* ranked = std::getenv(rankedName.c_str()); + if (ranked != nullptr && ranked[0] != '\0') { + return ranked; + } + const char* value = std::getenv(base); + return value != nullptr && value[0] != '\0' ? value : nullptr; +} + +bool ParseUnsignedEnv(const char* value, uint64_t* out) +{ + if (value == nullptr || value[0] == '\0' || out == nullptr) { + return false; + } + errno = 0; + char* end = nullptr; + const unsigned long long parsed = std::strtoull(value, &end, 0); + if (errno != 0 || end == value || *end != '\0') { + return false; + } + *out = static_cast(parsed); + return true; +} + +uint64_t SelectRemoteCcuVaOffset() +{ + uint64_t value = 0; + return ParseUnsignedEnv(std::getenv(TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET_ENV), &value) ? value : 0; +} + +bool ParseU32RankedEnv(const char* base, int rank, uint32_t* out) +{ + uint64_t value = 0; + if (!ParseUnsignedEnv(SelectRankedEnv(base, rank), &value) || value > 0xffffffffULL || out == nullptr) { + return false; + } + *out = static_cast(value); + return true; +} + +bool HasRankedEnv(const char* base, int rank) +{ + return SelectRankedEnv(base, rank) != nullptr; +} + +bool IsRaCtxResourceWindowRegistrationMode() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV); + if (value == nullptr || value[0] == '\0') { + return false; + } + const std::string mode(value); + return mode == "ra_ctx" || mode == "ractx" || mode == "public_ra_ctx" || mode == "1"; +} + +bool TrustSyntheticEndpointRouteForDiagnostics() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE_ENV); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +bool TraceEndpointRoute() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE_ENV); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +void TraceEndpointRouteStep(const std::string& message) +{ + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute " << message << std::endl; + } +} + +bool IsRaCtxLoopEndpointRouteCollectionMode() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV); + if (value == nullptr || value[0] == '\0') { + return false; + } + const std::string mode(value); + return mode == "ra_ctx_loop" || mode == "ractx_loop" || mode == "public_ra_ctx_loop"; +} + +uint64_t AlignResourceWindowAddr(uint64_t addr) +{ + return addr & ~(TILEXR_CCU_UB_MEM_PAGE_SIZE - 1ULL); +} + +bool HasRaCtxResourceWindowSymbols(const TileXRCcuHccpLoader& loader) +{ + return loader.RaGetDevEidInfoNum != nullptr && + loader.RaGetDevEidInfoList != nullptr && + loader.RaCtxInit != nullptr && + loader.RaCtxDeinit != nullptr && + loader.RaCtxTokenIdAlloc != nullptr && + loader.RaCtxTokenIdFree != nullptr && + loader.RaGetSecRandom != nullptr && + loader.RaCtxLmemRegister != nullptr && + loader.RaCtxLmemUnregister != nullptr; +} + +bool HasRaCtxEndpointRouteSymbols(const TileXRCcuHccpLoader& loader) +{ + return loader.RaCtxCqCreate != nullptr && + loader.RaCtxCqDestroy != nullptr && + loader.RaCtxQpCreate != nullptr && + loader.RaCtxQpDestroy != nullptr && + loader.RaCtxQpImport != nullptr && + loader.RaCtxQpUnimport != nullptr && + loader.RaGetTpInfoListAsync != nullptr && + loader.RaGetAsyncReqResult != nullptr; +} + +uint32_t SelectEndpointRouteSqDepth() +{ + return TILEXR_CCU_DEFAULT_DIRECT_SQ_DEPTH; +} + +uint32_t SelectEndpointRouteSqBytes(uint32_t sqDepth) +{ + return sqDepth * TILEXR_CCU_DIRECT_SQ_EBB_WORDS * TILEXR_CCU_DIRECT_CCUM_SQE_BYTES; +} + +uint64_t SelectEndpointRouteSqVa(const TileXRCcuLocalResourceWindowInfo& localResourceWindow) +{ + return localResourceWindow.addr + TILEXR_CCU_V1_WQE_BASIC_BLOCK_OFFSET + + static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_CTX_ID) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; +} + +int WaitRaCtxAsyncRequest(TileXRCcuHccpLoader& loader, void* reqHandle) +{ + if (loader.RaGetAsyncReqResult == nullptr || reqHandle == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + for (uint32_t poll = 0; poll < TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_MAX_POLLS; ++poll) { + int reqResult = 0; + const int ret = loader.RaGetAsyncReqResult(reqHandle, &reqResult); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute asyncPoll poll=" << poll + << " ret=" << ret + << " reqResult=" << reqResult << std::endl; + } + if (ret == TILEXR_CCU_HCCP_ASYNC_EAGAIN) { + std::this_thread::sleep_for( + std::chrono::microseconds(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_SLEEP_US)); + continue; + } + if (ret != 0 || reqResult != 0) { + return TILEXR_ERROR_MKIRT; + } + return TILEXR_SUCCESS; + } + return TILEXR_ERROR_TIMEOUT; +} + +bool ApplyResourceWindowTokenOverride( + int rank, + TileXRCcuLocalResourceWindowInfo* window) +{ + if (window == nullptr || !HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_ENV, rank)) { + return false; + } + uint32_t tokenId = 0; + if (!ParseU32RankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_ENV, rank, &tokenId) || tokenId == 0) { + return false; + } + + uint32_t rawTokenId = tokenId; + if (HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID_ENV, rank) && + !ParseU32RankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID_ENV, rank, &rawTokenId)) { + return false; + } + + uint32_t tokenValue = 0; + if (HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_ENV, rank) && + !ParseU32RankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_ENV, rank, &tokenValue)) { + return false; + } else if (!HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_ENV, rank)) { + tokenValue = window->tokenValue; + } + + window->tokenId = tokenId; + window->rawTokenId = rawTokenId; + window->tokenValue = tokenValue; + return true; +} + +int SelectDirectCcuHdcType() +{ + uint64_t value = 0; + if (!ParseUnsignedEnv(std::getenv(TILEXR_CCU_DIRECT_HDC_TYPE_ENV), &value)) { + return TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2; + } + if (value == TILEXR_CCU_HDC_SERVICE_TYPE_RDMA || value == TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2) { + return static_cast(value); + } + return TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2; +} + +bool ParseHexBytePair(char high, char low, uint8_t* out) +{ + auto hexValue = [](char c) -> int { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + }; + const int highValue = hexValue(high); + const int lowValue = hexValue(low); + if (highValue < 0 || lowValue < 0 || out == nullptr) { + return false; + } + *out = static_cast((highValue << 4U) | lowValue); + return true; +} + +bool ParseEndpointEid(const char* value, std::array* eid) +{ + if (value == nullptr || eid == nullptr) { + return false; + } + std::string text(value); + if (text.size() >= 2 && text[0] == '0' && (text[1] == 'x' || text[1] == 'X')) { + text.erase(0, 2); + } + text.erase(std::remove_if(text.begin(), text.end(), [](char c) { + return c == ':' || c == '-' || c == '_' || c == ' '; + }), text.end()); + if (text.size() != TILEXR_CCU_EID_BYTES * 2U) { + return false; + } + std::array parsed {}; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + if (!ParseHexBytePair(text[i * 2U], text[i * 2U + 1U], &parsed[i])) { + return false; + } + } + *eid = parsed; + return true; +} + +int CollectLocalEndpointRouteFromEnv( + int rank, + TileXRCcuLowerLayerTransportRoute* route, + TileXRCcuDirectRuntimeReport* report) +{ + if (route == nullptr) { + return Fail(report, "missing output direct CCU local endpoint route", TILEXR_ERROR_PARA_CHECK_FAIL); + } + + const char* eidEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID", rank); + const char* tpnEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN", rank); + const char* doorbellVaEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA", rank); + const char* tokenIdEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_ID", rank); + const char* tokenValueEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_VALUE", rank); + const char* sqDepthEnv = SelectRankedEnv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_SQ_DEPTH", rank); + + if (eidEnv == nullptr && tpnEnv == nullptr && doorbellVaEnv == nullptr && + tokenIdEnv == nullptr && tokenValueEnv == nullptr && sqDepthEnv == nullptr) { + return Fail(report, "direct CCU local endpoint route collector is not configured", TILEXR_ERROR_NOT_FOUND); + } + if (eidEnv == nullptr || tpnEnv == nullptr || doorbellVaEnv == nullptr || + tokenIdEnv == nullptr || sqDepthEnv == nullptr) { + return Fail(report, "direct CCU local endpoint route env is incomplete", TILEXR_ERROR_PARA_CHECK_FAIL); + } + + TileXRCcuLowerLayerTransportRoute parsed; + if (!ParseEndpointEid(eidEnv, &parsed.remoteEid)) { + return Fail(report, "invalid direct CCU local endpoint EID env", TILEXR_ERROR_PARA_CHECK_FAIL); + } + uint64_t value = 0; + if (!ParseUnsignedEnv(tpnEnv, &value) || value == 0 || value > 0xffffffffULL) { + return Fail(report, "invalid direct CCU local endpoint TPN env", TILEXR_ERROR_PARA_CHECK_FAIL); + } + parsed.tpn = static_cast(value); + if (!ParseUnsignedEnv(doorbellVaEnv, &parsed.doorbellVa)) { + return Fail(report, "invalid direct CCU local endpoint doorbell VA env", TILEXR_ERROR_PARA_CHECK_FAIL); + } + if (!ParseUnsignedEnv(tokenIdEnv, &value) || value == 0 || value > 0xffffffffULL) { + return Fail(report, "invalid direct CCU local endpoint doorbell token id env", TILEXR_ERROR_PARA_CHECK_FAIL); + } + parsed.doorbellTokenId = static_cast(value); + if (tokenValueEnv != nullptr) { + if (!ParseUnsignedEnv(tokenValueEnv, &value) || value > 0xffffffffULL) { + return Fail(report, "invalid direct CCU local endpoint doorbell token value env", + TILEXR_ERROR_PARA_CHECK_FAIL); + } + parsed.doorbellTokenValue = static_cast(value); + } + if (!ParseUnsignedEnv(sqDepthEnv, &value) || value == 0 || value > 0xffffffffULL) { + return Fail(report, "invalid direct CCU local endpoint SQ depth env", TILEXR_ERROR_PARA_CHECK_FAIL); + } + parsed.sqDepth = static_cast(value); + parsed.endpointRouteVerified = true; + *route = parsed; + return TILEXR_SUCCESS; +} + +void FillProviderResourceWindow( + const TileXRCcuLocalResourceWindowInfo& localResourceWindow, + TileXRCcuEndpointRouteProviderResourceWindow* providerWindow) +{ + if (providerWindow == nullptr) { + return; + } + providerWindow->addr = localResourceWindow.addr; + providerWindow->bytes = localResourceWindow.bytes; + providerWindow->tokenId = localResourceWindow.tokenId; + providerWindow->rawTokenId = localResourceWindow.rawTokenId; + providerWindow->tokenValue = localResourceWindow.tokenValue; +} + +void CopyProviderRoute( + const TileXRCcuEndpointRouteProviderRoute& providerRoute, + TileXRCcuLowerLayerTransportRoute* route) +{ + if (route == nullptr) { + return; + } + TileXRCcuLowerLayerTransportRoute copied; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + copied.remoteEid[i] = providerRoute.remoteEid[i]; + } + copied.tpn = providerRoute.tpn; + copied.doorbellVa = providerRoute.doorbellVa; + copied.doorbellTokenId = providerRoute.doorbellTokenId; + copied.doorbellTokenValue = providerRoute.doorbellTokenValue; + copied.sqDepth = providerRoute.sqDepth; + copied.endpointRouteVerified = providerRoute.endpointRouteVerified; + *route = copied; +} + +} // namespace + +int TileXRCcuDirectRuntime::Init( + const TileXRCcuDirectRuntimeOptions& options, + TileXRCcuDirectRuntimeReport* report) +{ + ResetReport(report); + Shutdown(); + options_ = options; + + TileXRCcuHccpLoaderReport loaderReport; + int ret = loader_.Load(&loaderReport); + if (ret != TILEXR_SUCCESS) { + return Fail(report, loaderReport.message); + } + + ret = ResolveDevicePhyId(&devicePhyId_, report); + if (ret != TILEXR_SUCCESS) { + Shutdown(); + return ret; + } + const int hdcType = SelectDirectCcuHdcType(); + if (report != nullptr) { + report->hdcType = hdcType; + } + TileXRCcuHccpLoaderReport raReport; + ret = loader_.InitRaHdc(devicePhyId_, hdcType, true, &raReport); + if (ret != TILEXR_SUCCESS) { + Shutdown(); + return Fail(report, raReport.message, ret); + } + if (report != nullptr) { + report->raInitialized = true; + } + + TileXRCcuHccpLoaderReport tlvReport; + ret = loader_.InitCcuTlv(devicePhyId_, &tlvReport); + if (ret != TILEXR_SUCCESS) { + Shutdown(); + return Fail(report, tlvReport.message, ret); + } + + initialized_ = true; + if (report != nullptr) { + report->initialized = true; + report->raInitialized = true; + report->ccuTlvInitialized = true; + report->logicDevId = static_cast(options_.devId); + report->devicePhyId = devicePhyId_; + report->hdcType = hdcType; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +void TileXRCcuDirectRuntime::Shutdown() +{ + ReleaseLocalEndpointRoute(); + ReleaseRegisteredResourceWindow(); + loader_.Unload(); + cachedBasicInfo_ = TileXRCcuBasicInfo{}; + localResourceWindow_ = TileXRCcuLocalResourceWindowInfo{}; + localVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute{}; + cachedBasicInfoValid_ = false; + resourceWindowRegistered_ = false; + localVerifiedEndpointRouteValid_ = false; + endpointChanHandle_ = nullptr; + endpointCqHandle_ = nullptr; + endpointQpHandle_ = nullptr; + endpointRemoteQpHandle_ = nullptr; + endpointPeerRemoteQpHandles_.clear(); + endpointQpKey_ = TileXRCcuHccpQpKey{}; + endpointQpKeyValid_ = false; + endpointRouteBound_ = false; + endpointPsn_ = 1; + devicePhyId_ = 0; + initialized_ = false; +} + +bool TileXRCcuDirectRuntime::IsAvailable() const +{ + return initialized_ && loader_.IsLoaded() && loader_.RaCustomChannel != nullptr; +} + +int TileXRCcuDirectRuntime::ResolveDevicePhyId(uint32_t* devicePhyId, TileXRCcuDirectRuntimeReport* report) const +{ + if (devicePhyId == nullptr) { + return Fail(report, "missing output CCU physical device id", TILEXR_ERROR_PARA_CHECK_FAIL); + } + TileXRCcuHccpLoaderReport loaderReport; + const int ret = loader_.ResolveDevicePhyId( + static_cast(options_.devId), + devicePhyId, + &loaderReport); + if (ret != TILEXR_SUCCESS) { + return Fail(report, loaderReport.message.empty() ? + "failed to resolve CCU physical device id" : loaderReport.message, ret); + } + if (report != nullptr) { + report->logicDevId = static_cast(options_.devId); + report->devicePhyId = *devicePhyId; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::CreateDriverAdapter( + TileXRCcuDriverAdapter* adapter, + TileXRCcuDriverAdapterReport* report) +{ + if (!IsAvailable()) { + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + report->message = "direct CCU runtime is unavailable"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuRaCustomChannelProviderReport providerReport; + int ret = raCustomChannelProvider_.Init(devicePhyId_, loader_.RaCustomChannel, &providerReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + report->message = providerReport.message; + } + return ret; + } + return raCustomChannelProvider_.CreateAdapter(adapter, report); +} + +int TileXRCcuDirectRuntime::QueryBasicInfo( + uint8_t dieId, + TileXRCcuBasicInfo* basicInfo, + TileXRCcuDriverAdapterReport* report) +{ + if (basicInfo == nullptr) { + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + report->message = "missing output CCU basic info"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuDriverAdapter adapter; + int ret = CreateDriverAdapter(&adapter, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + bool enabled = false; + ret = adapter.GetDieEnabled(dieId, &enabled, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + if (!enabled) { + if (report != nullptr) { + report->message = "direct CCU die is not enabled"; + } + return TILEXR_ERROR_NOT_FOUND; + } + ret = adapter.GetBasicInfo(dieId, basicInfo, report); + if (ret == TILEXR_SUCCESS) { + cachedBasicInfo_ = *basicInfo; + cachedBasicInfoValid_ = true; + } + return ret; +} + +int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer(uint64_t resourceAddr) +{ + if (!IsAvailable()) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + ReleaseRegisteredResourceWindow(); + if (!cachedBasicInfoValid_ || resourceAddr == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (cachedBasicInfo_.resourceAddr != 0 && cachedBasicInfo_.resourceAddr != resourceAddr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint64_t resourceBytes = SelectResourceWindowBytes(cachedBasicInfo_); + if (IsRaCtxResourceWindowRegistrationMode()) { + return RegisterCcuResourceRmaBufferWithRaCtx(resourceAddr, resourceBytes); + } + + localResourceWindow_ = TileXRCcuLocalResourceWindowInfo{}; + localResourceWindow_.addr = resourceAddr; + localResourceWindow_.bytes = resourceBytes; + localResourceWindow_.tokenId = cachedBasicInfo_.msidToken.tokenId; + localResourceWindow_.rawTokenId = cachedBasicInfo_.msidToken.tokenId; + localResourceWindow_.tokenValue = cachedBasicInfo_.msidToken.tokenValue; + if (HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_ENV, options_.rank) && + !ApplyResourceWindowTokenOverride(options_.rank, &localResourceWindow_)) { + localResourceWindow_ = TileXRCcuLocalResourceWindowInfo{}; + resourceWindowRegistered_ = false; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + resourceWindowRegistered_ = true; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( + uint64_t resourceAddr, + uint64_t resourceBytes) +{ + if (!HasRaCtxResourceWindowSymbols(loader_)) { + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuRaInfo raInfo {}; + raInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + raInfo.phyId = devicePhyId_; + + uint32_t eidNum = 0; + int ret = loader_.RaGetDevEidInfoNum(raInfo, &eidNum); + if (ret != 0 || eidNum == 0) { + return ret == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_ERROR_MKIRT; + } + + std::vector eidInfos(eidNum); + uint32_t queriedEidNum = eidNum; + ret = loader_.RaGetDevEidInfoList(raInfo, eidInfos.data(), &queriedEidNum); + if (ret != 0 || queriedEidNum == 0) { + return ret == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_ERROR_MKIRT; + } + + void* ctxHandle = nullptr; + void* tokenIdHandle = nullptr; + void* lmemHandle = nullptr; + + TileXRCcuHccpCtxInitCfg ctxCfg {}; + ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; + ctxCfg.rdma.disabledLiteThread = false; + TileXRCcuHccpCtxInitAttr ctxAttr {}; + ctxAttr.phyId = devicePhyId_; + ctxAttr.ub.eidIndex = eidInfos[0].eidIndex; + ctxAttr.ub.eid = eidInfos[0].eid; + + ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); + if (ret != 0 || ctxHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuHccpTokenId allocatedToken {}; + ret = loader_.RaCtxTokenIdAlloc(ctxHandle, &allocatedToken, &tokenIdHandle); + if (ret != 0 || tokenIdHandle == nullptr) { + if (loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(ctxHandle); + } + return TILEXR_ERROR_MKIRT; + } + + uint32_t tokenValue = 0; + TileXRCcuRaInfo randomInfo {}; + randomInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + randomInfo.phyId = devicePhyId_; + ret = loader_.RaGetSecRandom(&randomInfo, &tokenValue); + if (ret != 0) { + if (loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(ctxHandle, tokenIdHandle); + } + if (loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(ctxHandle); + } + return TILEXR_ERROR_MKIRT; + } + + const uint64_t alignedAddr = AlignResourceWindowAddr(resourceAddr); + const uint64_t alignedBytes = resourceBytes + (resourceAddr - alignedAddr); + TileXRCcuHccpMrRegInfo mr {}; + mr.in.mem.addr = alignedAddr; + mr.in.mem.size = alignedBytes; + mr.in.ub.flags.value = 0; + mr.in.ub.flags.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + mr.in.ub.flags.bs.tokenIdValid = 1; + mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; + mr.in.ub.flags.bs.nonPin = 1; + mr.in.ub.tokenValue = tokenValue; + mr.in.ub.tokenIdHandle = tokenIdHandle; + + ret = loader_.RaCtxLmemRegister(ctxHandle, &mr, &lmemHandle); + if (ret != 0 || lmemHandle == nullptr) { + if (lmemHandle != nullptr && loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister(ctxHandle, lmemHandle); + } + if (loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(ctxHandle, tokenIdHandle); + } + if (loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(ctxHandle); + } + return TILEXR_ERROR_MKIRT; + } + + const uint32_t rawTokenId = mr.out.ub.tokenId != 0 ? mr.out.ub.tokenId : allocatedToken.tokenId; + localResourceWindow_ = TileXRCcuLocalResourceWindowInfo{}; + localResourceWindow_.addr = resourceAddr; + localResourceWindow_.bytes = resourceBytes; + localResourceWindow_.rawTokenId = rawTokenId; + localResourceWindow_.tokenId = rawTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + localResourceWindow_.tokenValue = tokenValue; + localResourceWindow_.targetSegHandle = mr.out.ub.targetSegHandle; + localResourceWindow_.raCtxHandle = ctxHandle; + localResourceWindow_.tokenIdHandle = tokenIdHandle; + localResourceWindow_.lmemHandle = lmemHandle; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + localResourceWindow_.eid[i] = eidInfos[0].eid.raw[i]; + } + localResourceWindow_.eidIndex = eidInfos[0].eidIndex; + localResourceWindow_.raCtxRegistered = true; + resourceWindowRegistered_ = true; + return TILEXR_SUCCESS; +} + +void TileXRCcuDirectRuntime::ReleaseRegisteredResourceWindow() +{ + ReleaseLocalEndpointRoute(); + if (localResourceWindow_.raCtxRegistered) { + if (localResourceWindow_.lmemHandle != nullptr && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister(localResourceWindow_.raCtxHandle, localResourceWindow_.lmemHandle); + } + if (localResourceWindow_.tokenIdHandle != nullptr && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(localResourceWindow_.raCtxHandle, localResourceWindow_.tokenIdHandle); + } + if (localResourceWindow_.raCtxHandle != nullptr && loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(localResourceWindow_.raCtxHandle); + } + } + localResourceWindow_ = TileXRCcuLocalResourceWindowInfo{}; + localVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute{}; + resourceWindowRegistered_ = false; + localVerifiedEndpointRouteValid_ = false; +} + +void TileXRCcuDirectRuntime::ReleasePeerEndpointImports() +{ + if (localResourceWindow_.raCtxHandle != nullptr && loader_.RaCtxQpUnimport != nullptr) { + for (void* handle : endpointPeerRemoteQpHandles_) { + if (handle != nullptr) { + (void)loader_.RaCtxQpUnimport(localResourceWindow_.raCtxHandle, handle); + } + } + } + endpointPeerRemoteQpHandles_.clear(); +} + +void TileXRCcuDirectRuntime::ReleaseLocalEndpointRoute() +{ + ReleasePeerEndpointImports(); + if (endpointRouteBound_ && endpointQpHandle_ != nullptr && loader_.RaCtxQpUnbind != nullptr) { + (void)loader_.RaCtxQpUnbind(endpointQpHandle_); + } + if (endpointRemoteQpHandle_ != nullptr && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxQpUnimport != nullptr) { + (void)loader_.RaCtxQpUnimport(localResourceWindow_.raCtxHandle, endpointRemoteQpHandle_); + } + if (endpointQpHandle_ != nullptr && loader_.RaCtxQpDestroy != nullptr) { + (void)loader_.RaCtxQpDestroy(endpointQpHandle_); + } + if (endpointCqHandle_ != nullptr && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxCqDestroy != nullptr) { + (void)loader_.RaCtxCqDestroy(localResourceWindow_.raCtxHandle, endpointCqHandle_); + } + if (endpointChanHandle_ != nullptr && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxChanDestroy != nullptr) { + (void)loader_.RaCtxChanDestroy(localResourceWindow_.raCtxHandle, endpointChanHandle_); + } + endpointChanHandle_ = nullptr; + endpointCqHandle_ = nullptr; + endpointQpHandle_ = nullptr; + endpointRemoteQpHandle_ = nullptr; + endpointQpKey_ = TileXRCcuHccpQpKey{}; + endpointQpKeyValid_ = false; + endpointRouteBound_ = false; + localVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute{}; + localVerifiedEndpointRouteValid_ = false; +} + +int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtx(TileXRCcuLowerLayerTransportRoute* route) +{ + if (route == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *route = TileXRCcuLowerLayerTransportRoute{}; + int lastRet = TILEXR_ERROR_NOT_FOUND; + for (uint32_t attempt = 0; attempt < TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS; ++attempt) { + TileXRCcuLowerLayerTransportRoute attemptRoute; + bool asyncWaitFailed = false; + const int ret = CollectLocalEndpointRouteWithRaCtxOnce(&attemptRoute, &asyncWaitFailed); + if (ret == TILEXR_SUCCESS) { + *route = attemptRoute; + return TILEXR_SUCCESS; + } + lastRet = ret; + if (!asyncWaitFailed || attempt + 1 >= TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS) { + return ret; + } + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute retryAfterAsyncFailure" + << " attempt=" << (attempt + 1) + << " ret=" << ret << std::endl; + } + } + return lastRet; +} + +int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( + TileXRCcuLowerLayerTransportRoute* route, + bool* asyncWaitFailed) +{ + if (route == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (asyncWaitFailed != nullptr) { + *asyncWaitFailed = false; + } + *route = TileXRCcuLowerLayerTransportRoute{}; + if (!localResourceWindow_.raCtxRegistered || + localResourceWindow_.raCtxHandle == nullptr || + localResourceWindow_.tokenIdHandle == nullptr || + IsEmptyEid(localResourceWindow_.eid) || + !HasRaCtxEndpointRouteSymbols(loader_)) { + TraceEndpointRouteStep("raCtxLoop precheck failed"); + return TILEXR_ERROR_NOT_FOUND; + } + + ReleaseLocalEndpointRoute(); + const uint32_t sqDepth = SelectEndpointRouteSqDepth(); + const uint64_t sqVa = SelectEndpointRouteSqVa(localResourceWindow_); + const uint32_t sqBytes = SelectEndpointRouteSqBytes(sqDepth); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute begin" + << " ctx=" << localResourceWindow_.raCtxHandle + << " sqDepth=" << sqDepth + << " sqVa=0x" << std::hex << sqVa + << " sqBytes=0x" << sqBytes + << std::dec << std::endl; + } + + TileXRCcuHccpCqInfo cqInfo {}; + cqInfo.in.chanHandle = nullptr; + cqInfo.in.depth = TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH; + cqInfo.in.ub.userCtx = 0; + cqInfo.in.ub.mode = TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL; + cqInfo.in.ub.ceqn = 0; + cqInfo.in.ub.flag.value = 0; + int ret = loader_.RaCtxCqCreate(localResourceWindow_.raCtxHandle, &cqInfo, &endpointCqHandle_); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute cqCreate ret=" << ret + << " cq=" << endpointCqHandle_ << std::endl; + } + if (ret != 0 || endpointCqHandle_ == nullptr) { + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuHccpQpCreateAttr qpAttr {}; + qpAttr.scqHandle = endpointCqHandle_; + qpAttr.rcqHandle = endpointCqHandle_; + qpAttr.srqHandle = endpointCqHandle_; + qpAttr.sqDepth = sqDepth; + qpAttr.rqDepth = TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT; + qpAttr.transportMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + qpAttr.ub.mode = static_cast(TILEXR_CCU_HCCP_JETTY_MODE_CCU); + qpAttr.ub.jettyId = TILEXR_CCU_DIRECT_LOOP_JETTY_ID; + qpAttr.ub.tokenIdHandle = localResourceWindow_.tokenIdHandle; + qpAttr.ub.tokenValue = localResourceWindow_.tokenValue; + qpAttr.ub.flag.value = 0; + qpAttr.ub.flag.bs.shareJfr = 1; + qpAttr.ub.jfsFlag.bs.errorSuspend = 1; + qpAttr.ub.priority = 2; + qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; + qpAttr.ub.extMode.cstmFlag.value = 0; + qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; + qpAttr.ub.extMode.sq.buffVa = sqVa; + qpAttr.ub.extMode.sq.buffSize = sqBytes; + qpAttr.ub.extMode.sqebbNum = sqDepth; + + TileXRCcuHccpQpCreateInfo qpInfo {}; + ret = loader_.RaCtxQpCreate(localResourceWindow_.raCtxHandle, &qpAttr, &qpInfo, &endpointQpHandle_); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute qpCreate ret=" << ret + << " qp=" << endpointQpHandle_ + << " keySize=" << static_cast(qpInfo.key.size) + << " dbAddr=0x" << std::hex << qpInfo.ub.dbAddr + << " dbTokenId=0x" << qpInfo.ub.dbTokenId + << std::dec << std::endl; + } + if (ret != 0 || endpointQpHandle_ == nullptr) { + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_MKIRT; + } + endpointQpKey_ = qpInfo.key; + endpointQpKeyValid_ = qpInfo.key.size != 0; + + TileXRCcuHccpGetTpCfg tpCfg {}; + tpCfg.flag.value = 0; + tpCfg.flag.bs.rtp = 1; + tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + tpCfg.localEid.raw[i] = localResourceWindow_.eid[i]; + tpCfg.peerEid.raw[i] = localResourceWindow_.eid[i]; + } + TileXRCcuHccpTpInfo tpInfo {}; + uint32_t tpInfoNum = 1; + void* reqHandle = nullptr; + ret = loader_.RaGetTpInfoListAsync( + localResourceWindow_.raCtxHandle, + &tpCfg, + &tpInfo, + &tpInfoNum, + &reqHandle); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute getTpInfoAsync ret=" << ret + << " req=" << reqHandle + << " num=" << tpInfoNum + << " tpHandle=0x" << std::hex << tpInfo.tpHandle + << std::dec << std::endl; + } + if (ret != 0 || reqHandle == nullptr || tpInfoNum == 0) { + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + ReleaseLocalEndpointRoute(); + if (asyncWaitFailed != nullptr) { + *asyncWaitFailed = true; + } + return ret; + } + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute getTpInfoDone" + << " tpHandle=0x" << std::hex << tpInfo.tpHandle + << std::dec << std::endl; + } + if (tpInfo.tpHandle == 0) { + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuHccpQpImportInfo importInfo {}; + importInfo.in.key = qpInfo.key; + importInfo.in.ub.mode = TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP; + importInfo.in.ub.tokenValue = localResourceWindow_.tokenValue; + importInfo.in.ub.policy = TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR; + importInfo.in.ub.type = TILEXR_CCU_HCCP_TARGET_TYPE_JETTY; + importInfo.in.ub.flag.value = 0; + importInfo.in.ub.flag.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + importInfo.in.ub.expImportCfg.tpHandle = tpInfo.tpHandle; + importInfo.in.ub.expImportCfg.peerTpHandle = tpInfo.tpHandle; + importInfo.in.ub.expImportCfg.txPsn = endpointPsn_; + importInfo.in.ub.expImportCfg.rxPsn = endpointPsn_; + importInfo.in.ub.tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; + + ret = loader_.RaCtxQpImport(localResourceWindow_.raCtxHandle, &importInfo, &endpointRemoteQpHandle_); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute qpImport ret=" << ret + << " remoteQp=" << endpointRemoteQpHandle_ + << " tpn=0x" << std::hex << importInfo.out.ub.tpn + << std::dec << std::endl; + } + if (ret != 0 || endpointRemoteQpHandle_ == nullptr) { + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_MKIRT; + } + ++endpointPsn_; + + TileXRCcuLowerLayerTransportRoute collected; + collected.remoteEid = localResourceWindow_.eid; + collected.tpn = importInfo.out.ub.tpn; + collected.doorbellVa = qpInfo.ub.dbAddr; + collected.doorbellTokenId = qpInfo.ub.dbTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + collected.doorbellTokenValue = localResourceWindow_.tokenValue; + collected.sqDepth = sqDepth; + collected.endpointRouteVerified = true; + if (!HasCompleteEndpointRoute(collected)) { + TraceEndpointRouteStep("collected route incomplete"); + ReleaseLocalEndpointRoute(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute verified" + << " tpn=0x" << std::hex << collected.tpn + << " doorbellVa=0x" << collected.doorbellVa + << " doorbellTokenId=0x" << collected.doorbellTokenId + << std::dec + << " sqDepth=" << collected.sqDepth << std::endl; + } + *route = collected; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRuntimeReport* report) +{ + ResetReport(report); + localVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute{}; + localVerifiedEndpointRouteValid_ = false; + if (!resourceWindowRegistered_) { + return Fail(report, "direct CCU resource window is not registered for endpoint route collection", + TILEXR_ERROR_NOT_INITIALIZED); + } + + TileXRCcuLowerLayerTransportRoute route; + int ret = TILEXR_ERROR_NOT_FOUND; + if (options_.localEndpointRouteCollector != nullptr) { + ret = options_.localEndpointRouteCollector( + devicePhyId_, + localResourceWindow_, + &route, + options_.localEndpointRouteCollectorUserData); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "direct CCU local endpoint route collector failed", ret); + } + } else if (IsRaCtxLoopEndpointRouteCollectionMode()) { + ret = CollectLocalEndpointRouteWithRaCtx(&route); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "direct CCU RA ctx loop endpoint route collection failed", ret); + } + } else { + TileXRCcuHccpLoaderReport providerReport; + ret = loader_.LoadEndpointRouteProviderFromEnv(&providerReport); + if (ret == TILEXR_SUCCESS && loader_.CollectLocalEndpointRoute != nullptr) { + TileXRCcuEndpointRouteProviderResourceWindow providerWindow; + FillProviderResourceWindow(localResourceWindow_, &providerWindow); + TileXRCcuEndpointRouteProviderRoute providerRoute; + ret = loader_.CollectLocalEndpointRoute(devicePhyId_, &providerWindow, &providerRoute); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "direct CCU endpoint route provider failed", ret); + } + CopyProviderRoute(providerRoute, &route); + } else if (ret == TILEXR_ERROR_NOT_FOUND && !providerReport.endpointRouteProviderConfigured) { + const bool canCollectRaCtxRoute = + IsRaCtxResourceWindowRegistrationMode() && + localResourceWindow_.raCtxRegistered && + HasRaCtxEndpointRouteSymbols(loader_); + if (canCollectRaCtxRoute) { + ret = CollectLocalEndpointRouteWithRaCtx(&route); + if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { + return Fail(report, "direct CCU RA ctx loop endpoint route collection failed", ret); + } + } + if (ret != TILEXR_SUCCESS) { + ret = CollectLocalEndpointRouteFromEnv(options_.rank, &route, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + } else { + return Fail(report, providerReport.message.empty() ? + "direct CCU endpoint route provider failed to load" : providerReport.message, ret); + } + } + + ret = ConfigureLocalVerifiedEndpointRoute(route); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "direct CCU local endpoint route collector returned incomplete route", ret); + } + if (report != nullptr) { + report->initialized = initialized_; + report->logicDevId = static_cast(options_.devId); + report->devicePhyId = devicePhyId_; + report->message = "direct CCU local endpoint route collected"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::ConfigureLocalVerifiedEndpointRoute( + const TileXRCcuLowerLayerTransportRoute& route) +{ + if (!route.endpointRouteVerified || !HasCompleteEndpointRoute(route)) { + localVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute{}; + localVerifiedEndpointRouteValid_ = false; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + localVerifiedEndpointRoute_ = route; + localVerifiedEndpointRouteValid_ = true; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo* info) const +{ + if (info == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *info = TileXRCcuLocalResourceWindowInfo{}; + if (!resourceWindowRegistered_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + *info = localResourceWindow_; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::QueryTpHandleForPeer( + const std::array& peerEid, + uint64_t* tpHandle) +{ + if (tpHandle == nullptr || IsEmptyEid(localResourceWindow_.eid) || IsEmptyEid(peerEid) || + localResourceWindow_.raCtxHandle == nullptr || loader_.RaGetTpInfoListAsync == nullptr || + loader_.RaGetAsyncReqResult == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *tpHandle = 0; + + TileXRCcuHccpGetTpCfg tpCfg {}; + tpCfg.flag.value = 0; + tpCfg.flag.bs.rtp = 1; + tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + tpCfg.localEid.raw[i] = localResourceWindow_.eid[i]; + tpCfg.peerEid.raw[i] = peerEid[i]; + } + + TileXRCcuHccpTpInfo tpInfo {}; + uint32_t tpInfoNum = 1; + void* reqHandle = nullptr; + const int ret = loader_.RaGetTpInfoListAsync( + localResourceWindow_.raCtxHandle, + &tpCfg, + &tpInfo, + &tpInfoNum, + &reqHandle); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute peerTpInfoAsync" + << " ret=" << ret + << " req=" << reqHandle + << " num=" << tpInfoNum + << " tpHandle=0x" << std::hex << tpInfo.tpHandle + << std::dec << std::endl; + } + if (ret != 0 || reqHandle == nullptr || tpInfoNum == 0) { + return TILEXR_ERROR_MKIRT; + } + const int waitRet = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (waitRet != TILEXR_SUCCESS) { + return waitRet; + } + if (tpInfo.tpHandle == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + *tpHandle = tpInfo.tpHandle; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::ImportPeerEndpointRoute( + const TileXRCcuResourceWindowExchange& peerWindow, + uint64_t localTpHandle, + uint64_t peerTpHandle, + uint32_t localPsn, + uint32_t peerPsn, + TileXRCcuLowerLayerTransportRoute* importedRoute) +{ + if (importedRoute == nullptr || peerWindow.qpKey.size == 0 || localTpHandle == 0 || peerTpHandle == 0 || + localResourceWindow_.raCtxHandle == nullptr || loader_.RaCtxQpImport == nullptr || + !localVerifiedEndpointRouteValid_ || !HasCompleteEndpointRoute(localVerifiedEndpointRoute_)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *importedRoute = TileXRCcuLowerLayerTransportRoute{}; + + TileXRCcuHccpQpImportInfo importInfo {}; + importInfo.in.key = peerWindow.qpKey; + importInfo.in.ub.mode = TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP; + importInfo.in.ub.tokenValue = peerWindow.tokenValue; + importInfo.in.ub.policy = TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR; + importInfo.in.ub.type = TILEXR_CCU_HCCP_TARGET_TYPE_JETTY; + importInfo.in.ub.flag.value = 0; + importInfo.in.ub.flag.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + importInfo.in.ub.expImportCfg.tpHandle = localTpHandle; + importInfo.in.ub.expImportCfg.peerTpHandle = peerTpHandle; + importInfo.in.ub.expImportCfg.txPsn = localPsn; + importInfo.in.ub.expImportCfg.rxPsn = peerPsn; + importInfo.in.ub.tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; + + void* remoteQpHandle = nullptr; + const int ret = loader_.RaCtxQpImport(localResourceWindow_.raCtxHandle, &importInfo, &remoteQpHandle); + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute peerQpImport" + << " ret=" << ret + << " remoteQp=" << remoteQpHandle + << " localTpHandle=0x" << std::hex << localTpHandle + << " peerTpHandle=0x" << peerTpHandle + << " tpn=0x" << importInfo.out.ub.tpn + << std::dec + << " localPsn=" << localPsn + << " peerPsn=" << peerPsn + << " peerEid=" << FormatEndpointEid(peerWindow.remoteEid) + << " channelEid=" << FormatEndpointEid(ReverseEndpointEid(peerWindow.remoteEid)) + << std::endl; + } + if (ret != 0 || remoteQpHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + endpointPeerRemoteQpHandles_.push_back(remoteQpHandle); + + importedRoute->remoteEid = ReverseEndpointEid(peerWindow.remoteEid); + importedRoute->tpn = importInfo.out.ub.tpn; + importedRoute->doorbellVa = localVerifiedEndpointRoute_.doorbellVa; + importedRoute->doorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId; + importedRoute->doorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue; + importedRoute->sqDepth = localVerifiedEndpointRoute_.sqDepth; + importedRoute->endpointRouteVerified = true; + return HasCompleteEndpointRoute(*importedRoute) ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers(std::vector* buffers) +{ + if (buffers == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + buffers->clear(); + if (!resourceWindowRegistered_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (options_.rankSize <= 1) { + return TILEXR_SUCCESS; + } + if (options_.rank < 0 || options_.rank >= options_.rankSize || + options_.rankSize > TILEXR_CCU_DIRECT_MAX_RANK_SIZE || + options_.allGather == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + ReleasePeerEndpointImports(); + + TileXRCcuResourceWindowExchange local { + localResourceWindow_.addr, + localResourceWindow_.bytes, + localResourceWindow_.tokenId, + localResourceWindow_.rawTokenId, + localResourceWindow_.tokenValue, + }; + if (localVerifiedEndpointRouteValid_) { + local.remoteEid = localVerifiedEndpointRoute_.remoteEid; + local.tpn = localVerifiedEndpointRoute_.tpn; + local.doorbellVa = localVerifiedEndpointRoute_.doorbellVa; + local.doorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId; + local.doorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue; + local.sqDepth = localVerifiedEndpointRoute_.sqDepth; + local.qpKey = endpointQpKey_; + local.psn = endpointPsn_; + local.endpointRouteVerified = true; + local.channelResourceOwnerVerified = localVerifiedEndpointRoute_.channelResourceOwnerVerified; + local.transportResourceExchangeVerified = localVerifiedEndpointRoute_.transportResourceExchangeVerified; + } + std::vector all(static_cast(options_.rankSize)); + const int exportRet = options_.allGather( + &local, + sizeof(local), + all.data(), + options_.allGatherUserData); + if (exportRet != TILEXR_SUCCESS) { + return exportRet; + } + + const bool useImportedPeerRoute = UseImportedPeerEndpointRoute(); + const bool canImportPeerRoutes = + useImportedPeerRoute && + endpointQpKeyValid_ && + localVerifiedEndpointRouteValid_ && + HasCompleteEndpointRoute(localVerifiedEndpointRoute_) && + localResourceWindow_.raCtxHandle != nullptr && + loader_.RaCtxQpImport != nullptr && + loader_.RaGetTpInfoListAsync != nullptr && + loader_.RaGetAsyncReqResult != nullptr; + TileXRCcuEndpointTpHandleExchange localTpHandles {}; + std::vector allTpHandles; + bool peerTpHandlesReady = false; + if (canImportPeerRoutes) { + localTpHandles.psn = endpointPsn_; + for (int peer = 0; peer < options_.rankSize; ++peer) { + if (peer == options_.rank) { + continue; + } + const auto& peerWindow = all[peer]; + if (peerWindow.addr == 0 || IsEmptyEid(peerWindow.remoteEid)) { + continue; + } + uint64_t tpHandle = 0; + if (QueryTpHandleForPeer(peerWindow.remoteEid, &tpHandle) == TILEXR_SUCCESS) { + localTpHandles.tpHandles[peer] = tpHandle; + } + } + allTpHandles.resize(static_cast(options_.rankSize)); + const int tpExchangeRet = options_.allGather( + &localTpHandles, + sizeof(localTpHandles), + allTpHandles.data(), + options_.allGatherUserData); + peerTpHandlesReady = tpExchangeRet == TILEXR_SUCCESS; + } + + const uint64_t remoteCcuVaOffset = SelectRemoteCcuVaOffset(); + buffers->reserve(static_cast(options_.rankSize - 1)); + for (int peer = 0; peer < options_.rankSize; ++peer) { + if (peer == options_.rank) { + continue; + } + const auto& peerWindow = all[peer]; + if (peerWindow.addr == 0) { + buffers->clear(); + return TILEXR_ERROR_NOT_FOUND; + } + TileXRCcuRemoteCcuBufferInfo remote; + if (remoteCcuVaOffset > std::numeric_limits::max() - peerWindow.addr) { + buffers->clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + remote.remoteCcuVa = peerWindow.addr + remoteCcuVaOffset; + remote.peerRank = static_cast(peer); + remote.memoryTokenId = peerWindow.tokenId; + remote.rawMemoryTokenId = peerWindow.rawTokenId; + remote.memoryTokenValue = peerWindow.tokenValue; + remote.remoteEid = ReverseEndpointEid(peerWindow.remoteEid); + TileXRCcuLowerLayerTransportRoute importedRoute; + const uint64_t localTpForPeer = peerTpHandlesReady ? + localTpHandles.tpHandles[peer] : 0; + const uint64_t peerTpForLocal = peerTpHandlesReady ? + allTpHandles[static_cast(peer)].tpHandles[options_.rank] : 0; + const bool importedPeerRoute = + peerTpHandlesReady && + peerTpForLocal != 0 && + localTpForPeer != 0 && + ImportPeerEndpointRoute( + peerWindow, + localTpForPeer, + peerTpForLocal, + localTpHandles.psn, + allTpHandles[static_cast(peer)].psn, + &importedRoute) == TILEXR_SUCCESS; + if (importedPeerRoute) { + remote.remoteEid = importedRoute.remoteEid; + remote.tpn = importedRoute.tpn; + remote.doorbellVa = importedRoute.doorbellVa; + remote.doorbellTokenId = importedRoute.doorbellTokenId; + remote.doorbellTokenValue = importedRoute.doorbellTokenValue; + remote.sqDepth = importedRoute.sqDepth; + } else { + remote.tpn = peerWindow.tpn; + remote.doorbellVa = peerWindow.doorbellVa; + remote.doorbellTokenId = peerWindow.doorbellTokenId; + remote.doorbellTokenValue = peerWindow.doorbellTokenValue; + remote.sqDepth = peerWindow.sqDepth; + if (!useImportedPeerRoute && TraceEndpointRoute() && peerWindow.endpointRouteVerified) { + std::cerr << "TileXRDirectCcuTrace endpointRoute usePeerExportedRoute" + << " peerRank=" << peer + << " tpn=0x" << std::hex << peerWindow.tpn + << " peerEid=" << FormatEndpointEid(peerWindow.remoteEid) + << " channelEid=" << FormatEndpointEid(remote.remoteEid) + << " remoteCcuVa=0x" << remote.remoteCcuVa + << " remoteCcuVaBase=0x" << peerWindow.addr + << " remoteCcuVaOffset=0x" << remoteCcuVaOffset + << std::dec << std::endl; + } + } + if (localVerifiedEndpointRouteValid_) { + remote.localDoorbellVa = localVerifiedEndpointRoute_.doorbellVa; + remote.localDoorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId; + remote.localDoorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue; + remote.localSqDepth = localVerifiedEndpointRoute_.sqDepth; + } + remote.endpointRouteVerified = peerWindow.endpointRouteVerified; + if (importedPeerRoute) { + remote.endpointRouteVerified = true; + } + remote.channelResourceOwnerVerified = peerWindow.channelResourceOwnerVerified; + remote.transportResourceExchangeVerified = peerWindow.transportResourceExchangeVerified; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute channelRoute" + << " peerRank=" << peer + << " importedPeerRoute=" << (importedPeerRoute ? 1 : 0) + << " peerTpHandlesReady=" << (peerTpHandlesReady ? 1 : 0) + << " localEid=" << FormatEndpointEid(localResourceWindow_.eid) + << " peerEid=" << FormatEndpointEid(peerWindow.remoteEid) + << " selectedRemoteEid=" << FormatEndpointEid(remote.remoteEid) + << " localTpHandle=0x" << std::hex << localTpForPeer + << " peerTpHandle=0x" << peerTpForLocal + << " tpn=0x" << remote.tpn + << " remoteCcuVa=0x" << remote.remoteCcuVa + << " remoteCcuVaBase=0x" << peerWindow.addr + << " remoteCcuVaOffset=0x" << remoteCcuVaOffset + << " doorbellVa=0x" << remote.doorbellVa + << " localDoorbellVa=0x" << remote.localDoorbellVa + << " memoryTokenId=0x" << remote.memoryTokenId + << " rawMemoryTokenId=0x" << remote.rawMemoryTokenId + << " memoryTokenValue=0x" << remote.memoryTokenValue + << std::dec + << " endpointRouteVerified=" << (remote.endpointRouteVerified ? 1 : 0) + << " channelResourceOwnerVerified=" << (remote.channelResourceOwnerVerified ? 1 : 0) + << " transportResourceExchangeVerified=" + << (remote.transportResourceExchangeVerified ? 1 : 0) + << std::endl; + } + buffers->push_back(remote); + } + if (peerTpHandlesReady) { + ++endpointPsn_; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot& templateSnapshot, + TileXRCcuLowerLayerTransportSnapshot* snapshot) const +{ + if (snapshot == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *snapshot = TileXRCcuLowerLayerTransportSnapshot{}; + if (!resourceWindowRegistered_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + + *snapshot = templateSnapshot; + for (uint32_t i = 0; i < snapshot->routes.size(); ++i) { + auto& route = snapshot->routes[i]; + const bool preserveVerifiedRoute = route.endpointRouteVerified && HasCompleteEndpointRoute(route); + bool filledSyntheticEndpointField = false; + if (IsEmptyEid(route.remoteEid)) { + route.remoteEid[0] = static_cast(devicePhyId_ & 0xffU); + route.remoteEid[1] = static_cast((devicePhyId_ >> 8U) & 0xffU); + route.remoteEid[2] = static_cast(route.peerRank & 0xffU); + route.remoteEid[3] = static_cast((route.peerRank >> 8U) & 0xffU); + route.remoteEid[4] = static_cast(options_.rank & 0xff); + route.remoteEid[5] = static_cast(snapshot->dieId); + route.remoteEid[6] = static_cast(route.channelId & 0xffU); + route.remoteEid[7] = static_cast((route.channelId >> 8U) & 0xffU); + filledSyntheticEndpointField = true; + } + if (!preserveVerifiedRoute && route.tpn == 0) { + route.tpn = route.channelId + 1U; + filledSyntheticEndpointField = true; + } + if (route.doorbellVa == 0) { + route.doorbellVa = localResourceWindow_.addr + TILEXR_CCU_V1_XN_RESOURCE_OFFSET + + static_cast(snapshot->xnStartId + i) * TILEXR_CCU_XN_SLOT_BYTES; + filledSyntheticEndpointField = true; + } + if (route.doorbellTokenId == 0) { + route.doorbellTokenId = localResourceWindow_.tokenId; + filledSyntheticEndpointField = true; + } + if (!preserveVerifiedRoute && route.doorbellTokenValue == 0) { + route.doorbellTokenValue = localResourceWindow_.tokenValue; + } + if (route.sqDepth == 0) { + route.sqDepth = TILEXR_CCU_DEFAULT_DIRECT_SQ_DEPTH; + filledSyntheticEndpointField = true; + } + route.endpointRouteVerified = (preserveVerifiedRoute && !filledSyntheticEndpointField) || + (filledSyntheticEndpointField && TrustSyntheticEndpointRouteForDiagnostics() && + HasCompleteEndpointRoute(route)); + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.h b/src/comm/ccu/tilexr_ccu_direct_runtime.h new file mode 100644 index 00000000..3e358f06 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.h @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_DIRECT_RUNTIME_H +#define TILEXR_CCU_DIRECT_RUNTIME_H + +#include "ccu/tilexr_ccu_driver_adapter.h" +#include "ccu/tilexr_ccu_hccp_loader.h" +#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" +#include "ccu/tilexr_ccu_ra_custom_channel_provider.h" + +#include +#include +#include +#include +#include + +namespace TileXR { + +using TileXRCcuDirectAllGatherFn = int (*)( + const void* sendBuf, + size_t sendBytes, + void* recvBuf, + void* userData); + +struct TileXRCcuLocalResourceWindowInfo; + +using TileXRCcuLocalEndpointRouteCollectorFn = int (*)( + uint32_t devicePhyId, + const TileXRCcuLocalResourceWindowInfo& localResourceWindow, + TileXRCcuLowerLayerTransportRoute* route, + void* userData); + +struct TileXRCcuDirectRuntimeOptions { + int rank = 0; + int rankSize = 0; + int devId = 0; + TileXRCcuDirectAllGatherFn allGather = nullptr; + void* allGatherUserData = nullptr; + TileXRCcuLocalEndpointRouteCollectorFn localEndpointRouteCollector = nullptr; + void* localEndpointRouteCollectorUserData = nullptr; +}; + +struct TileXRCcuLocalResourceWindowInfo { + uint64_t addr = 0; + uint64_t bytes = 0; + uint32_t tokenId = 0; + uint32_t rawTokenId = 0; + uint32_t tokenValue = 0; + uint64_t targetSegHandle = 0; + void* raCtxHandle = nullptr; + void* tokenIdHandle = nullptr; + void* lmemHandle = nullptr; + std::array eid {}; + uint32_t eidIndex = 0; + bool raCtxRegistered = false; +}; + +struct TileXRCcuResourceWindowExchange { + uint64_t addr = 0; + uint64_t bytes = 0; + uint32_t tokenId = 0; + uint32_t rawTokenId = 0; + uint32_t tokenValue = 0; + std::array remoteEid {}; + uint32_t tpn = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + TileXRCcuHccpQpKey qpKey {}; + uint32_t psn = 0; + bool endpointRouteVerified = false; + bool channelResourceOwnerVerified = false; + bool transportResourceExchangeVerified = false; +}; + +struct TileXRCcuDirectRuntimeReport { + bool initialized = false; + bool raInitialized = false; + bool ccuTlvInitialized = false; + uint32_t logicDevId = 0; + uint32_t devicePhyId = 0; + int hdcType = 0; + std::string message; +}; + +class TileXRCcuDirectRuntime { +public: + int Init(const TileXRCcuDirectRuntimeOptions& options, TileXRCcuDirectRuntimeReport* report); + void Shutdown(); + bool IsAvailable() const; + + int QueryBasicInfo(uint8_t dieId, TileXRCcuBasicInfo* basicInfo, TileXRCcuDriverAdapterReport* report); + int CreateDriverAdapter(TileXRCcuDriverAdapter* adapter, TileXRCcuDriverAdapterReport* report); + int RegisterCcuResourceRmaBuffer(uint64_t resourceAddr); + int RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRuntimeReport* report); + int ConfigureLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute& route); + int ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo* info) const; + int ExportRemoteCcuRmaBuffers(std::vector* buffers); + int ExportLowerLayerTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot& templateSnapshot, + TileXRCcuLowerLayerTransportSnapshot* snapshot) const; + +private: + int ResolveDevicePhyId(uint32_t* devicePhyId, TileXRCcuDirectRuntimeReport* report) const; + int RegisterCcuResourceRmaBufferWithRaCtx(uint64_t resourceAddr, uint64_t resourceBytes); + int CollectLocalEndpointRouteWithRaCtx(TileXRCcuLowerLayerTransportRoute* route); + int CollectLocalEndpointRouteWithRaCtxOnce( + TileXRCcuLowerLayerTransportRoute* route, + bool* asyncWaitFailed); + int QueryTpHandleForPeer( + const std::array& peerEid, + uint64_t* tpHandle); + int ImportPeerEndpointRoute( + const TileXRCcuResourceWindowExchange& peerWindow, + uint64_t localTpHandle, + uint64_t peerTpHandle, + uint32_t localPsn, + uint32_t peerPsn, + TileXRCcuLowerLayerTransportRoute* importedRoute); + void ReleasePeerEndpointImports(); + void ReleaseRegisteredResourceWindow(); + void ReleaseLocalEndpointRoute(); + + TileXRCcuDirectRuntimeOptions options_; + TileXRCcuHccpLoader loader_; + TileXRCcuRaCustomChannelProvider raCustomChannelProvider_; + TileXRCcuBasicInfo cachedBasicInfo_ = {}; + TileXRCcuLocalResourceWindowInfo localResourceWindow_ = {}; + TileXRCcuLowerLayerTransportRoute localVerifiedEndpointRoute_ = {}; + bool cachedBasicInfoValid_ = false; + bool resourceWindowRegistered_ = false; + bool localVerifiedEndpointRouteValid_ = false; + void* endpointChanHandle_ = nullptr; + void* endpointCqHandle_ = nullptr; + void* endpointQpHandle_ = nullptr; + void* endpointRemoteQpHandle_ = nullptr; + std::vector endpointPeerRemoteQpHandles_; + TileXRCcuHccpQpKey endpointQpKey_ = {}; + bool endpointQpKeyValid_ = false; + bool endpointRouteBound_ = false; + uint32_t endpointPsn_ = 1; + uint32_t devicePhyId_ = 0; + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_DIRECT_RUNTIME_H diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp new file mode 100644 index 00000000..3c2c1bb0 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp @@ -0,0 +1,524 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_driver_adapter.h" + +#include +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuDriverAdapterReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRCcuDriverAdapterReport{}; +} + +int Fail(TileXRCcuDriverAdapterReport* report, const std::string& message, int code = TILEXR_ERROR_PARA_CHECK_FAIL) +{ + if (report != nullptr) { + report->message = message; + } + return code; +} + +void InitRequest(uint8_t dieId, uint32_t opcode, TileXRCcuCustomChannelIn* in) +{ + std::memset(in, 0, sizeof(*in)); + in->op = opcode; + in->offsetStartIdx = 0; + in->data.dataInfo.udieIdx = dieId; +} + +void FillCallReport( + uint32_t devicePhyId, + uint8_t dieId, + uint32_t opcode, + int driverRet, + int opRet, + const std::string& message, + TileXRCcuDriverAdapterReport* report) +{ + if (report == nullptr) { + return; + } + report->devicePhyId = devicePhyId; + report->dieId = dieId; + report->opcode = opcode; + report->driverRet = driverRet; + report->opRet = opRet; + report->message = message; +} + +std::string CcuCustomChannelFailureMessage( + const char* prefix, + uint32_t opcode, + int driverRet, + int opRet) +{ + std::ostringstream message; + message << prefix + << " op=" << opcode + << " driverRet=" << driverRet + << " opRet=" << opRet; + return message.str(); +} + +template +void CopyPayloadToSlot(const Payload& payload, TileXRCcuDataTypeUnion* slot) +{ + std::memcpy(slot, payload.raw, sizeof(payload.raw)); +} + +bool DirectTraceEnabled() +{ + const char* value = std::getenv("TILEXR_CCU_DIRECT_TRACE"); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +uint64_t LoadWord(const void* data, uint32_t offset, uint32_t bytes) +{ + uint64_t word = 0; + if (offset >= bytes) { + return word; + } + const uint32_t copyBytes = std::min(sizeof(word), bytes - offset); + std::memcpy(&word, static_cast(data) + offset, copyBytes); + return word; +} + +void TraceWords(const char* label, const void* data, uint32_t bytes) +{ + const uint32_t wordCount = (bytes + 7U) / 8U; + std::cerr << "TileXRDirectCcuTrace " << label << "Words=" << wordCount; + for (uint32_t i = 0; i < wordCount; ++i) { + std::cerr << " w" << i << "=" << std::hex << std::showbase + << LoadWord(data, i * 8U, bytes) + << std::dec << std::noshowbase; + } + std::cerr << "\n"; +} + +void TraceCustomChannelRequest( + uint32_t devicePhyId, + uint8_t dieId, + uint32_t opcode, + const TileXRCcuCustomChannelIn& in) +{ + if (!DirectTraceEnabled()) { + return; + } + + const uint32_t payloadBytes = std::min( + in.data.dataInfo.dataLen == 0 ? TILEXR_CCU_DATA_ARRAY_SLOT_BYTES : in.data.dataInfo.dataLen, + sizeof(in.data.dataInfo.dataArray)); + std::cerr << "TileXRDirectCcuTrace customChannel" + << " devicePhyId=" << devicePhyId + << " op=" << opcode + << " dieId=" << static_cast(dieId) + << " requestDieId=" << in.data.dataInfo.udieIdx + << " offset=" << in.offsetStartIdx + << " dataLen=" << in.data.dataInfo.dataLen + << " arraySize=" << in.data.dataInfo.dataArraySize + << " payloadWords=" << ((payloadBytes + 7U) / 8U) + << "\n"; + TraceWords("customChannel.request", &in, std::min(sizeof(in), 256U)); + TraceWords("customChannel.requestTrailer", &in.offsetStartIdx, sizeof(in.offsetStartIdx) + sizeof(in.op)); + TraceWords("customChannel.payload", in.data.dataInfo.dataArray, payloadBytes); +} + +void TraceCustomChannelReturn( + uint32_t devicePhyId, + uint8_t dieId, + uint32_t opcode, + int driverRet, + const TileXRCcuCustomChannelOut& out) +{ + if (!DirectTraceEnabled()) { + return; + } + + std::cerr << "TileXRDirectCcuTrace customChannel.return" + << " devicePhyId=" << devicePhyId + << " op=" << opcode + << " dieId=" << static_cast(dieId) + << " driverRet=" << driverRet + << " opRet=" << out.opRet + << " offsetNext=" << out.offsetNextIdx + << "\n"; + TraceWords("customChannel.response", &out, std::min(sizeof(out), 256U)); + TraceWords("customChannel.responseTrailer", &out.offsetNextIdx, sizeof(out.offsetNextIdx) + sizeof(out.opRet)); +} + +} // namespace + +int TileXRCcuDriverAdapter::Init( + uint32_t devicePhyId, + TileXRCcuCustomChannelFn customChannel, + void* userData, + TileXRCcuDriverAdapterReport* report) +{ + ResetReport(report); + if (customChannel == nullptr) { + initialized_ = false; + return Fail(report, "missing CCU custom channel callback"); + } + devicePhyId_ = devicePhyId; + customChannel_ = customChannel; + userData_ = userData; + initialized_ = true; + FillCallReport(devicePhyId_, 0, 0, 0, 0, "ok", report); + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::Call( + uint8_t dieId, + uint32_t opcode, + TileXRCcuCustomChannelOut* out, + TileXRCcuDriverAdapterReport* report) const +{ + TileXRCcuCustomChannelIn in; + InitRequest(dieId, opcode, &in); + return CallPrepared(dieId, opcode, in, out, report); +} + +int TileXRCcuDriverAdapter::CallPrepared( + uint8_t dieId, + uint32_t opcode, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (!initialized_ || customChannel_ == nullptr) { + return Fail(report, "CCU driver adapter is not initialized"); + } + if (out == nullptr) { + return Fail(report, "missing CCU custom channel output"); + } + + std::memset(out, 0, sizeof(*out)); + TraceCustomChannelRequest(devicePhyId_, dieId, opcode, in); + const int driverRet = customChannel_(devicePhyId_, in, out, userData_); + TraceCustomChannelReturn(devicePhyId_, dieId, opcode, driverRet, *out); + FillCallReport(devicePhyId_, dieId, opcode, driverRet, out->opRet, "ok", report); + if (driverRet != 0) { + return Fail( + report, + CcuCustomChannelFailureMessage("CCU custom channel call failed", opcode, driverRet, out->opRet), + TILEXR_ERROR_MKIRT); + } + if (out->opRet != 0) { + return Fail( + report, + CcuCustomChannelFailureMessage("CCU custom channel operation failed", opcode, driverRet, out->opRet), + TILEXR_ERROR_MKIRT); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::GetBasicInfo( + uint8_t dieId, + TileXRCcuBasicInfo* basicInfo, + TileXRCcuDriverAdapterReport* report) const +{ + if (basicInfo == nullptr) { + ResetReport(report); + return Fail(report, "missing output CCU basic info"); + } + *basicInfo = TileXRCcuBasicInfo{}; + + TileXRCcuCustomChannelOut out; + const int ret = Call(dieId, TILEXR_CCU_U_OP_GET_BASIC_INFO, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + const auto& raw = out.data.dataInfo.dataArray[0].baseinfo; + basicInfo->dieId = dieId; + basicInfo->msId = raw.msId; + basicInfo->msidToken.tokenId = raw.tokenId; + basicInfo->msidToken.tokenValue = raw.tokenValue; + basicInfo->msidToken.valid = raw.tokenValid != 0; + basicInfo->missionKey = raw.missionKey; + basicInfo->resourceAddr = raw.resourceAddr; + basicInfo->caps.cap0 = raw.caps.cap0; + basicInfo->caps.cap1 = raw.caps.cap1; + basicInfo->caps.cap2 = raw.caps.cap2; + basicInfo->caps.cap3 = raw.caps.cap3; + basicInfo->caps.cap4 = raw.caps.cap4; + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::GetDieEnabled( + uint8_t dieId, + bool* enabled, + TileXRCcuDriverAdapterReport* report) const +{ + if (enabled == nullptr) { + ResetReport(report); + return Fail(report, "missing output CCU die enabled flag"); + } + *enabled = false; + + TileXRCcuCustomChannelOut out; + const int ret = Call(dieId, TILEXR_CCU_U_OP_GET_DIE_WORKING, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + *enabled = out.data.dataInfo.dataArray[0].dieinfo.enableFlag == TILEXR_CCU_ENABLE_FLAG; + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::ReadInstructions( + uint8_t dieId, + uint16_t instructionStartId, + void* instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (instructions == nullptr) { + return Fail(report, "missing output CCU instruction readback buffer"); + } + if (instructionCount == 0 || instructionCount > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { + return Fail(report, "invalid CCU instruction readback count"); + } + const uint32_t expectedBytes = instructionCount * TILEXR_CCU_INSTRUCTION_BYTES; + if (instructionBytes != expectedBytes) { + return Fail(report, "CCU instruction readback byte size mismatch"); + } + + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_GET_INSTRUCTION, &in); + in.offsetStartIdx = instructionStartId; + in.data.dataInfo.dataArraySize = instructionCount; + in.data.dataInfo.dataLen = instructionBytes; + + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, TILEXR_CCU_U_OP_GET_INSTRUCTION, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + auto* dst = static_cast(instructions); + for (uint32_t i = 0; i < instructionCount; ++i) { + std::memcpy( + dst + i * TILEXR_CCU_INSTRUCTION_BYTES, + out.data.dataInfo.dataArray[i].byte32.raw, + TILEXR_CCU_INSTRUCTION_BYTES); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::InstallInstructions( + uint8_t dieId, + uint16_t instructionStartId, + uint16_t instructionCount, + uint64_t deviceInstructionAddr, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport* report) const +{ + return InstallInstructionsWithDataLen( + dieId, + instructionStartId, + instructionCount, + deviceInstructionAddr, + instructionBytes, + instructionBytes, + report); +} + +int TileXRCcuDriverAdapter::InstallInstructionsWithDataLen( + uint8_t dieId, + uint16_t instructionStartId, + uint16_t instructionCount, + uint64_t deviceInstructionAddr, + uint32_t instructionBytes, + uint32_t customChannelDataLen, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (instructionCount == 0) { + return Fail(report, "missing CCU instruction image"); + } + if (deviceInstructionAddr == 0) { + return Fail(report, "missing device CCU instruction image address"); + } + const uint32_t expectedBytes = static_cast(instructionCount) * TILEXR_CCU_INSTRUCTION_BYTES; + if (instructionBytes == 0 || instructionBytes != expectedBytes) { + return Fail(report, "CCU instruction image byte size mismatch"); + } + if (customChannelDataLen == 0) { + return Fail(report, "missing CCU instruction custom channel data length"); + } + + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_INSTRUCTION, &in); + in.offsetStartIdx = instructionStartId; + in.data.dataInfo.dataArraySize = 1; + in.data.dataInfo.dataLen = customChannelDataLen; + in.data.dataInfo.dataArray[0].insinfo.resourceAddr = deviceInstructionAddr; + + TileXRCcuCustomChannelOut out; + return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_INSTRUCTION, in, &out, report); +} + +int TileXRCcuDriverAdapter::InstallMsidToken( + uint8_t dieId, + uint32_t msId, + uint32_t tokenId, + uint32_t tokenValue, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_MSID_TOKEN, &in); + in.offsetStartIdx = 0; + in.data.dataInfo.dataArray[0].baseinfo.msId = msId; + in.data.dataInfo.dataArray[0].baseinfo.tokenId = tokenId; + in.data.dataInfo.dataArray[0].baseinfo.tokenValue = tokenValue; + + TileXRCcuCustomChannelOut out; + return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_MSID_TOKEN, in, &out, report); +} + +int TileXRCcuDriverAdapter::InstallPfeCtx( + uint8_t dieId, + uint32_t pfeOffset, + const TileXRCcuPfeCtx& ctx, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_PFE, &in); + in.offsetStartIdx = pfeOffset; + in.data.dataInfo.dataArraySize = 1; + in.data.dataInfo.dataLen = TILEXR_CCU_PFE_CTX_BYTES; + std::memcpy(&in.data.dataInfo.dataArray[0], ctx.raw, TILEXR_CCU_PFE_CTX_BYTES); + + TileXRCcuCustomChannelOut out; + return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_PFE, in, &out, report); +} + +int TileXRCcuDriverAdapter::InstallJettyCtx( + uint8_t dieId, + uint16_t startJettyCtxId, + const TileXRCcuLocalJettyCtxData* ctxs, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (ctxs == nullptr) { + return Fail(report, "missing CCU local jetty context payloads"); + } + if (count == 0 || count > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { + return Fail(report, "invalid CCU local jetty context count"); + } + + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, &in); + in.offsetStartIdx = startJettyCtxId; + in.data.dataInfo.dataArraySize = count; + in.data.dataInfo.dataLen = count * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; + for (uint32_t i = 0; i < count; ++i) { + CopyPayloadToSlot(ctxs[i], &in.data.dataInfo.dataArray[i]); + } + + TileXRCcuCustomChannelOut out; + return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, in, &out, report); +} + +int TileXRCcuDriverAdapter::InstallChannelCtxV1( + uint8_t dieId, + uint32_t channelId, + const TileXRCcuChannelCtxDataV1& ctx, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_CHANNEL, &in); + in.offsetStartIdx = channelId; + in.data.dataInfo.dataArraySize = 1; + in.data.dataInfo.dataLen = TILEXR_CCU_CHANNEL_CTX_V1_BYTES; + std::memcpy(&in.data.dataInfo.dataArray[0], ctx.raw, TILEXR_CCU_CHANNEL_CTX_V1_BYTES); + + TileXRCcuCustomChannelOut out; + return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_CHANNEL, in, &out, report); +} + +int TileXRCcuDriverAdapter::ClearCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (count == 0) { + return Fail(report, "missing CCU CKE range"); + } + + uint32_t remaining = count; + uint32_t offset = startCkeId; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_CKE, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * TILEXR_CCU_CKE_SLOT_BYTES; + + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, TILEXR_CCU_U_OP_SET_CKE, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + remaining -= batch; + offset += batch; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::InstallXnRange( + uint8_t dieId, + uint32_t startXnId, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (count == 0) { + return Fail(report, "missing CCU XN range"); + } + + uint32_t remaining = count; + uint32_t offset = startXnId; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_XN, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * TILEXR_CCU_XN_SLOT_BYTES; + + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, TILEXR_CCU_U_OP_SET_XN, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + remaining -= batch; + offset += batch; + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.h b/src/comm/ccu/tilexr_ccu_driver_adapter.h new file mode 100644 index 00000000..65534113 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.h @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_DRIVER_ADAPTER_H +#define TILEXR_CCU_DRIVER_ADAPTER_H + +#include "ccu/tilexr_ccu_hccp_types.h" +#include "ccu/tilexr_ccu_specs.h" + +#include +#include + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_U_OP_GET_BASIC_INFO = 11; +constexpr uint32_t TILEXR_CCU_U_OP_GET_DIE_WORKING = 15; +constexpr uint32_t TILEXR_CCU_U_OP_GET_INSTRUCTION = 201; +constexpr uint32_t TILEXR_CCU_U_OP_SET_MSID_TOKEN = 53; +constexpr uint32_t TILEXR_CCU_U_OP_SET_INSTRUCTION = 251; +constexpr uint32_t TILEXR_CCU_U_OP_SET_XN = 253; +constexpr uint32_t TILEXR_CCU_U_OP_SET_CKE = 254; +constexpr uint32_t TILEXR_CCU_U_OP_SET_PFE = 255; +constexpr uint32_t TILEXR_CCU_U_OP_SET_CHANNEL = 256; +constexpr uint32_t TILEXR_CCU_U_OP_SET_JETTY_CTX = 257; +constexpr uint32_t TILEXR_CCU_ENABLE_FLAG = 1; +constexpr uint32_t TILEXR_CCU_INSTRUCTION_BYTES = 32; +constexpr uint32_t TILEXR_CCU_DATA_ARRAY_SLOT_BYTES = 64; +constexpr uint32_t TILEXR_CCU_XN_SLOT_BYTES = 8; +constexpr uint32_t TILEXR_CCU_CKE_SLOT_BYTES = 8; +constexpr uint32_t TILEXR_CCU_PFE_CTX_BYTES = 8; +constexpr uint32_t TILEXR_CCU_LOCAL_JETTY_CTX_BYTES = 32; +constexpr uint32_t TILEXR_CCU_CHANNEL_CTX_V1_BYTES = 64; +constexpr uint32_t TILEXR_CCU_MAX_DATA_ARRAY_SIZE = 8; + +struct TileXRCcuPfeCtx { + uint8_t raw[TILEXR_CCU_PFE_CTX_BYTES]; +}; + +struct TileXRCcuLocalJettyCtxData { + uint8_t raw[TILEXR_CCU_LOCAL_JETTY_CTX_BYTES]; +}; + +struct TileXRCcuChannelCtxDataV1 { + uint8_t raw[TILEXR_CCU_CHANNEL_CTX_V1_BYTES]; +}; + +struct TileXRCcuDriverAdapterReport { + uint32_t devicePhyId = 0; + uint8_t dieId = 0; + uint32_t opcode = 0; + int driverRet = 0; + int opRet = 0; + std::string message; +}; + +using TileXRCcuCustomChannelFn = int (*)( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData); + +class TileXRCcuDriverAdapter { +public: + int Init( + uint32_t devicePhyId, + TileXRCcuCustomChannelFn customChannel, + void* userData, + TileXRCcuDriverAdapterReport* report); + + int GetBasicInfo(uint8_t dieId, TileXRCcuBasicInfo* basicInfo, TileXRCcuDriverAdapterReport* report) const; + int GetDieEnabled(uint8_t dieId, bool* enabled, TileXRCcuDriverAdapterReport* report) const; + int ReadInstructions( + uint8_t dieId, + uint16_t instructionStartId, + void* instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport* report) const; + int InstallInstructions( + uint8_t dieId, + uint16_t instructionStartId, + uint16_t instructionCount, + uint64_t deviceInstructionAddr, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport* report) const; + int InstallInstructionsWithDataLen( + uint8_t dieId, + uint16_t instructionStartId, + uint16_t instructionCount, + uint64_t deviceInstructionAddr, + uint32_t instructionBytes, + uint32_t customChannelDataLen, + TileXRCcuDriverAdapterReport* report) const; + int InstallMsidToken( + uint8_t dieId, + uint32_t msId, + uint32_t tokenId, + uint32_t tokenValue, + TileXRCcuDriverAdapterReport* report) const; + int InstallPfeCtx( + uint8_t dieId, + uint32_t pfeOffset, + const TileXRCcuPfeCtx& ctx, + TileXRCcuDriverAdapterReport* report) const; + int InstallJettyCtx( + uint8_t dieId, + uint16_t startJettyCtxId, + const TileXRCcuLocalJettyCtxData* ctxs, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; + int InstallChannelCtxV1( + uint8_t dieId, + uint32_t channelId, + const TileXRCcuChannelCtxDataV1& ctx, + TileXRCcuDriverAdapterReport* report) const; + int ClearCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; + int InstallXnRange( + uint8_t dieId, + uint32_t startXnId, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; + +private: + int Call(uint8_t dieId, uint32_t opcode, TileXRCcuCustomChannelOut* out, TileXRCcuDriverAdapterReport* report) + const; + int CallPrepared( + uint8_t dieId, + uint32_t opcode, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + TileXRCcuDriverAdapterReport* report) const; + + uint32_t devicePhyId_ = 0; + TileXRCcuCustomChannelFn customChannel_ = nullptr; + void* userData_ = nullptr; + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_DRIVER_ADAPTER_H diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.cpp b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp new file mode 100644 index 00000000..3f28935a --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp @@ -0,0 +1,581 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_hccp_loader.h" + +#include "tilexr_types.h" + +#include +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuHccpLoaderReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuHccpLoaderReport{}; + } +} + +int Fail(TileXRCcuHccpLoaderReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_NOT_FOUND; +} + +int FailWithCode(TileXRCcuHccpLoaderReport* report, const std::string& message, int code) +{ + if (report != nullptr) { + report->message = message; + } + return code; +} + +template +bool LoadSymbol(void* handle, Fn& out, const char* primary, const char* fallback) +{ + out = reinterpret_cast(dlsym(handle, primary)); + if (out == nullptr && fallback != nullptr) { + out = reinterpret_cast(dlsym(handle, fallback)); + } + return out != nullptr; +} + +template +void LoadOptionalSymbol(void* handle, Fn& out, const char* primary, const char* fallback) +{ + if (handle == nullptr) { + out = nullptr; + return; + } + out = reinterpret_cast(dlsym(handle, primary)); + if (out == nullptr && fallback != nullptr) { + out = reinterpret_cast(dlsym(handle, fallback)); + } +} + +using RaHdcKey = std::pair; + +std::mutex g_raHdcMtx; +std::map g_raHdcRefs; +uint32_t g_netServiceRefs = 0; +int g_netServiceHdcType = 0; + +struct CcuTlvSession { + void* handle = nullptr; + uint32_t bufferSize = 0; + uint32_t refs = 0; +}; + +std::mutex g_ccuTlvMtx; +std::map g_ccuTlvSessions; + +bool EnvFlag(const char* name) +{ + const char* value = std::getenv(name); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +std::string RaConfigText(const TileXRCcuRaInitConfig& config) +{ + std::ostringstream text; + text << "phyId=" << config.phyId + << " nicPosition=" << config.nicPosition + << " hdcType=" << config.hdcType + << " enableHdcAsync=" << (config.enableHdcAsync ? 1 : 0); + return text.str(); +} + +} // namespace + +TileXRCcuHccpLoader::~TileXRCcuHccpLoader() +{ + Unload(); +} + +int TileXRCcuHccpLoader::Load(TileXRCcuHccpLoaderReport* report) +{ + ResetReport(report); + if (loaded_) { + if (report != nullptr) { + report->loaded = true; + report->message = "ok"; + } + return TILEXR_SUCCESS; + } + + raHandle_ = dlopen("libra.so", RTLD_NOW); + if (raHandle_ == nullptr) { + return Fail(report, std::string("failed to load libra.so: ") + dlerror()); + } + + if (!LoadSymbol(raHandle_, RaCustomChannel, "RaCustomChannel", "ra_custom_channel")) { + Unload(); + return Fail(report, "missing RaCustomChannel/ra_custom_channel in libra.so"); + } + if (!LoadSymbol(raHandle_, RaInit, "RaInit", nullptr)) { + Unload(); + return Fail(report, "missing RaInit in libra.so"); + } + if (!LoadSymbol(raHandle_, RaDeinit, "RaDeinit", nullptr)) { + Unload(); + return Fail(report, "missing RaDeinit in libra.so"); + } + LoadOptionalSymbol(raHandle_, RaTlvInit, "RaTlvInit", nullptr); + LoadOptionalSymbol(raHandle_, RaTlvRequest, "RaTlvRequest", nullptr); + LoadOptionalSymbol(raHandle_, RaTlvDeinit, "RaTlvDeinit", nullptr); + LoadOptionalSymbol(raHandle_, RaGetDevEidInfoNum, "RaGetDevEidInfoNum", "ra_get_dev_eid_info_num"); + LoadOptionalSymbol(raHandle_, RaGetDevEidInfoList, "RaGetDevEidInfoList", "ra_get_dev_eid_info_list"); + LoadOptionalSymbol(raHandle_, RaCtxInit, "RaCtxInit", "ra_ctx_init"); + LoadOptionalSymbol(raHandle_, RaCtxDeinit, "RaCtxDeinit", "ra_ctx_deinit"); + LoadOptionalSymbol(raHandle_, RaCtxTokenIdAlloc, "RaCtxTokenIdAlloc", "ra_ctx_token_id_alloc"); + LoadOptionalSymbol(raHandle_, RaCtxTokenIdFree, "RaCtxTokenIdFree", "ra_ctx_token_id_free"); + LoadOptionalSymbol(raHandle_, RaCtxLmemRegister, "RaCtxLmemRegister", "ra_ctx_lmem_register"); + LoadOptionalSymbol(raHandle_, RaCtxLmemUnregister, "RaCtxLmemUnregister", "ra_ctx_lmem_unregister"); + LoadOptionalSymbol(raHandle_, RaGetSecRandom, "RaGetSecRandom", "ra_get_sec_random"); + LoadOptionalSymbol(raHandle_, RaCtxChanCreate, "RaCtxChanCreate", "ra_ctx_chan_create"); + LoadOptionalSymbol(raHandle_, RaCtxChanDestroy, "RaCtxChanDestroy", "ra_ctx_chan_destroy"); + LoadOptionalSymbol(raHandle_, RaCtxCqCreate, "RaCtxCqCreate", "ra_ctx_cq_create"); + LoadOptionalSymbol(raHandle_, RaCtxCqDestroy, "RaCtxCqDestroy", "ra_ctx_cq_destroy"); + LoadOptionalSymbol(raHandle_, RaCtxQpCreate, "RaCtxQpCreate", "ra_ctx_qp_create"); + LoadOptionalSymbol(raHandle_, RaCtxQpDestroy, "RaCtxQpDestroy", "ra_ctx_qp_destroy"); + LoadOptionalSymbol(raHandle_, RaCtxQpImport, "RaCtxQpImport", "ra_ctx_qp_import"); + LoadOptionalSymbol(raHandle_, RaCtxQpUnimport, "RaCtxQpUnimport", "ra_ctx_qp_unimport"); + LoadOptionalSymbol(raHandle_, RaCtxQpBind, "RaCtxQpBind", "ra_ctx_qp_bind"); + LoadOptionalSymbol(raHandle_, RaCtxQpUnbind, "RaCtxQpUnbind", "ra_ctx_qp_unbind"); + LoadOptionalSymbol(raHandle_, RaGetTpInfoListAsync, "RaGetTpInfoListAsync", "ra_get_tp_info_list_async"); + LoadOptionalSymbol(raHandle_, RaGetAsyncReqResult, "RaGetAsyncReqResult", "ra_get_async_req_result"); + + runtimeHandle_ = dlopen("libruntime.so", RTLD_NOW); + LoadOptionalSymbol(runtimeHandle_, RtGetDevicePhyIdByIndex, "rtGetDevicePhyIdByIndex", nullptr); + LoadOptionalSymbol(runtimeHandle_, RtOpenNetService, "rtOpenNetService", nullptr); + LoadOptionalSymbol(runtimeHandle_, RtCloseNetService, "rtCloseNetService", nullptr); + + loaded_ = true; + if (report != nullptr) { + report->loaded = true; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuHccpLoader::LoadEndpointRouteProviderFromEnv(TileXRCcuHccpLoaderReport* report) +{ + ResetReport(report); + if (CollectLocalEndpointRoute != nullptr) { + if (report != nullptr) { + report->endpointRouteProviderLoaded = true; + report->message = "ok"; + } + return TILEXR_SUCCESS; + } + + const char* providerPath = std::getenv("TILEXR_CCU_ENDPOINT_ROUTE_PROVIDER"); + if (providerPath == nullptr || providerPath[0] == '\0') { + return Fail(report, "direct CCU endpoint route provider is not configured"); + } + if (report != nullptr) { + report->endpointRouteProviderConfigured = true; + } + + endpointRouteProviderHandle_ = dlopen(providerPath, RTLD_NOW); + if (endpointRouteProviderHandle_ == nullptr) { + return Fail(report, std::string("failed to load direct CCU endpoint route provider: ") + dlerror()); + } + + if (!LoadSymbol( + endpointRouteProviderHandle_, + CollectLocalEndpointRoute, + "TileXRCcuCollectLocalEndpointRoute", + "tilexr_ccu_collect_local_endpoint_route")) { + dlclose(endpointRouteProviderHandle_); + endpointRouteProviderHandle_ = nullptr; + return Fail(report, "missing TileXRCcuCollectLocalEndpointRoute in direct CCU endpoint route provider"); + } + + if (report != nullptr) { + report->endpointRouteProviderLoaded = true; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +void TileXRCcuHccpLoader::Unload() +{ + ReleaseCcuTlv(); + ReleaseRaHdc(); + RaCustomChannel = nullptr; + RtGetDevicePhyIdByIndex = nullptr; + RtOpenNetService = nullptr; + RtCloseNetService = nullptr; + RaInit = nullptr; + RaDeinit = nullptr; + RaTlvInit = nullptr; + RaTlvRequest = nullptr; + RaTlvDeinit = nullptr; + RaGetDevEidInfoNum = nullptr; + RaGetDevEidInfoList = nullptr; + RaCtxInit = nullptr; + RaCtxDeinit = nullptr; + RaCtxTokenIdAlloc = nullptr; + RaCtxTokenIdFree = nullptr; + RaCtxLmemRegister = nullptr; + RaCtxLmemUnregister = nullptr; + RaGetSecRandom = nullptr; + RaCtxChanCreate = nullptr; + RaCtxChanDestroy = nullptr; + RaCtxCqCreate = nullptr; + RaCtxCqDestroy = nullptr; + RaCtxQpCreate = nullptr; + RaCtxQpDestroy = nullptr; + RaCtxQpImport = nullptr; + RaCtxQpUnimport = nullptr; + RaCtxQpBind = nullptr; + RaCtxQpUnbind = nullptr; + RaGetTpInfoListAsync = nullptr; + RaGetAsyncReqResult = nullptr; + CollectLocalEndpointRoute = nullptr; + loaded_ = false; + if (endpointRouteProviderHandle_ != nullptr) { + dlclose(endpointRouteProviderHandle_); + endpointRouteProviderHandle_ = nullptr; + } + if (runtimeHandle_ != nullptr) { + dlclose(runtimeHandle_); + runtimeHandle_ = nullptr; + } + if (raHandle_ != nullptr) { + dlclose(raHandle_); + raHandle_ = nullptr; + } +} + +bool TileXRCcuHccpLoader::IsLoaded() const +{ + return loaded_; +} + +int TileXRCcuHccpLoader::ResolveDevicePhyId( + uint32_t logicDevId, + uint32_t* phyId, + TileXRCcuHccpLoaderReport* report) const +{ + if (report != nullptr) { + report->logicDevId = logicDevId; + report->runtimePhyIdRet = 0; + } + if (phyId == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!loaded_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (RtGetDevicePhyIdByIndex == nullptr) { + if (EnvFlag("TILEXR_CCU_DIRECT_ALLOW_LOGIC_PHY_FALLBACK")) { + *phyId = logicDevId; + if (report != nullptr) { + report->devicePhyId = *phyId; + report->message = "rtGetDevicePhyIdByIndex missing, using logic device id fallback"; + } + return TILEXR_SUCCESS; + } + return FailWithCode(report, "missing rtGetDevicePhyIdByIndex in libruntime.so", TILEXR_ERROR_NOT_FOUND); + } + uint32_t resolvedPhyId = 0; + const int ret = RtGetDevicePhyIdByIndex(logicDevId, &resolvedPhyId); + if (report != nullptr) { + report->runtimePhyIdRet = ret; + } + if (ret != 0) { + return FailWithCode(report, "rtGetDevicePhyIdByIndex failed", TILEXR_ERROR_MKIRT); + } + *phyId = resolvedPhyId; + if (report != nullptr) { + report->devicePhyId = *phyId; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuHccpLoader::InitRaHdc( + uint32_t devicePhyId, + int hdcType, + bool enableHdcAsync, + TileXRCcuHccpLoaderReport* report) +{ + if (report != nullptr) { + report->devicePhyId = devicePhyId; + report->hdcType = hdcType; + } + if (!loaded_) { + return FailWithCode(report, "CCU HCCP loader is not initialized for RA init", TILEXR_ERROR_NOT_INITIALIZED); + } + if (RaInit == nullptr || RaDeinit == nullptr) { + return FailWithCode(report, "missing RaInit/RaDeinit in libra.so", TILEXR_ERROR_NOT_FOUND); + } + if (raHdcInitialized_) { + std::lock_guard lock(g_raHdcMtx); + const auto it = g_raHdcRefs.find(RaHdcKey(raInitConfig_.phyId, raInitConfig_.hdcType)); + if (report != nullptr) { + report->raInitialized = true; + report->raInitRefCount = it == g_raHdcRefs.end() ? 0U : it->second; + report->message = "ok"; + } + return TILEXR_SUCCESS; + } + + TileXRCcuRaInitConfig config {}; + config.phyId = devicePhyId; + config.nicPosition = TILEXR_CCU_NETWORK_OFFLINE; + config.hdcType = hdcType; + config.enableHdcAsync = enableHdcAsync; + + const RaHdcKey key(config.phyId, config.hdcType); + std::lock_guard lock(g_raHdcMtx); + int ret = AcquireNetServiceLocked(hdcType, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + uint32_t& refCount = g_raHdcRefs[key]; + if (refCount == 0) { + ret = RaInit(&config); + if (report != nullptr) { + report->raInitRet = ret; + } + if (ret != 0) { + g_raHdcRefs.erase(key); + ReleaseNetServiceLocked(nullptr); + std::ostringstream message; + message << "RaInit failed ret=" << ret << ": " << RaConfigText(config); + return FailWithCode(report, message.str(), TILEXR_ERROR_MKIRT); + } + } + ++refCount; + raInitConfig_ = config; + raHdcInitialized_ = true; + if (report != nullptr) { + report->raInitialized = true; + report->raInitRefCount = refCount; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuHccpLoader::InitCcuTlv(uint32_t devicePhyId, TileXRCcuHccpLoaderReport* report) +{ + if (report != nullptr) { + report->devicePhyId = devicePhyId; + } + if (!loaded_) { + return FailWithCode(report, "CCU HCCP loader is not initialized for TLV init", + TILEXR_ERROR_NOT_INITIALIZED); + } + if (RaTlvInit == nullptr || RaTlvRequest == nullptr || RaTlvDeinit == nullptr) { + return FailWithCode(report, "missing RaTlvInit/RaTlvRequest/RaTlvDeinit in libra.so", + TILEXR_ERROR_NOT_FOUND); + } + if (ccuTlvInitialized_) { + std::lock_guard lock(g_ccuTlvMtx); + const auto it = g_ccuTlvSessions.find(ccuTlvDevicePhyId_); + if (report != nullptr) { + report->ccuTlvInitialized = true; + report->ccuTlvRefCount = it == g_ccuTlvSessions.end() ? 0U : it->second.refs; + report->ccuTlvBufferSize = it == g_ccuTlvSessions.end() ? 0U : it->second.bufferSize; + report->message = "ok"; + } + return TILEXR_SUCCESS; + } + + std::lock_guard lock(g_ccuTlvMtx); + CcuTlvSession& session = g_ccuTlvSessions[devicePhyId]; + if (session.refs == 0) { + TileXRCcuTlvInitInfo initInfo {}; + initInfo.version = TILEXR_CCU_TLV_VERSION; + initInfo.phyId = devicePhyId; + initInfo.nicPosition = TILEXR_CCU_NETWORK_OFFLINE; + uint32_t bufferSize = 0; + void* tlvHandle = nullptr; + int ret = RaTlvInit(&initInfo, &bufferSize, &tlvHandle); + if (report != nullptr) { + report->raTlvInitRet = ret; + report->ccuTlvBufferSize = bufferSize; + } + if (ret != 0 || tlvHandle == nullptr) { + g_ccuTlvSessions.erase(devicePhyId); + std::ostringstream message; + message << "RaTlvInit failed ret=" << ret + << ": phyId=" << initInfo.phyId + << " nicPosition=" << initInfo.nicPosition + << " version=" << initInfo.version; + return FailWithCode(report, message.str(), TILEXR_ERROR_MKIRT); + } + + TileXRCcuTlvMsg sendMsg {}; + TileXRCcuTlvMsg recvMsg {}; + sendMsg.type = TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT; + ret = RaTlvRequest(tlvHandle, TILEXR_CCU_TLV_MODULE_TYPE_CCU, &sendMsg, &recvMsg); + if (report != nullptr) { + report->raTlvRequestRet = ret; + } + if (ret != 0) { + (void)RaTlvDeinit(tlvHandle); + g_ccuTlvSessions.erase(devicePhyId); + std::ostringstream message; + message << "RaTlvRequest CCU_INIT failed ret=" << ret + << ": phyId=" << devicePhyId + << " moduleType=" << TILEXR_CCU_TLV_MODULE_TYPE_CCU + << " msgType=" << TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT; + return FailWithCode(report, message.str(), TILEXR_ERROR_MKIRT); + } + session.handle = tlvHandle; + session.bufferSize = bufferSize; + } + + ++session.refs; + ccuTlvDevicePhyId_ = devicePhyId; + ccuTlvInitialized_ = true; + if (report != nullptr) { + report->ccuTlvInitialized = true; + report->ccuTlvRefCount = session.refs; + report->ccuTlvBufferSize = session.bufferSize; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuHccpLoader::AcquireNetServiceLocked(int hdcType, TileXRCcuHccpLoaderReport* report) +{ + if (RtOpenNetService == nullptr || RtCloseNetService == nullptr) { + return FailWithCode(report, + "missing rtOpenNetService/rtCloseNetService in libruntime.so", + TILEXR_ERROR_NOT_FOUND); + } + if (g_netServiceRefs > 0) { + if (g_netServiceHdcType != hdcType) { + std::ostringstream message; + message << "runtime net service already opened for hdcType=" << g_netServiceHdcType + << ", requested hdcType=" << hdcType; + return FailWithCode(report, message.str(), TILEXR_ERROR_PARA_CHECK_FAIL); + } + ++g_netServiceRefs; + if (report != nullptr) { + report->netServiceRefCount = g_netServiceRefs; + } + return TILEXR_SUCCESS; + } + + std::string extParamText("--hdcType=" + std::to_string(hdcType)); + TileXRCcuRtProcExtParam extParam {}; + extParam.paramInfo = extParamText.c_str(); + extParam.paramLen = extParamText.size(); + TileXRCcuRtNetServiceOpenArgs openArgs {}; + openArgs.extParamList = &extParam; + openArgs.extParamCnt = 1; + const int ret = RtOpenNetService(&openArgs); + if (report != nullptr) { + report->rtOpenNetServiceRet = ret; + } + if (ret != 0) { + std::ostringstream message; + message << "rtOpenNetService failed ret=" << ret << ": " << extParamText; + return FailWithCode(report, message.str(), TILEXR_ERROR_MKIRT); + } + + g_netServiceHdcType = hdcType; + g_netServiceRefs = 1; + if (report != nullptr) { + report->netServiceRefCount = g_netServiceRefs; + } + return TILEXR_SUCCESS; +} + +void TileXRCcuHccpLoader::ReleaseNetServiceLocked(TileXRCcuHccpLoaderReport* report) +{ + if (g_netServiceRefs == 0) { + return; + } + if (g_netServiceRefs > 1U) { + --g_netServiceRefs; + if (report != nullptr) { + report->netServiceRefCount = g_netServiceRefs; + } + return; + } + + int ret = 0; + if (RtCloseNetService != nullptr) { + ret = RtCloseNetService(); + } + if (report != nullptr) { + report->rtCloseNetServiceRet = ret; + report->netServiceRefCount = 0; + } + g_netServiceRefs = 0; + g_netServiceHdcType = 0; +} + +void TileXRCcuHccpLoader::ReleaseCcuTlv() +{ + if (!ccuTlvInitialized_) { + return; + } + std::lock_guard lock(g_ccuTlvMtx); + auto it = g_ccuTlvSessions.find(ccuTlvDevicePhyId_); + if (it != g_ccuTlvSessions.end() && it->second.refs > 1U) { + --it->second.refs; + ccuTlvInitialized_ = false; + ccuTlvDevicePhyId_ = 0; + return; + } + if (it != g_ccuTlvSessions.end()) { + if (RaTlvRequest != nullptr && it->second.handle != nullptr) { + TileXRCcuTlvMsg sendMsg {}; + TileXRCcuTlvMsg recvMsg {}; + sendMsg.type = TILEXR_CCU_TLV_MSG_TYPE_CCU_UNINIT; + (void)RaTlvRequest(it->second.handle, TILEXR_CCU_TLV_MODULE_TYPE_CCU, &sendMsg, &recvMsg); + } + if (RaTlvDeinit != nullptr && it->second.handle != nullptr) { + (void)RaTlvDeinit(it->second.handle); + } + g_ccuTlvSessions.erase(it); + } + ccuTlvInitialized_ = false; + ccuTlvDevicePhyId_ = 0; +} + +void TileXRCcuHccpLoader::ReleaseRaHdc() +{ + if (!raHdcInitialized_) { + return; + } + const RaHdcKey key(raInitConfig_.phyId, raInitConfig_.hdcType); + std::lock_guard lock(g_raHdcMtx); + auto it = g_raHdcRefs.find(key); + if (it != g_raHdcRefs.end() && it->second > 1U) { + --it->second; + ReleaseNetServiceLocked(nullptr); + raHdcInitialized_ = false; + raInitConfig_ = TileXRCcuRaInitConfig {}; + return; + } + if (RaDeinit != nullptr) { + (void)RaDeinit(&raInitConfig_); + } + if (it != g_raHdcRefs.end()) { + g_raHdcRefs.erase(it); + } + ReleaseNetServiceLocked(nullptr); + raHdcInitialized_ = false; + raInitConfig_ = TileXRCcuRaInitConfig {}; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.h b/src/comm/ccu/tilexr_ccu_hccp_loader.h new file mode 100644 index 00000000..93ea1348 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_HCCP_LOADER_H +#define TILEXR_CCU_HCCP_LOADER_H + +#include "ccu/tilexr_ccu_hccp_types.h" +#include "tilexr_types.h" + +#include + +namespace TileXR { + +struct TileXRCcuHccpLoaderReport { + bool loaded = false; + bool endpointRouteProviderConfigured = false; + bool endpointRouteProviderLoaded = false; + bool raInitialized = false; + bool ccuTlvInitialized = false; + uint32_t logicDevId = 0; + uint32_t devicePhyId = 0; + uint32_t raInitRefCount = 0; + uint32_t netServiceRefCount = 0; + uint32_t ccuTlvRefCount = 0; + uint32_t ccuTlvBufferSize = 0; + int hdcType = 0; + int runtimePhyIdRet = 0; + int rtOpenNetServiceRet = 0; + int rtCloseNetServiceRet = 0; + int raInitRet = 0; + int raDeinitRet = 0; + int raTlvInitRet = 0; + int raTlvRequestRet = 0; + int raTlvDeinitRet = 0; + std::string message; +}; + +class TileXRCcuHccpLoader { +public: + TileXRCcuHccpLoader() = default; + ~TileXRCcuHccpLoader(); + TileXRCcuHccpLoader(const TileXRCcuHccpLoader&) = delete; + TileXRCcuHccpLoader& operator=(const TileXRCcuHccpLoader&) = delete; + + int Load(TileXRCcuHccpLoaderReport* report); + int LoadEndpointRouteProviderFromEnv(TileXRCcuHccpLoaderReport* report); + int InitRaHdc( + uint32_t devicePhyId, + int hdcType, + bool enableHdcAsync, + TileXRCcuHccpLoaderReport* report); + int InitCcuTlv(uint32_t devicePhyId, TileXRCcuHccpLoaderReport* report); + void Unload(); + bool IsLoaded() const; + int ResolveDevicePhyId(uint32_t logicDevId, uint32_t* phyId, TileXRCcuHccpLoaderReport* report = nullptr) const; + + TileXRCcuRaCustomChannelFunc RaCustomChannel = nullptr; + TileXRCcuRtGetDevicePhyIdByIndexFunc RtGetDevicePhyIdByIndex = nullptr; + TileXRCcuRtOpenNetServiceFunc RtOpenNetService = nullptr; + TileXRCcuRtCloseNetServiceFunc RtCloseNetService = nullptr; + TileXRCcuRaInitFunc RaInit = nullptr; + TileXRCcuRaDeinitFunc RaDeinit = nullptr; + TileXRCcuRaTlvInitFunc RaTlvInit = nullptr; + TileXRCcuRaTlvRequestFunc RaTlvRequest = nullptr; + TileXRCcuRaTlvDeinitFunc RaTlvDeinit = nullptr; + TileXRCcuRaGetDevEidInfoNumFunc RaGetDevEidInfoNum = nullptr; + TileXRCcuRaGetDevEidInfoListFunc RaGetDevEidInfoList = nullptr; + TileXRCcuRaCtxInitFunc RaCtxInit = nullptr; + TileXRCcuRaCtxDeinitFunc RaCtxDeinit = nullptr; + TileXRCcuRaCtxTokenIdAllocFunc RaCtxTokenIdAlloc = nullptr; + TileXRCcuRaCtxTokenIdFreeFunc RaCtxTokenIdFree = nullptr; + TileXRCcuRaCtxLmemRegisterFunc RaCtxLmemRegister = nullptr; + TileXRCcuRaCtxLmemUnregisterFunc RaCtxLmemUnregister = nullptr; + TileXRCcuRaGetSecRandomFunc RaGetSecRandom = nullptr; + TileXRCcuRaCtxChanCreateFunc RaCtxChanCreate = nullptr; + TileXRCcuRaCtxChanDestroyFunc RaCtxChanDestroy = nullptr; + TileXRCcuRaCtxCqCreateFunc RaCtxCqCreate = nullptr; + TileXRCcuRaCtxCqDestroyFunc RaCtxCqDestroy = nullptr; + TileXRCcuRaCtxQpCreateFunc RaCtxQpCreate = nullptr; + TileXRCcuRaCtxQpDestroyFunc RaCtxQpDestroy = nullptr; + TileXRCcuRaCtxQpImportFunc RaCtxQpImport = nullptr; + TileXRCcuRaCtxQpUnimportFunc RaCtxQpUnimport = nullptr; + TileXRCcuRaCtxQpBindFunc RaCtxQpBind = nullptr; + TileXRCcuRaCtxQpUnbindFunc RaCtxQpUnbind = nullptr; + TileXRCcuRaGetTpInfoListAsyncFunc RaGetTpInfoListAsync = nullptr; + TileXRCcuRaGetAsyncReqResultFunc RaGetAsyncReqResult = nullptr; + TileXRCcuEndpointRouteProviderFunc CollectLocalEndpointRoute = nullptr; + +private: + void ReleaseCcuTlv(); + void ReleaseRaHdc(); + int AcquireNetServiceLocked(int hdcType, TileXRCcuHccpLoaderReport* report); + void ReleaseNetServiceLocked(TileXRCcuHccpLoaderReport* report = nullptr); + + void* raHandle_ = nullptr; + void* runtimeHandle_ = nullptr; + void* endpointRouteProviderHandle_ = nullptr; + TileXRCcuRaInitConfig raInitConfig_ = {}; + uint32_t ccuTlvDevicePhyId_ = 0; + bool raHdcInitialized_ = false; + bool ccuTlvInitialized_ = false; + bool loaded_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_HCCP_LOADER_H diff --git a/src/comm/ccu/tilexr_ccu_hccp_types.h b/src/comm/ccu/tilexr_ccu_hccp_types.h new file mode 100644 index 00000000..abc75b1a --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_hccp_types.h @@ -0,0 +1,584 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_HCCP_TYPES_H +#define TILEXR_CCU_HCCP_TYPES_H + +#include "ccu/tilexr_ccu_abi_constants.h" + +#include + +namespace TileXR { + +constexpr int TILEXR_CCU_NETWORK_OFFLINE = 1; +constexpr int TILEXR_CCU_HDC_SERVICE_TYPE_RDMA = 6; +constexpr int TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2 = 18; +constexpr uint32_t TILEXR_CCU_CUSTOM_CHAN_DATA_MAX_SIZE = 2048; +constexpr int TILEXR_CCU_TLV_VERSION = 1; +constexpr uint32_t TILEXR_CCU_TLV_MODULE_TYPE_CCU = 1; +constexpr uint32_t TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT = 0; +constexpr uint32_t TILEXR_CCU_TLV_MSG_TYPE_CCU_UNINIT = 1; +constexpr uint32_t TILEXR_CCU_HCCP_DEV_EID_INFO_NAME_BYTES = 64; +constexpr uint32_t TILEXR_CCU_HCCP_MEM_KEY_BYTES = 128; +constexpr uint32_t TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT = 1; +constexpr uint32_t TILEXR_CCU_HCCP_MEM_SEG_ACCESS_READ = 1U << 1U; +constexpr uint32_t TILEXR_CCU_HCCP_MEM_SEG_ACCESS_WRITE = 1U << 2U; +constexpr uint32_t TILEXR_CCU_HCCP_MEM_SEG_ACCESS_ATOMIC = 1U << 3U; +constexpr uint32_t TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT = + TILEXR_CCU_HCCP_MEM_SEG_ACCESS_READ | + TILEXR_CCU_HCCP_MEM_SEG_ACCESS_WRITE | + TILEXR_CCU_HCCP_MEM_SEG_ACCESS_ATOMIC; +constexpr uint32_t TILEXR_CCU_HCCP_QP_KEY_BYTES = 64; +constexpr uint32_t TILEXR_CCU_HCCP_CQ_DEPTH_DEFAULT = 16384; +constexpr uint32_t TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT = 256; +constexpr uint32_t TILEXR_CCU_HCCP_JETTY_MODE_CCU = 2; +constexpr uint32_t TILEXR_CCU_HCCP_TRANSPORT_MODE_RM = 1; +constexpr uint32_t TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP = 1; +constexpr uint32_t TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR = 0; +constexpr uint32_t TILEXR_CCU_HCCP_TARGET_TYPE_JETTY = 1; +constexpr uint32_t TILEXR_CCU_HCCP_TP_TYPE_RTP = 0; +constexpr uint32_t TILEXR_CCU_HCCP_TP_TYPE_CTP = 1; +constexpr uint8_t TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT = 7; + +struct TileXRCcuDataByte8 { + char raw[8]; +}; + +struct TileXRCcuDataByte32 { + char raw[32]; +}; + +struct TileXRCcuDataByte64 { + char raw[64]; +}; + +struct TileXRCcuCustomChannelCaps { + uint32_t cap0; + uint32_t cap1; + uint32_t cap2; + uint32_t cap3; + uint32_t cap4; +}; + +struct TileXRCcuInstrInfo { + uint64_t resourceAddr; +}; + +struct TileXRCcuDieInfo { + uint32_t enableFlag; +}; + +struct TileXRCcuBaseInfoData { + uint32_t msId; + uint32_t tokenId; + uint32_t tokenValue; + uint32_t tokenValid; + uint32_t missionKey; + uint64_t resourceAddr; + TileXRCcuCustomChannelCaps caps; +}; + +union TileXRCcuDataTypeUnion { + TileXRCcuDataByte8 byte8; + TileXRCcuDataByte32 byte32; + TileXRCcuDataByte64 byte64; + TileXRCcuBaseInfoData baseinfo; + TileXRCcuInstrInfo insinfo; + TileXRCcuDieInfo dieinfo; +}; + +struct TileXRCcuData { + uint32_t udieIdx; + uint32_t dataLen; + uint32_t dataArraySize; + TileXRCcuDataTypeUnion dataArray[8]; +}; + +union TileXRCcuDataUnion { + char raw[TILEXR_CCU_CUSTOM_CHAN_DATA_MAX_SIZE]; + TileXRCcuData dataInfo; +}; + +struct TileXRCcuCustomChannelIn { + TileXRCcuDataUnion data; + uint32_t offsetStartIdx; + uint32_t op; +}; + +struct TileXRCcuCustomChannelOut { + TileXRCcuDataUnion data; + uint32_t offsetNextIdx; + int opRet; +}; + +struct TileXRCcuRaInfo { + int mode; + uint32_t phyId; +}; + +union TileXRCcuHccpEid { + uint8_t raw[TILEXR_CCU_EID_BYTES]; + struct { + uint64_t reserved; + uint32_t prefix; + uint32_t addr; + } in4; + struct { + uint64_t subnetPrefix; + uint64_t interfaceId; + } in6; +}; + +struct TileXRCcuHccpDevEidInfo { + char name[TILEXR_CCU_HCCP_DEV_EID_INFO_NAME_BYTES]; + uint32_t type; + uint32_t eidIndex; + TileXRCcuHccpEid eid; + uint32_t dieId; + uint32_t chipId; + uint32_t funcId; + uint32_t resv; +}; + +struct TileXRCcuHccpCtxInitCfg { + int mode; + union { + struct { + bool disabledLiteThread; + } rdma; + }; +}; + +struct TileXRCcuHccpCtxInitAttr { + uint32_t phyId; + union { + uint8_t rdmaPad[24]; + struct { + uint32_t eidIndex; + TileXRCcuHccpEid eid; + } ub; + }; + uint32_t resv[16]; +}; + +struct TileXRCcuHccpTokenId { + uint32_t tokenId; +}; + +struct TileXRCcuHccpMemKey { + uint8_t value[TILEXR_CCU_HCCP_MEM_KEY_BYTES]; + uint8_t size; +}; + +struct TileXRCcuHccpMemInfo { + uint64_t addr; + uint64_t size; +}; + +union TileXRCcuHccpRegSegFlag { + struct { + uint32_t tokenPolicy : 3; + uint32_t cacheable : 1; + uint32_t dsva : 1; + uint32_t access : 6; + uint32_t nonPin : 1; + uint32_t userIova : 1; + uint32_t tokenIdValid : 1; + uint32_t reserved : 18; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpMemRegAttr { + TileXRCcuHccpMemInfo mem; + union { + struct { + int access; + } rdma; + struct { + TileXRCcuHccpRegSegFlag flags; + uint32_t tokenValue; + void* tokenIdHandle; + } ub; + }; + uint32_t resv[8]; +}; + +struct TileXRCcuHccpMemRegInfo { + TileXRCcuHccpMemKey key; + union { + struct { + uint32_t lkey; + } rdma; + struct { + uint32_t tokenId; + uint64_t targetSegHandle; + } ub; + }; + uint32_t resv[8]; +}; + +struct TileXRCcuHccpMrRegInfo { + TileXRCcuHccpMemRegAttr in; + TileXRCcuHccpMemRegInfo out; +}; + +union TileXRCcuHccpDataPlaneCstmFlag { + struct { + uint32_t pollCqCstm : 1; + uint32_t reserved : 31; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpChanInfo { + struct { + TileXRCcuHccpDataPlaneCstmFlag dataPlaneFlag; + } in; + struct { + int fd; + } out; +}; + +union TileXRCcuHccpJfcFlag { + struct { + uint32_t lockFree : 1; + uint32_t jfcInline : 1; + uint32_t reserved : 30; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpCqInfo { + struct { + void* chanHandle; + uint32_t depth; + union { + struct { + uint64_t cqContext; + uint32_t mode; + uint32_t compVector; + } rdma; + struct { + uint64_t userCtx; + int mode; + uint32_t ceqn; + TileXRCcuHccpJfcFlag flag; + struct { + bool valid; + uint32_t cqeFlag; + } ccuExCfg; + } ub; + }; + } in; + struct { + uint64_t va; + uint32_t id; + uint32_t cqeSize; + uint64_t bufAddr; + uint64_t swdbAddr; + } out; +}; + +union TileXRCcuHccpJettyFlag { + struct { + uint32_t shareJfr : 1; + uint32_t reserved : 31; + } bs; + uint32_t value; +}; + +union TileXRCcuHccpJfsFlag { + struct { + uint32_t lockFree : 1; + uint32_t errorSuspend : 1; + uint32_t outorderComp : 1; + uint32_t orderType : 8; + uint32_t multiPath : 1; + uint32_t reserved : 20; + } bs; + uint32_t value; +}; + +union TileXRCcuHccpCstmJfsFlag { + struct { + uint32_t sqCstm : 1; + uint32_t dbCstm : 1; + uint32_t dbCtlCstm : 1; + uint32_t reserved : 29; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpJettyQueCfgEx { + uint32_t buffSize; + uint64_t buffVa; +}; + +struct TileXRCcuHccpQpCreateAttr { + void* scqHandle; + void* rcqHandle; + void* srqHandle; + uint32_t sqDepth; + uint32_t rqDepth; + int transportMode; + union { + struct { + uint32_t mode; + uint32_t udpSport; + uint8_t trafficClass; + uint8_t sl; + uint8_t timeout; + uint8_t rnrRetry; + uint8_t retryCnt; + } rdma; + struct { + int mode; + uint32_t jettyId; + TileXRCcuHccpJettyFlag flag; + TileXRCcuHccpJfsFlag jfsFlag; + void* tokenIdHandle; + uint32_t tokenValue; + uint8_t priority; + uint8_t rnrRetry; + uint8_t errTimeout; + union { + struct { + TileXRCcuHccpJettyQueCfgEx sq; + bool piType; + TileXRCcuHccpCstmJfsFlag cstmFlag; + uint32_t sqebbNum; + } extMode; + struct { + bool lockFlag; + uint32_t sqeBufIdx; + } taCacheMode; + }; + } ub; + }; + uint32_t resv[16]; +}; + +struct TileXRCcuHccpQpKey { + uint8_t value[TILEXR_CCU_HCCP_QP_KEY_BYTES]; + uint8_t size; +}; + +struct TileXRCcuHccpQpCreateInfo { + TileXRCcuHccpQpKey key; + union { + struct { + uint32_t qpn; + } rdma; + struct { + uint32_t uasid; + uint32_t id; + uint64_t sqBuffVa; + uint64_t wqebbSize; + uint64_t dbAddr; + uint32_t dbTokenId; + uint64_t ciAddr; + } ub; + }; + uint64_t va; + uint32_t resv[16]; +}; + +union TileXRCcuHccpImportJettyFlag { + struct { + uint32_t tokenPolicy : 3; + uint32_t orderType : 8; + uint32_t shareTp : 1; + uint32_t reserved : 20; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpJettyImportExpCfg { + uint64_t tpHandle; + uint64_t peerTpHandle; + uint64_t tag; + uint32_t txPsn; + uint32_t rxPsn; + uint32_t rsv[16]; +}; + +struct TileXRCcuHccpQpImportInfo { + struct { + TileXRCcuHccpQpKey key; + union { + struct { + int mode; + uint32_t tokenValue; + int policy; + int type; + TileXRCcuHccpImportJettyFlag flag; + TileXRCcuHccpJettyImportExpCfg expImportCfg; + uint32_t tpType; + } ub; + }; + uint32_t resv[7]; + } in; + struct { + union { + struct { + uint64_t tjettyHandle; + uint32_t tpn; + } ub; + }; + uint32_t resv[8]; + } out; +}; + +union TileXRCcuHccpGetTpCfgFlag { + struct { + uint32_t ctp : 1; + uint32_t rtp : 1; + uint32_t utp : 1; + uint32_t uboe : 1; + uint32_t preDefined : 1; + uint32_t dynamicDefined : 1; + uint32_t reserved : 26; + } bs; + uint32_t value; +}; + +struct TileXRCcuHccpGetTpCfg { + TileXRCcuHccpGetTpCfgFlag flag; + int transMode; + TileXRCcuHccpEid localEid; + TileXRCcuHccpEid peerEid; +}; + +struct TileXRCcuHccpTpInfo { + uint64_t tpHandle; + uint32_t resv; +}; + +struct TileXRCcuRaInitConfig { + uint32_t phyId; + uint32_t nicPosition; + int hdcType; + bool enableHdcAsync; +}; + +struct TileXRCcuRtProcExtParam { + const char* paramInfo; + uint64_t paramLen; +}; + +struct TileXRCcuRtNetServiceOpenArgs { + TileXRCcuRtProcExtParam* extParamList; + uint64_t extParamCnt; +}; + +struct TileXRCcuTlvInitInfo { + int version; + uint32_t phyId; + uint32_t nicPosition; + uint32_t reserved[16]; +}; + +struct TileXRCcuTlvMsg { + uint32_t type; + uint32_t length; + char* data; +}; + +struct TileXRCcuEndpointRouteProviderResourceWindow { + uint64_t addr = 0; + uint64_t bytes = 0; + uint32_t tokenId = 0; + uint32_t rawTokenId = 0; + uint32_t tokenValue = 0; +}; + +struct TileXRCcuEndpointRouteProviderRoute { + uint8_t remoteEid[TILEXR_CCU_EID_BYTES] = {}; + uint32_t tpn = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + bool endpointRouteVerified = false; +}; + +using TileXRCcuRaCustomChannelFunc = int (*)( + TileXRCcuRaInfo info, + TileXRCcuCustomChannelIn* in, + TileXRCcuCustomChannelOut* out); + +using TileXRCcuRtGetDevicePhyIdByIndexFunc = int (*)(uint32_t logicDevId, uint32_t* phyId); +using TileXRCcuRtOpenNetServiceFunc = int (*)(const TileXRCcuRtNetServiceOpenArgs* args); +using TileXRCcuRtCloseNetServiceFunc = int (*)(); + +using TileXRCcuRaInitFunc = int (*)(TileXRCcuRaInitConfig* config); +using TileXRCcuRaDeinitFunc = int (*)(TileXRCcuRaInitConfig* config); +using TileXRCcuRaTlvInitFunc = int (*)(TileXRCcuTlvInitInfo* initInfo, uint32_t* bufferSize, void** tlvHandle); +using TileXRCcuRaTlvRequestFunc = int (*)( + void* tlvHandle, + uint32_t moduleType, + TileXRCcuTlvMsg* sendMsg, + TileXRCcuTlvMsg* recvMsg); +using TileXRCcuRaTlvDeinitFunc = int (*)(void* tlvHandle); +using TileXRCcuRaGetDevEidInfoNumFunc = int (*)(TileXRCcuRaInfo info, uint32_t* num); +using TileXRCcuRaGetDevEidInfoListFunc = int (*)( + TileXRCcuRaInfo info, + TileXRCcuHccpDevEidInfo list[], + uint32_t* num); +using TileXRCcuRaCtxInitFunc = int (*)( + TileXRCcuHccpCtxInitCfg* cfg, + TileXRCcuHccpCtxInitAttr* attr, + void** ctx); +using TileXRCcuRaCtxDeinitFunc = int (*)(void* ctx); +using TileXRCcuRaCtxTokenIdAllocFunc = int (*)( + void* ctx, + TileXRCcuHccpTokenId* token, + void** tokenHandle); +using TileXRCcuRaCtxTokenIdFreeFunc = int (*)(void* ctx, void* tokenHandle); +using TileXRCcuRaCtxLmemRegisterFunc = int (*)( + void* ctx, + TileXRCcuHccpMrRegInfo* mr, + void** lmemHandle); +using TileXRCcuRaCtxLmemUnregisterFunc = int (*)(void* ctx, void* lmemHandle); +using TileXRCcuRaGetSecRandomFunc = int (*)(TileXRCcuRaInfo* info, uint32_t* value); +using TileXRCcuRaCtxChanCreateFunc = int (*)(void* ctx, TileXRCcuHccpChanInfo* info, void** chanHandle); +using TileXRCcuRaCtxChanDestroyFunc = int (*)(void* ctx, void* chanHandle); +using TileXRCcuRaCtxCqCreateFunc = int (*)(void* ctx, TileXRCcuHccpCqInfo* info, void** cqHandle); +using TileXRCcuRaCtxCqDestroyFunc = int (*)(void* ctx, void* cqHandle); +using TileXRCcuRaCtxQpCreateFunc = int (*)( + void* ctx, + TileXRCcuHccpQpCreateAttr* attr, + TileXRCcuHccpQpCreateInfo* info, + void** qpHandle); +using TileXRCcuRaCtxQpDestroyFunc = int (*)(void* qpHandle); +using TileXRCcuRaCtxQpImportFunc = int (*)( + void* ctx, + TileXRCcuHccpQpImportInfo* info, + void** remoteQpHandle); +using TileXRCcuRaCtxQpUnimportFunc = int (*)(void* ctx, void* remoteQpHandle); +using TileXRCcuRaCtxQpBindFunc = int (*)(void* qpHandle, void* remoteQpHandle); +using TileXRCcuRaCtxQpUnbindFunc = int (*)(void* qpHandle); +using TileXRCcuRaGetTpInfoListAsyncFunc = int (*)( + void* ctx, + TileXRCcuHccpGetTpCfg* cfg, + TileXRCcuHccpTpInfo infoList[], + uint32_t* num, + void** reqHandle); +using TileXRCcuRaGetAsyncReqResultFunc = int (*)(void* reqHandle, int* reqResult); + +using TileXRCcuEndpointRouteProviderFunc = int (*)( + uint32_t devicePhyId, + const TileXRCcuEndpointRouteProviderResourceWindow* localResourceWindow, + TileXRCcuEndpointRouteProviderRoute* route); + +static_assert(sizeof(TileXRCcuCustomChannelIn::data.raw) == TILEXR_CCU_CUSTOM_CHAN_DATA_MAX_SIZE, + "CCU custom channel input must match HCCP custom_chan_info_in data size"); +static_assert(sizeof(TileXRCcuCustomChannelOut::data.raw) == TILEXR_CCU_CUSTOM_CHAN_DATA_MAX_SIZE, + "CCU custom channel output must match HCCP custom_chan_info_out data size"); + +} // namespace TileXR + +#endif // TILEXR_CCU_HCCP_TYPES_H diff --git a/src/comm/ccu/tilexr_ccu_install_provider.cpp b/src/comm/ccu/tilexr_ccu_install_provider.cpp new file mode 100644 index 00000000..f0adbe71 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_install_provider.cpp @@ -0,0 +1,1534 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_install_provider.h" + +#include + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuInstallProviderReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRCcuInstallProviderReport{}; +} + +void ResetManifestReport(TileXRCcuInstallManifestReport* report) +{ + if (report == nullptr) { + return; + } + report->requirementCount = 0; + report->message.clear(); +} + +int FailManifest(TileXRCcuInstallManifest* manifest, TileXRCcuInstallManifestReport* report, const std::string& message) +{ + if (manifest != nullptr) { + *manifest = TileXRCcuInstallManifest{}; + } + if (report != nullptr) { + report->requirementCount = 0; + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +void MarkMissingStep(TileXRCcuInstallStepEvidence* step, const std::string& message) +{ + if (step == nullptr) { + return; + } + step->satisfied = false; + step->source.kind = TileXRCcuEvidenceKind::Missing; + step->source.surface = TileXRCcuEvidenceSurface::Unspecified; + step->source.source.clear(); + step->source.detail = message; + step->message = message; +} + +void FillUnsupportedReport(TileXRCcuInstallProviderReport* report) +{ + if (report == nullptr) { + return; + } + const std::string message = "no public no-hcomm CCU install provider is available"; + report->offlineOnly = true; + report->installAttempted = false; + report->installSucceeded = false; + report->message = message; + MarkMissingStep(&report->mission, message); + MarkMissingStep(&report->repository, message); + MarkMissingStep(&report->localXn, message); + MarkMissingStep(&report->remoteXn, message); + MarkMissingStep(&report->notifyCke, message); + MarkMissingStep(&report->channelBinding, message); +} + +void FillScopeFailureReport(TileXRCcuInstallProviderReport* report, const std::string& message) +{ + if (report == nullptr) { + return; + } + report->offlineOnly = true; + report->installAttempted = false; + report->installSucceeded = false; + report->message = message; + MarkMissingStep(&report->mission, message); + MarkMissingStep(&report->repository, message); + MarkMissingStep(&report->localXn, message); + MarkMissingStep(&report->remoteXn, message); + MarkMissingStep(&report->notifyCke, message); + MarkMissingStep(&report->channelBinding, message); +} + +TileXRCcuEvidenceSource MissingSource(const std::string& detail) +{ + TileXRCcuEvidenceSource source; + source.kind = TileXRCcuEvidenceKind::Missing; + source.surface = TileXRCcuEvidenceSurface::Unspecified; + source.detail = detail; + return source; +} + +TileXRCcuEvidenceSource SourceOrMissing(const TileXRCcuInstallStepEvidence& step, const std::string& detail) +{ + return step.satisfied ? step.source : MissingSource(detail); +} + +uint64_t MixReceiptWord(uint64_t hash, uint64_t value) +{ + constexpr uint64_t prime = 1099511628211ULL; + for (uint32_t i = 0; i < 8U; ++i) { + hash ^= static_cast((value >> (i * 8U)) & 0xffU); + hash *= prime; + } + return hash; +} + +uint64_t BuildRepositoryInstallReceiptId( + const TileXRCcuLaunchPackage& package, + const TileXRCcuRepositoryInstallReceipt& receipt) +{ + uint64_t hash = 1469598103934665603ULL; + hash = MixReceiptWord(hash, TileXRCcuComputeLaunchPackageFingerprint(package)); + hash = MixReceiptWord(hash, receipt.dieId); + hash = MixReceiptWord(hash, receipt.instructionStartId); + hash = MixReceiptWord(hash, receipt.instructionCount); + hash = MixReceiptWord(hash, receipt.instructionBytes); + hash = MixReceiptWord(hash, receipt.deviceInstructionAddr); + return hash == 0 ? 1 : hash; +} + +TileXRCcuInstallStepEvidence PublicVerifiedStep( + const TileXRCcuLaunchPackage& package, + const TileXRCcuInstallRequest& request, + uint64_t receiptId, + const std::string& source, + const std::string& detail, + bool endpointRouteVerified = false) +{ + TileXRCcuInstallStepEvidence step; + step.satisfied = true; + step.source.kind = TileXRCcuEvidenceKind::PublicVerified; + step.source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + step.source.packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + step.source.deviceId = request.deviceId; + step.source.rank = request.rank; + step.source.provider = request.provider; + step.source.installAttemptReceiptId = receiptId; + step.source.endpointRouteVerified = endpointRouteVerified; + step.source.source = source; + step.source.detail = detail; + step.message = detail; + return step; +} + +TileXRCcuInstallRequirement BaseRequirement( + TileXRCcuInstallRequirementKind kind, + uint64_t packageFingerprint, + uint8_t dieId, + const std::string& label, + const std::string& detail) +{ + TileXRCcuInstallRequirement requirement; + requirement.kind = kind; + requirement.packageFingerprint = packageFingerprint; + requirement.dieId = dieId; + requirement.label = label; + requirement.detail = detail; + return requirement; +} + +uint32_t TotalBindingCount(const std::vector& resources) +{ + uint32_t total = 0; + for (const auto& resource : resources) { + total += resource.bindingCount; + } + return total; +} + +bool SameBindingChannel( + const TileXRCcuRemoteXnBindingProof& lhs, + const TileXRCcuRemoteXnBindingProof& rhs) +{ + return lhs.dieId == rhs.dieId && lhs.channelId == rhs.channelId; +} + +bool HasInstalledChannel( + const std::vector& channels, + uint8_t dieId, + uint16_t channelId) +{ + for (const auto& channel : channels) { + if (channel.dieId == dieId && channel.channelId == channelId) { + return true; + } + } + return false; +} + +uint32_t CountUniqueRemoteBindingChannels(const std::vector& bindings) +{ + uint32_t count = 0; + for (size_t i = 0; i < bindings.size(); ++i) { + bool seen = false; + for (size_t j = 0; j < i; ++j) { + if (SameBindingChannel(bindings[i], bindings[j])) { + seen = true; + break; + } + } + if (!seen) { + ++count; + } + } + return count; +} + +uint32_t CountInstalledRemoteBindingChannels( + const std::vector& bindings, + const std::vector& channels) +{ + uint32_t count = 0; + for (size_t i = 0; i < bindings.size(); ++i) { + bool seen = false; + for (size_t j = 0; j < i; ++j) { + if (SameBindingChannel(bindings[i], bindings[j])) { + seen = true; + break; + } + } + if (!seen && HasInstalledChannel(channels, bindings[i].dieId, bindings[i].channelId)) { + ++count; + } + } + return count; +} + +const TileXRCcuInstallRequirement* FindRequirement( + const TileXRCcuInstallManifest& manifest, + TileXRCcuInstallRequirementKind kind); + +bool RangeContainsId(const TileXRCcuRange& range, uint8_t dieId, uint16_t id) +{ + if (range.dieId != dieId || range.num == 0) { + return false; + } + const uint32_t begin = range.startId; + const uint32_t end = begin + range.num; + return id >= begin && id < end; +} + +bool RepositoryMissionWindowContainsTask(const TileXRCcuRepositoryImage& repository, const TileXRCcuTask& task) +{ + if (repository.dieId != task.dieId || repository.missionCount == 0 || task.instCnt == 0) { + return false; + } + const uint32_t begin = repository.missionStartId; + const uint32_t end = begin + repository.missionCount; + const uint32_t taskBegin = task.instStartId; + const uint32_t taskEnd = taskBegin + task.instCnt; + return taskBegin >= begin && taskEnd <= end; +} + +bool FailMissionLaunchDescriptorProof(std::string* diagnostic, const std::string& reason) +{ + if (diagnostic != nullptr) { + *diagnostic = "mission/key launch task descriptor proof failed: " + reason; + } + return false; +} + +bool ValidateMissionLaunchDescriptorProof( + const TileXRCcuLaunchPackage& package, + const TileXRCcuInstallManifest* manifest, + std::string* diagnostic) +{ + if (diagnostic != nullptr) { + diagnostic->clear(); + } + if (!package.plan.mission.installed || package.plan.mission.key == 0) { + return FailMissionLaunchDescriptorProof(diagnostic, "mission key is not installed in the producer plan"); + } + if (!RangeContainsId( + package.plan.kernelLocalMission, + package.plan.mission.dieId, + package.plan.mission.missionId)) { + return FailMissionLaunchDescriptorProof( + diagnostic, + "mission id is outside the kernel-local mission range"); + } + if (manifest == nullptr) { + return FailMissionLaunchDescriptorProof(diagnostic, "install manifest mission requirement is missing"); + } + const TileXRCcuInstallRequirement* mission = + FindRequirement(*manifest, TileXRCcuInstallRequirementKind::MissionKey); + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (mission == nullptr || + mission->packageFingerprint != packageFingerprint || + mission->dieId != package.plan.mission.dieId || + mission->missionId != package.plan.mission.missionId || + mission->missionKey != package.plan.mission.key || + mission->resourceStartId != package.plan.kernelLocalMission.startId || + mission->resourceCount != package.plan.kernelLocalMission.num) { + return FailMissionLaunchDescriptorProof( + diagnostic, + "install manifest mission requirement does not match the launch package"); + } + if (package.repository.instructions.empty() || + package.repository.dieId != package.plan.mission.dieId || + package.repository.missionCount == 0) { + return FailMissionLaunchDescriptorProof(diagnostic, "repository mission instruction window is missing"); + } + if (package.tasks.empty()) { + return FailMissionLaunchDescriptorProof(diagnostic, "launch task descriptor list is missing"); + } + for (const auto& task : package.tasks) { + if (task.dieId != package.plan.mission.dieId || + task.missionId != package.plan.mission.missionId || + task.key != package.plan.mission.key) { + return FailMissionLaunchDescriptorProof( + diagnostic, + "launch task descriptor mission id or mission key does not match the producer plan"); + } + if (!RepositoryMissionWindowContainsTask(package.repository, task)) { + return FailMissionLaunchDescriptorProof( + diagnostic, + "launch task descriptor instruction range is outside the repository mission window"); + } + } + if (diagnostic != nullptr) { + *diagnostic = + "mission/key carried by launch task descriptor mission id and mission key within repository window"; + } + return true; +} + +bool FailRemoteXnExchangeBindingProof(std::string* diagnostic, const std::string& reason) +{ + if (diagnostic != nullptr) { + *diagnostic = "remote XN peer exchange proof failed: " + reason; + } + return false; +} + +bool HasInstalledChannelRoute( + const TileXRCcuLowerLayerInstallPlan& lowerLayerPlan, + uint8_t dieId, + uint16_t channelId) +{ + for (const auto& channel : lowerLayerPlan.channels) { + if (channel.dieId == dieId && channel.channelId == channelId) { + return true; + } + } + return false; +} + +bool HasVerifiedEndpointRoutes(const TileXRCcuLowerLayerInstallPlan* lowerLayerPlan) +{ + if (lowerLayerPlan == nullptr || lowerLayerPlan->remoteXnBindings.empty()) { + return false; + } + for (const auto& proof : lowerLayerPlan->remoteXnBindings) { + if (!proof.peerExchangeObserved || !proof.endpointRouteVerified || + !HasInstalledChannelRoute(*lowerLayerPlan, proof.dieId, proof.channelId)) { + return false; + } + } + return true; +} + +bool HasVerifiedChannelResourceBindings(const TileXRCcuLowerLayerInstallPlan* lowerLayerPlan) +{ + if (lowerLayerPlan == nullptr || lowerLayerPlan->remoteXnBindings.empty()) { + return false; + } + for (const auto& proof : lowerLayerPlan->remoteXnBindings) { + if (!proof.peerExchangeObserved || + !proof.endpointRouteVerified || + !proof.channelResourceOwnerVerified || + !proof.transportResourceExchangeVerified || + !HasInstalledChannelRoute(*lowerLayerPlan, proof.dieId, proof.channelId)) { + return false; + } + } + return true; +} + +bool RequiresRemoteXnBindingEvidence(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXn || + mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +bool RequiresChannelBindingEvidence(TileXRCcuBarrierMode mode) +{ + return mode != TileXRCcuBarrierMode::LocalCke && + mode != TileXRCcuBarrierMode::LocalCkePostOnly; +} + +uint16_t EffectiveResourceLocalWaitCke(const TileXRCcuSyncResource& resource) +{ + return resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; +} + +uint16_t EffectiveProofLocalWaitCke(const TileXRCcuRemoteXnBindingProof& proof) +{ + return proof.localWaitCke == 0 ? proof.notifyCke : proof.localWaitCke; +} + +bool ValidateRemoteXnExchangeBindingProof( + const TileXRCcuLaunchPackage& package, + const TileXRCcuInstallManifest* manifest, + const TileXRCcuLowerLayerInstallPlan* lowerLayerPlan, + std::string* diagnostic) +{ + if (diagnostic != nullptr) { + diagnostic->clear(); + } + if (manifest == nullptr) { + return FailRemoteXnExchangeBindingProof(diagnostic, "install manifest remote XN requirement is missing"); + } + const TileXRCcuInstallRequirement* remoteXn = + FindRequirement(*manifest, TileXRCcuInstallRequirementKind::RemoteXnBinding); + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (remoteXn == nullptr || + remoteXn->packageFingerprint != packageFingerprint || + remoteXn->syncResourceCount != package.plan.syncResources.size()) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "install manifest remote XN requirement does not match the launch package"); + } + if (lowerLayerPlan == nullptr || lowerLayerPlan->remoteXnBindings.empty()) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "remote XN install provider is missing; peer exchange proof is missing"); + } + if (lowerLayerPlan->remoteXnBindings.size() != package.plan.syncResources.size()) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "peer exchange proof count does not match sync resource count"); + } + + for (const auto& resource : package.plan.syncResources) { + bool identityMatched = false; + bool remoteNotifyCkeMismatch = false; + bool localWaitCkeMismatch = false; + bool matched = false; + for (const auto& proof : lowerLayerPlan->remoteXnBindings) { + if (proof.dieId != resource.dieId || + proof.channelId != resource.channelId || + proof.localXn != resource.localXn || + proof.remoteXn != resource.remoteXn) { + continue; + } + identityMatched = true; + if (proof.notifyCke != resource.notifyCke) { + remoteNotifyCkeMismatch = true; + continue; + } + if (EffectiveProofLocalWaitCke(proof) != EffectiveResourceLocalWaitCke(resource)) { + localWaitCkeMismatch = true; + continue; + } + if (!proof.peerExchangeObserved) { + return FailRemoteXnExchangeBindingProof(diagnostic, "peer exchange was not observed"); + } + if (!HasInstalledChannelRoute(*lowerLayerPlan, proof.dieId, proof.channelId)) { + return FailRemoteXnExchangeBindingProof(diagnostic, "matching channel route is not installed"); + } + if (!proof.endpointRouteVerified) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "endpoint route provenance was not verified"); + } + if (!proof.channelResourceOwnerVerified) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "channel resource owner did not verify channel-bound remote XN allocation"); + } + if (!proof.transportResourceExchangeVerified) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "transport resource exchange did not verify remote XN and notify CKE binding"); + } + matched = true; + break; + } + if (!matched) { + if (identityMatched && remoteNotifyCkeMismatch) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "syncXn remote notify CKE is not covered by peer exchange proof"); + } + if (identityMatched && localWaitCkeMismatch) { + return FailRemoteXnExchangeBindingProof( + diagnostic, + "syncXn local wait CKE is not covered by peer exchange proof"); + } + return FailRemoteXnExchangeBindingProof( + diagnostic, + "syncXn remote XN operand is not covered by peer exchange proof"); + } + } + + if (diagnostic != nullptr) { + *diagnostic = + "remote XN peer exchange proof matches syncXn operands, verified endpoint route channel contexts, " + "channel resource owner allocation, and transport resource exchange"; + } + return true; +} + +int ValidateInstallManifestScope( + const TileXRCcuLaunchPackage& package, + uint64_t packageFingerprint, + TileXRCcuInstallManifest* manifest, + TileXRCcuInstallManifestReport* report) +{ + if (package.tasks.empty()) { + return FailManifest(manifest, report, "missing CCU launch tasks for install manifest"); + } + if (package.repository.instructions.empty()) { + return FailManifest(manifest, report, "missing CCU repository image for install manifest"); + } + if (package.installScope.packageFingerprint == 0 || package.installScope.provider.empty()) { + return FailManifest(manifest, report, "launch install scope is not bound"); + } + if (package.installScope.packageFingerprint != packageFingerprint) { + return FailManifest(manifest, report, "launch install scope is stale"); + } + return TILEXR_SUCCESS; +} + +bool HasRequirementKind(const TileXRCcuInstallManifest& manifest, TileXRCcuInstallRequirementKind kind) +{ + bool found = false; + for (const auto& requirement : manifest.requirements) { + if (requirement.kind != kind) { + continue; + } + if (found) { + return false; + } + found = true; + } + return found; +} + +const TileXRCcuInstallRequirement* FindRequirement( + const TileXRCcuInstallManifest& manifest, + TileXRCcuInstallRequirementKind kind) +{ + for (const auto& requirement : manifest.requirements) { + if (requirement.kind == kind) { + return &requirement; + } + } + return nullptr; +} + +int ValidateRequirementMetadata( + const TileXRCcuInstallRequirement* requirement, + uint64_t packageFingerprint, + TileXRCcuInstallProviderReport* report) +{ + if (requirement == nullptr || requirement->label.empty() || requirement->detail.empty() || + requirement->packageFingerprint != packageFingerprint) { + FillScopeFailureReport(report, "install manifest requirement metadata mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +bool IsPublicVerifiedStep(const TileXRCcuInstallStepEvidence& step) +{ + return step.satisfied && step.source.kind == TileXRCcuEvidenceKind::PublicVerified; +} + +uint32_t CountPublicVerifiedSteps(const TileXRCcuInstallProviderReport& report) +{ + uint32_t count = 0; + count += IsPublicVerifiedStep(report.mission) ? 1U : 0U; + count += IsPublicVerifiedStep(report.repository) ? 1U : 0U; + count += IsPublicVerifiedStep(report.localXn) ? 1U : 0U; + count += IsPublicVerifiedStep(report.remoteXn) ? 1U : 0U; + count += IsPublicVerifiedStep(report.notifyCke) ? 1U : 0U; + count += IsPublicVerifiedStep(report.channelBinding) ? 1U : 0U; + return count; +} + +bool IsRequirementSatisfied( + TileXRCcuInstallRequirementKind kind, + const TileXRCcuInstallProviderReport& report) +{ + switch (kind) { + case TileXRCcuInstallRequirementKind::MissionKey: + return IsPublicVerifiedStep(report.mission); + case TileXRCcuInstallRequirementKind::RepositoryImage: + return IsPublicVerifiedStep(report.repository); + case TileXRCcuInstallRequirementKind::LocalXn: + return IsPublicVerifiedStep(report.localXn); + case TileXRCcuInstallRequirementKind::RemoteXnBinding: + return IsPublicVerifiedStep(report.remoteXn); + case TileXRCcuInstallRequirementKind::NotifyCke: + return IsPublicVerifiedStep(report.notifyCke); + case TileXRCcuInstallRequirementKind::ChannelBinding: + return IsPublicVerifiedStep(report.channelBinding); + default: + return false; + } +} + +uint32_t CountPublicVerifiedRequiredSteps( + const TileXRCcuInstallManifest& manifest, + const TileXRCcuInstallProviderReport& report) +{ + uint32_t count = 0; + for (const auto& requirement : manifest.requirements) { + count += IsRequirementSatisfied(requirement.kind, report) ? 1U : 0U; + } + return count; +} + +void FillManifestInstallSurfaceCounts( + const TileXRCcuInstallManifest* manifest, + TileXRCcuInstallProviderReport* report) +{ + if (manifest == nullptr || report == nullptr) { + return; + } + report->requiredInstallSurfaceCount = static_cast(manifest->requirements.size()); + report->publicVerifiedInstallSurfaceCount = CountPublicVerifiedRequiredSteps(*manifest, *report); + report->missingInstallSurfaceCount = + report->requiredInstallSurfaceCount > report->publicVerifiedInstallSurfaceCount ? + report->requiredInstallSurfaceCount - report->publicVerifiedInstallSurfaceCount : + 0U; +} + +bool RejectOfflinePublicVerified(const TileXRCcuInstallProviderReport& installReport) +{ + return installReport.offlineOnly && + (IsPublicVerifiedStep(installReport.mission) || + IsPublicVerifiedStep(installReport.repository) || + IsPublicVerifiedStep(installReport.localXn) || + IsPublicVerifiedStep(installReport.remoteXn) || + IsPublicVerifiedStep(installReport.notifyCke) || + IsPublicVerifiedStep(installReport.channelBinding)); +} + +bool HasPublicVerifiedStep(const TileXRCcuInstallProviderReport& installReport) +{ + return IsPublicVerifiedStep(installReport.mission) || + IsPublicVerifiedStep(installReport.repository) || + IsPublicVerifiedStep(installReport.localXn) || + IsPublicVerifiedStep(installReport.remoteXn) || + IsPublicVerifiedStep(installReport.notifyCke) || + IsPublicVerifiedStep(installReport.channelBinding); +} + +int ValidateInstallReceiptStep( + const std::string& label, + uint64_t expectedReceiptId, + const TileXRCcuInstallStepEvidence& step) +{ + (void)label; + if (!IsPublicVerifiedStep(step)) { + return TILEXR_SUCCESS; + } + if (expectedReceiptId == 0 || step.source.installAttemptReceiptId == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (step.source.installAttemptReceiptId != expectedReceiptId) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int ValidatePublicVerifiedStepScope(const std::string& label, const TileXRCcuInstallStepEvidence& step) +{ + (void)label; + if (!IsPublicVerifiedStep(step)) { + return TILEXR_SUCCESS; + } + if (step.source.surface != TileXRCcuEvidenceSurface::PublicInstallProvider || + step.source.packageFingerprint == 0 || + step.source.provider.empty() || + step.source.source.empty() || + step.source.detail.empty()) { + const std::string reason = "public verified evidence scope is incomplete"; + (void)reason; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int ValidateInstallReceipt(const TileXRCcuInstallProviderReport& installReport) +{ + if (HasPublicVerifiedStep(installReport) && + (!installReport.installAttempted || !installReport.installSucceeded)) { + const std::string reason = "install attempt did not succeed"; + (void)reason; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("mission/key", installReport.mission) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("repository", installReport.repository) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("local XN", installReport.localXn) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("remote XN", installReport.remoteXn) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("notify CKE", installReport.notifyCke) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidatePublicVerifiedStepScope("channel binding", installReport.channelBinding) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "mission/key", installReport.installAttemptReceiptId, installReport.mission) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "repository", installReport.installAttemptReceiptId, installReport.repository) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "local XN", installReport.installAttemptReceiptId, installReport.localXn) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "remote XN", installReport.installAttemptReceiptId, installReport.remoteXn) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "notify CKE", installReport.installAttemptReceiptId, installReport.notifyCke) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceiptStep( + "channel binding", installReport.installAttemptReceiptId, installReport.channelBinding) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int ValidateInstallRequestScope(const TileXRCcuInstallRequest& request, TileXRCcuInstallProviderReport* report) +{ + const TileXRCcuLaunchPackage& package = *request.package; + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (package.installScope.packageFingerprint == 0 || + package.installScope.packageFingerprint != packageFingerprint || + package.installScope.provider.empty()) { + FillScopeFailureReport(report, "launch install scope is not bound"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (request.deviceId != package.installScope.deviceId) { + FillScopeFailureReport(report, "device scope mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (request.rank != package.installScope.rank) { + FillScopeFailureReport(report, "rank scope mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (request.provider.empty() || request.provider != package.installScope.provider) { + FillScopeFailureReport(report, "provider scope mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int ValidateInstallRequestManifest(const TileXRCcuInstallRequest& request, TileXRCcuInstallProviderReport* report) +{ + if (request.manifest == nullptr) { + return TILEXR_SUCCESS; + } + + const TileXRCcuLaunchPackage& package = *request.package; + const TileXRCcuInstallManifest& manifest = *request.manifest; + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (manifest.packageFingerprint == 0 || manifest.packageFingerprint != packageFingerprint) { + FillScopeFailureReport(report, "install manifest fingerprint mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (manifest.deviceId != request.deviceId) { + FillScopeFailureReport(report, "install manifest device mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (manifest.rank != request.rank) { + FillScopeFailureReport(report, "install manifest rank mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (manifest.provider.empty() || manifest.provider != request.provider) { + FillScopeFailureReport(report, "install manifest provider mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (manifest.requiredEvidenceKind != TileXRCcuEvidenceKind::PublicVerified || + manifest.requiredEvidenceSurface != TileXRCcuEvidenceSurface::PublicInstallProvider) { + FillScopeFailureReport(report, "install manifest evidence contract mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!manifest.installAttemptReceiptRequired) { + FillScopeFailureReport(report, "install manifest receipt contract mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + uint32_t expectedRequirementCount = 4U; + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode)) { + ++expectedRequirementCount; + } + if (RequiresChannelBindingEvidence(package.plan.barrierMode)) { + ++expectedRequirementCount; + } + if (manifest.requirements.size() != expectedRequirementCount) { + FillScopeFailureReport(report, "install manifest requirement count mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::MissionKey) || + !HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::RepositoryImage) || + !HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::LocalXn) || + !HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::NotifyCke)) { + FillScopeFailureReport(report, "install manifest requirement kind mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode) != + HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::RemoteXnBinding)) { + FillScopeFailureReport(report, "install manifest remote XN requirement kind mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresChannelBindingEvidence(package.plan.barrierMode) != + HasRequirementKind(manifest, TileXRCcuInstallRequirementKind::ChannelBinding)) { + FillScopeFailureReport(report, "install manifest channel binding requirement kind mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (manifest.requiresHardwareInstall != package.requiresHardwareInstall) { + FillScopeFailureReport(report, "install manifest hardware requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuInstallRequirement* mission = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::MissionKey); + if (ValidateRequirementMetadata(mission, packageFingerprint, report) != TILEXR_SUCCESS || + mission->dieId != package.plan.mission.dieId || + mission->missionId != package.plan.mission.missionId || + mission->missionKey != package.plan.mission.key || + mission->resourceStartId != package.plan.kernelLocalMission.startId || + mission->resourceCount != package.plan.kernelLocalMission.num) { + FillScopeFailureReport(report, "install manifest mission requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuInstallRequirement* repository = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::RepositoryImage); + if (ValidateRequirementMetadata(repository, packageFingerprint, report) != TILEXR_SUCCESS || + repository->dieId != package.repository.dieId || + repository->repositoryStartId != package.repository.repositoryStartId || + repository->repositoryCount != package.repository.repositoryCount || + repository->missionStartId != package.repository.missionStartId || + repository->missionCount != package.repository.missionCount || + repository->instructionCount != package.repository.instructions.size()) { + FillScopeFailureReport(report, "install manifest repository requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuInstallRequirement* localXn = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::LocalXn); + if (ValidateRequirementMetadata(localXn, packageFingerprint, report) != TILEXR_SUCCESS || + localXn->dieId != package.plan.kernelLocalXn.dieId || + localXn->resourceStartId != package.plan.kernelLocalXn.startId || + localXn->resourceCount != package.plan.kernelLocalXn.num) { + FillScopeFailureReport(report, "install manifest local XN requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode)) { + const TileXRCcuInstallRequirement* remoteXn = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::RemoteXnBinding); + if (ValidateRequirementMetadata(remoteXn, packageFingerprint, report) != TILEXR_SUCCESS || + remoteXn->dieId != package.plan.kernelLocalXn.dieId || + remoteXn->syncResourceCount != package.plan.syncResources.size()) { + FillScopeFailureReport(report, "install manifest remote XN requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + + const TileXRCcuInstallRequirement* notifyCke = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::NotifyCke); + if (ValidateRequirementMetadata(notifyCke, packageFingerprint, report) != TILEXR_SUCCESS || + notifyCke->dieId != package.plan.kernelLocalCke.dieId || + notifyCke->resourceStartId != package.plan.kernelLocalCke.startId || + notifyCke->resourceCount != package.plan.kernelLocalCke.num || + notifyCke->syncResourceCount != package.plan.syncResources.size()) { + FillScopeFailureReport(report, "install manifest notify CKE requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (RequiresChannelBindingEvidence(package.plan.barrierMode)) { + const TileXRCcuInstallRequirement* channel = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::ChannelBinding); + if (ValidateRequirementMetadata(channel, packageFingerprint, report) != TILEXR_SUCCESS || + channel->dieId != package.plan.kernelLocalXn.dieId || + channel->syncResourceCount != package.plan.syncResources.size() || + channel->bindingCount != TotalBindingCount(package.plan.syncResources)) { + FillScopeFailureReport(report, "install manifest channel requirement mismatch"); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; +} + +bool HasRepositoryInstallProviderInputs(const TileXRCcuInstallRequest& request) +{ + return !request.offlineOnly && + request.driverAdapter != nullptr && + request.repositoryMemoryOps.alloc != nullptr && + request.repositoryMemoryOps.copyHostToDevice != nullptr && + request.repositoryMemoryOps.free != nullptr && + request.repositoryReceipt != nullptr; +} + +struct TileXRCcuLowerLayerInstallResult { + uint32_t msidTokenCount = 0; + uint32_t pfeCount = 0; + uint32_t jettyCount = 0; + uint32_t channelCount = 0; + uint32_t xnClearCount = 0; + uint32_t ckeClearCount = 0; + bool localXnInstalled = false; + bool notifyCkeInstalled = false; + bool channelBindingInstalled = false; + std::string message; +}; + +std::string FormatLowerLayerPreconditionSummary(const TileXRCcuLowerLayerInstallResult& result) +{ + std::ostringstream summary; + summary << "lowerLayerPreconditions{" + << "msidTokenCount=" << result.msidTokenCount + << " pfeCount=" << result.pfeCount + << " jettyCount=" << result.jettyCount + << " channelCount=" << result.channelCount + << " xnClearCount=" << result.xnClearCount + << " ckeClearCount=" << result.ckeClearCount + << " localXnInstalled=" << (result.localXnInstalled ? 1 : 0) + << " notifyCkeInstalled=" << (result.notifyCkeInstalled ? 1 : 0) + << " channelBindingInstalled=" << (result.channelBindingInstalled ? 1 : 0) + << "}"; + return summary.str(); +} + +std::string FormatLowerLayerPlanSummary(const TileXRCcuLowerLayerInstallPlan& plan) +{ + std::ostringstream summary; + if (!plan.msidTokens.empty()) { + const auto& token = plan.msidTokens.front(); + summary << " msidToken0{dieId=" << static_cast(token.dieId) + << " msId=" << token.msId + << " tokenId=0x" << std::hex << token.tokenId + << " tokenValue=0x" << token.tokenValue << std::dec + << "}"; + } + if (!plan.pfes.empty()) { + const auto& pfe = plan.pfes.front(); + summary << " pfe0{dieId=" << static_cast(pfe.dieId) + << " offset=" << pfe.pfeOffset + << "}"; + } + if (!plan.jettys.empty()) { + const auto& jetty = plan.jettys.front(); + summary << " jetty0{dieId=" << static_cast(jetty.dieId) + << " startJettyCtxId=" << jetty.startJettyCtxId + << " ctxCount=" << jetty.ctxs.size() + << "}"; + } + if (!plan.channels.empty()) { + const auto& channel = plan.channels.front(); + summary << " channel0{dieId=" << static_cast(channel.dieId) + << " channelId=" << channel.channelId + << "}"; + } + if (!plan.xnClears.empty()) { + const auto& xn = plan.xnClears.front(); + summary << " xnClear0{dieId=" << static_cast(xn.dieId) + << " startXnId=" << xn.startXnId + << " count=" << xn.count + << "}"; + } + if (!plan.ckeClears.empty()) { + const auto& cke = plan.ckeClears.front(); + summary << " ckeClear0{dieId=" << static_cast(cke.dieId) + << " startCkeId=" << cke.startCkeId + << " count=" << cke.count + << "}"; + } + if (!plan.remoteXnBindings.empty()) { + const auto& remote = plan.remoteXnBindings.front(); + summary << " remoteXn0{dieId=" << static_cast(remote.dieId) + << " channelId=" << remote.channelId + << " localXn=" << remote.localXn + << " remoteXn=" << remote.remoteXn + << " notifyCke=" << remote.notifyCke + << " localWaitCke=" << remote.localWaitCke + << " peerExchangeObserved=" << (remote.peerExchangeObserved ? 1 : 0) + << " endpointRouteVerified=" << (remote.endpointRouteVerified ? 1 : 0) + << " channelResourceOwnerVerified=" << (remote.channelResourceOwnerVerified ? 1 : 0) + << " transportResourceExchangeVerified=" << (remote.transportResourceExchangeVerified ? 1 : 0) + << "}"; + } + return summary.str(); +} + +int InstallLowerLayerResources( + const TileXRCcuDriverAdapter& adapter, + const TileXRCcuLowerLayerInstallPlan& plan, + TileXRCcuLowerLayerInstallResult* result) +{ + if (result == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *result = TileXRCcuLowerLayerInstallResult{}; + + TileXRCcuDriverAdapterReport driverReport; + for (const auto& token : plan.msidTokens) { + const int ret = adapter.InstallMsidToken( + token.dieId, + token.msId, + token.tokenId, + token.tokenValue, + &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to install CCU MSID token via SET_MSID_TOKEN: " + driverReport.message; + return ret; + } + ++result->msidTokenCount; + } + + for (const auto& pfe : plan.pfes) { + const int ret = adapter.InstallPfeCtx(pfe.dieId, pfe.pfeOffset, pfe.ctx, &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to install CCU PFE context via SET_PFE: " + driverReport.message; + return ret; + } + ++result->pfeCount; + } + + for (const auto& jetty : plan.jettys) { + const TileXRCcuLocalJettyCtxData* ctxs = jetty.ctxs.empty() ? nullptr : jetty.ctxs.data(); + const int ret = adapter.InstallJettyCtx( + jetty.dieId, + jetty.startJettyCtxId, + ctxs, + static_cast(jetty.ctxs.size()), + &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to install CCU local jetty contexts via SET_JETTY_CTX: " + + driverReport.message; + return ret; + } + ++result->jettyCount; + } + + for (const auto& channel : plan.channels) { + const int ret = adapter.InstallChannelCtxV1(channel.dieId, channel.channelId, channel.ctx, &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to install CCU channel context via SET_CHANNEL: " + driverReport.message; + return ret; + } + ++result->channelCount; + } + + for (const auto& xn : plan.xnClears) { + const int ret = adapter.InstallXnRange(xn.dieId, xn.startXnId, xn.count, &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to install CCU local XN range via SET_XN: " + driverReport.message; + return ret; + } + ++result->xnClearCount; + } + + for (const auto& cke : plan.ckeClears) { + const int ret = adapter.ClearCkeRange(cke.dieId, cke.startCkeId, cke.count, &driverReport); + if (ret != TILEXR_SUCCESS) { + result->message = "failed to clear CCU CKE range via SET_CKE: " + driverReport.message; + return ret; + } + ++result->ckeClearCount; + } + + result->localXnInstalled = result->xnClearCount > 0; + result->notifyCkeInstalled = result->ckeClearCount > 0; + const uint32_t expectedChannelCount = plan.remoteXnBindings.empty() ? + static_cast(plan.channels.size()) : + CountUniqueRemoteBindingChannels(plan.remoteXnBindings); + const uint32_t installedChannelCount = plan.remoteXnBindings.empty() ? + static_cast(result->channelCount) : + CountInstalledRemoteBindingChannels(plan.remoteXnBindings, plan.channels); + result->channelBindingInstalled = + result->pfeCount > 0 && result->jettyCount > 0 && expectedChannelCount > 0 && + installedChannelCount >= expectedChannelCount; + result->message = + "lower-layer CCU resources installed via SET_MSID_TOKEN, SET_PFE, SET_JETTY_CTX, SET_CHANNEL, SET_XN, SET_CKE"; + return TILEXR_SUCCESS; +} + +int InstallRepositoryImageForRequest( + const TileXRCcuInstallRequest& request, + TileXRCcuRepositoryReport* repositoryReport) +{ + return TileXRCcuInstallRepositoryImageWithOptions( + request.package->repository, + request.repositoryInstallOptions, + request.repositoryMemoryOps, + request.repositoryMemoryUserData, + *request.driverAdapter, + request.repositoryReceipt, + repositoryReport); +} + +int InstallRepositoryOnly( + const TileXRCcuInstallRequest& request, + TileXRCcuHardwareInstallEvidence* evidence, + TileXRCcuInstallProviderReport* report) +{ + const TileXRCcuLaunchPackage& package = *request.package; + TileXRCcuRepositoryReport repositoryReport; + TileXRCcuLowerLayerInstallResult lowerLayerResult; + + int lowerLayerRet = TILEXR_SUCCESS; + if (request.installOrder == TileXRCcuInstallOrder::InstallLowerLayerFirst && + request.lowerLayerPlan != nullptr) { + lowerLayerRet = InstallLowerLayerResources(*request.driverAdapter, *request.lowerLayerPlan, &lowerLayerResult); + if (lowerLayerRet != TILEXR_SUCCESS) { + TileXRCcuInstallProviderReport nextReport; + nextReport.offlineOnly = false; + nextReport.installAttempted = true; + nextReport.installSucceeded = false; + nextReport.message = lowerLayerResult.message; + MarkMissingStep(&nextReport.mission, "mission/key install provider is not implemented"); + MarkMissingStep(&nextReport.repository, "repository install skipped after lower-layer install failure"); + MarkMissingStep(&nextReport.localXn, lowerLayerResult.message); + MarkMissingStep(&nextReport.remoteXn, "remote XN binding provider is not implemented"); + MarkMissingStep(&nextReport.notifyCke, lowerLayerResult.message); + MarkMissingStep(&nextReport.channelBinding, lowerLayerResult.message); + FillManifestInstallSurfaceCounts(request.manifest, &nextReport); + if (report != nullptr) { + *report = nextReport; + } + if (TileXRCcuBuildInstallEvidence(nextReport, evidence) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + return lowerLayerRet; + } + } + + const int installRet = InstallRepositoryImageForRequest(request, &repositoryReport); + + TileXRCcuInstallProviderReport nextReport; + nextReport.offlineOnly = false; + nextReport.installAttempted = true; + + if (installRet != TILEXR_SUCCESS) { + nextReport.installSucceeded = false; + const bool lowerLayerFirstAttempted = + request.installOrder == TileXRCcuInstallOrder::InstallLowerLayerFirst && + request.lowerLayerPlan != nullptr; + const std::string lowerLayerSummary = lowerLayerFirstAttempted ? + FormatLowerLayerPreconditionSummary(lowerLayerResult) + + FormatLowerLayerPlanSummary(*request.lowerLayerPlan) : + std::string(); + const std::string repositoryFailureMessage = lowerLayerFirstAttempted ? + lowerLayerSummary + "; " + repositoryReport.message : + repositoryReport.message; + nextReport.message = repositoryFailureMessage; + MarkMissingStep(&nextReport.mission, "mission/key install provider is not implemented"); + MarkMissingStep(&nextReport.repository, repositoryReport.message); + MarkMissingStep( + &nextReport.localXn, + lowerLayerFirstAttempted ? lowerLayerSummary : "local XN install provider is not implemented"); + MarkMissingStep( + &nextReport.remoteXn, + lowerLayerFirstAttempted ? lowerLayerSummary : "remote XN binding provider is not implemented"); + MarkMissingStep( + &nextReport.notifyCke, + lowerLayerFirstAttempted ? lowerLayerSummary : "notify CKE install provider is not implemented"); + MarkMissingStep( + &nextReport.channelBinding, + lowerLayerFirstAttempted ? lowerLayerSummary : "channel binding provider is not implemented"); + FillManifestInstallSurfaceCounts(request.manifest, &nextReport); + if (report != nullptr) { + *report = nextReport; + } + if (TileXRCcuBuildInstallEvidence(nextReport, evidence) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + return installRet; + } + + nextReport.installSucceeded = true; + nextReport.installAttemptReceiptId = BuildRepositoryInstallReceiptId(package, *request.repositoryReceipt); + + if (request.lowerLayerPlan != nullptr && + request.installOrder != TileXRCcuInstallOrder::InstallLowerLayerFirst) { + lowerLayerRet = InstallLowerLayerResources(*request.driverAdapter, *request.lowerLayerPlan, &lowerLayerResult); + if (lowerLayerRet != TILEXR_SUCCESS) { + nextReport.installSucceeded = false; + nextReport.message = lowerLayerResult.message; + MarkMissingStep(&nextReport.mission, "mission/key install provider is not implemented"); + MarkMissingStep( + &nextReport.repository, + "repository install evidence withheld after lower-layer install failure"); + MarkMissingStep(&nextReport.localXn, lowerLayerResult.message); + MarkMissingStep(&nextReport.remoteXn, "remote XN binding provider is not implemented"); + MarkMissingStep(&nextReport.notifyCke, lowerLayerResult.message); + MarkMissingStep(&nextReport.channelBinding, lowerLayerResult.message); + FillManifestInstallSurfaceCounts(request.manifest, &nextReport); + if (report != nullptr) { + *report = nextReport; + } + if (TileXRCcuBuildInstallEvidence(nextReport, evidence) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + return lowerLayerRet; + } + } + + std::string missionProofMessage; + if (ValidateMissionLaunchDescriptorProof(package, request.manifest, &missionProofMessage)) { + nextReport.mission = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "ValidateMissionLaunchDescriptorProof", + missionProofMessage); + } else { + MarkMissingStep(&nextReport.mission, missionProofMessage); + } + nextReport.repository = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "TileXRCcuInstallRepositoryImage", + "repository instruction image installed via SET_INSTRUCTION"); + if (lowerLayerResult.localXnInstalled) { + nextReport.localXn = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "InstallLowerLayerResources", + "kernel-local XN resources initialized via SET_XN"); + } else { + MarkMissingStep(&nextReport.localXn, "local XN install provider is not implemented"); + } + std::string remoteXnProofMessage; + if (!RequiresRemoteXnBindingEvidence(package.plan.barrierMode)) { + MarkMissingStep( + &nextReport.remoteXn, + "remote XN binding is not required for this CCU barrier mode"); + } else if (ValidateRemoteXnExchangeBindingProof( + package, + request.manifest, + request.lowerLayerPlan, + &remoteXnProofMessage)) { + nextReport.remoteXn = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "ValidateRemoteXnExchangeBindingProof", + remoteXnProofMessage, + true); + } else { + MarkMissingStep(&nextReport.remoteXn, remoteXnProofMessage); + } + if (lowerLayerResult.notifyCkeInstalled) { + nextReport.notifyCke = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "InstallLowerLayerResources", + "notify CKE resources cleared via SET_CKE"); + } else { + MarkMissingStep(&nextReport.notifyCke, "notify CKE install provider is not implemented"); + } + if (!RequiresChannelBindingEvidence(package.plan.barrierMode)) { + MarkMissingStep( + &nextReport.channelBinding, + "channel binding is not required for this CCU barrier mode"); + } else if (lowerLayerResult.channelBindingInstalled && HasVerifiedChannelResourceBindings(request.lowerLayerPlan)) { + nextReport.channelBinding = PublicVerifiedStep( + package, + request, + nextReport.installAttemptReceiptId, + "InstallLowerLayerResources", + "channel binding contexts installed via SET_PFE, SET_JETTY_CTX, SET_CHANNEL with verified endpoint routes, " + "channel resource owner allocation, and transport resource exchange", + true); + } else if (lowerLayerResult.channelBindingInstalled && HasVerifiedEndpointRoutes(request.lowerLayerPlan)) { + MarkMissingStep( + &nextReport.channelBinding, + "channel binding channel resource owner or transport resource exchange provenance was not verified"); + } else if (lowerLayerResult.channelBindingInstalled) { + MarkMissingStep(&nextReport.channelBinding, "channel binding endpoint route provenance was not verified"); + } else { + MarkMissingStep(&nextReport.channelBinding, "channel binding provider is not implemented"); + } + if (request.lowerLayerPlan == nullptr) { + nextReport.message = + "repository instruction image installed via SET_INSTRUCTION; " + + (nextReport.mission.satisfied ? missionProofMessage : nextReport.mission.message) + + "; lower-layer CCU resources are missing"; + } else { + nextReport.message = + "repository instruction image installed via SET_INSTRUCTION; " + lowerLayerResult.message + + "; " + (nextReport.mission.satisfied ? remoteXnProofMessage : + nextReport.mission.message + "; " + remoteXnProofMessage); + } + FillManifestInstallSurfaceCounts(request.manifest, &nextReport); + + if (report != nullptr) { + *report = nextReport; + } + if (TileXRCcuBuildInstallEvidence(nextReport, evidence) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + if (nextReport.missingInstallSurfaceCount == 0) { + return TILEXR_SUCCESS; + } + return TILEXR_ERROR_NOT_FOUND; +} + +} // namespace + +int TileXRCcuBuildInstallEvidence( + const TileXRCcuInstallProviderReport& installReport, + TileXRCcuHardwareInstallEvidence* evidence) +{ + if (evidence == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *evidence = TileXRCcuHardwareInstallEvidence{}; + + if (RejectOfflinePublicVerified(installReport)) { + const std::string reason = "offline install evidence cannot be public verified"; + (void)reason; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallReceipt(installReport) != TILEXR_SUCCESS) { + const std::string reason = "install attempt receipt mismatch"; + (void)reason; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuHardwareInstallEvidence result; + result.missionInstalled = installReport.mission.satisfied; + result.repositoryInstalled = installReport.repository.satisfied; + result.localXnInstalled = installReport.localXn.satisfied; + result.remoteXnBound = installReport.remoteXn.satisfied; + result.notifyCkeInstalled = installReport.notifyCke.satisfied; + result.channelBindingsInstalled = installReport.channelBinding.satisfied; + result.missionSource = SourceOrMissing(installReport.mission, "mission/key install evidence is missing"); + result.repositorySource = SourceOrMissing(installReport.repository, "repository install evidence is missing"); + result.localXnSource = SourceOrMissing(installReport.localXn, "local XN install evidence is missing"); + result.remoteXnSource = SourceOrMissing(installReport.remoteXn, "remote XN binding evidence is missing"); + result.notifyCkeSource = SourceOrMissing(installReport.notifyCke, "notify CKE install evidence is missing"); + result.channelBindingSource = + SourceOrMissing(installReport.channelBinding, "channel binding install evidence is missing"); + + *evidence = result; + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildInstallManifest( + const TileXRCcuLaunchPackage& package, + TileXRCcuInstallManifest* manifest, + TileXRCcuInstallManifestReport* report) +{ + ResetManifestReport(report); + if (manifest == nullptr) { + return FailManifest(nullptr, report, "missing output CCU install manifest"); + } + *manifest = TileXRCcuInstallManifest{}; + + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (ValidateInstallManifestScope(package, packageFingerprint, manifest, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuInstallManifest result; + result.packageFingerprint = packageFingerprint; + result.deviceId = package.installScope.deviceId; + result.rank = package.installScope.rank; + result.provider = package.installScope.provider; + result.requiresHardwareInstall = package.requiresHardwareInstall; + result.installAttemptReceiptRequired = true; + result.requiredEvidenceKind = TileXRCcuEvidenceKind::PublicVerified; + result.requiredEvidenceSurface = TileXRCcuEvidenceSurface::PublicInstallProvider; + + TileXRCcuInstallRequirement mission = BaseRequirement( + TileXRCcuInstallRequirementKind::MissionKey, + packageFingerprint, + package.plan.mission.dieId, + "mission/key", + "install CCU mission id and key"); + mission.missionId = package.plan.mission.missionId; + mission.missionKey = package.plan.mission.key; + mission.resourceStartId = package.plan.kernelLocalMission.startId; + mission.resourceCount = package.plan.kernelLocalMission.num; + result.requirements.push_back(mission); + + TileXRCcuInstallRequirement repository = BaseRequirement( + TileXRCcuInstallRequirementKind::RepositoryImage, + packageFingerprint, + package.repository.dieId, + "repository", + "install generated CCU repository image"); + repository.repositoryStartId = package.repository.repositoryStartId; + repository.repositoryCount = package.repository.repositoryCount; + repository.missionStartId = package.repository.missionStartId; + repository.missionCount = package.repository.missionCount; + repository.instructionCount = static_cast(package.repository.instructions.size()); + result.requirements.push_back(repository); + + TileXRCcuInstallRequirement localXn = BaseRequirement( + TileXRCcuInstallRequirementKind::LocalXn, + packageFingerprint, + package.plan.kernelLocalXn.dieId, + "local XN", + "install kernel-local CCU XN resource window"); + localXn.resourceStartId = package.plan.kernelLocalXn.startId; + localXn.resourceCount = package.plan.kernelLocalXn.num; + result.requirements.push_back(localXn); + + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode)) { + TileXRCcuInstallRequirement remoteXn = BaseRequirement( + TileXRCcuInstallRequirementKind::RemoteXnBinding, + packageFingerprint, + package.plan.kernelLocalXn.dieId, + "remote XN binding", + "bind remote CCU XN resources referenced by sync instructions"); + remoteXn.syncResourceCount = static_cast(package.plan.syncResources.size()); + result.requirements.push_back(remoteXn); + } + + TileXRCcuInstallRequirement notifyCke = BaseRequirement( + TileXRCcuInstallRequirementKind::NotifyCke, + packageFingerprint, + package.plan.kernelLocalCke.dieId, + "notify CKE", + "install notify CKE resources referenced by sync instructions"); + notifyCke.resourceStartId = package.plan.kernelLocalCke.startId; + notifyCke.resourceCount = package.plan.kernelLocalCke.num; + notifyCke.syncResourceCount = static_cast(package.plan.syncResources.size()); + result.requirements.push_back(notifyCke); + + if (RequiresChannelBindingEvidence(package.plan.barrierMode)) { + TileXRCcuInstallRequirement channel = BaseRequirement( + TileXRCcuInstallRequirementKind::ChannelBinding, + packageFingerprint, + package.plan.kernelLocalXn.dieId, + "channel binding", + "bind CCU channel routes for sync resources"); + channel.syncResourceCount = static_cast(package.plan.syncResources.size()); + channel.bindingCount = TotalBindingCount(package.plan.syncResources); + result.requirements.push_back(channel); + } + + if (report != nullptr) { + report->requirementCount = static_cast(result.requirements.size()); + report->message = "ok"; + } + *manifest = result; + return TILEXR_SUCCESS; +} + +int TileXRCcuInstallHardware( + const TileXRCcuInstallRequest& request, + TileXRCcuHardwareInstallEvidence* evidence, + TileXRCcuInstallProviderReport* report) +{ + ResetReport(report); + if (evidence == nullptr) { + FillUnsupportedReport(report); + if (report != nullptr) { + report->message = "missing output CCU hardware install evidence"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *evidence = TileXRCcuHardwareInstallEvidence{}; + + if (request.package == nullptr) { + FillUnsupportedReport(report); + if (report != nullptr) { + report->message = "missing CCU launch package for hardware install"; + } + TileXRCcuInstallProviderReport missingReport; + FillUnsupportedReport(&missingReport); + return TileXRCcuBuildInstallEvidence(missingReport, evidence) == TILEXR_SUCCESS ? + TILEXR_ERROR_PARA_CHECK_FAIL : + TILEXR_ERROR_INTERNAL; + } + + if (ValidateInstallRequestScope(request, report) != TILEXR_SUCCESS) { + TileXRCcuInstallProviderReport missingReport; + FillScopeFailureReport(&missingReport, report == nullptr ? "install request scope mismatch" : report->message); + return TileXRCcuBuildInstallEvidence(missingReport, evidence) == TILEXR_SUCCESS ? + TILEXR_ERROR_PARA_CHECK_FAIL : + TILEXR_ERROR_INTERNAL; + } + if (ValidateInstallRequestManifest(request, report) != TILEXR_SUCCESS) { + TileXRCcuInstallProviderReport missingReport; + FillScopeFailureReport( + &missingReport, + report == nullptr ? "install request manifest mismatch" : report->message); + return TileXRCcuBuildInstallEvidence(missingReport, evidence) == TILEXR_SUCCESS ? + TILEXR_ERROR_PARA_CHECK_FAIL : + TILEXR_ERROR_INTERNAL; + } + + if (HasRepositoryInstallProviderInputs(request)) { + return InstallRepositoryOnly(request, evidence, report); + } + + FillUnsupportedReport(report); + FillManifestInstallSurfaceCounts(request.manifest, report); + TileXRCcuInstallProviderReport missingReport; + FillUnsupportedReport(&missingReport); + if (TileXRCcuBuildInstallEvidence(missingReport, evidence) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + return TILEXR_ERROR_NOT_FOUND; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_install_provider.h b/src/comm/ccu/tilexr_ccu_install_provider.h new file mode 100644 index 00000000..5d12b51f --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_install_provider.h @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_INSTALL_PROVIDER_H +#define TILEXR_CCU_INSTALL_PROVIDER_H + +#include "ccu/tilexr_ccu_provider.h" + +#include +#include +#include + +namespace TileXR { + +enum class TileXRCcuInstallRequirementKind : uint8_t { + MissionKey = 0, + RepositoryImage = 1, + LocalXn = 2, + RemoteXnBinding = 3, + NotifyCke = 4, + ChannelBinding = 5, +}; + +struct TileXRCcuInstallRequirement { + TileXRCcuInstallRequirementKind kind = TileXRCcuInstallRequirementKind::MissionKey; + uint64_t packageFingerprint = 0; + uint8_t dieId = 0; + uint8_t missionId = 0; + uint32_t missionKey = 0; + uint16_t resourceStartId = 0; + uint16_t resourceCount = 0; + uint16_t repositoryStartId = 0; + uint16_t repositoryCount = 0; + uint16_t missionStartId = 0; + uint16_t missionCount = 0; + uint32_t instructionCount = 0; + uint32_t syncResourceCount = 0; + uint32_t bindingCount = 0; + std::string label; + std::string detail; +}; + +struct TileXRCcuInstallManifest { + uint64_t packageFingerprint = 0; + uint32_t deviceId = 0; + uint32_t rank = 0; + std::string provider; + bool requiresHardwareInstall = true; + bool installAttemptReceiptRequired = true; + TileXRCcuEvidenceKind requiredEvidenceKind = TileXRCcuEvidenceKind::PublicVerified; + TileXRCcuEvidenceSurface requiredEvidenceSurface = TileXRCcuEvidenceSurface::PublicInstallProvider; + std::vector requirements; +}; + +struct TileXRCcuInstallManifestReport { + uint32_t requirementCount = 0; + std::string message; +}; + +struct TileXRCcuMsidTokenInstall { + uint8_t dieId = 0; + uint32_t msId = 0; + uint32_t tokenId = 0; + uint32_t tokenValue = 0; +}; + +struct TileXRCcuPfeInstall { + uint8_t dieId = 0; + uint32_t pfeOffset = 0; + TileXRCcuPfeCtx ctx; +}; + +struct TileXRCcuJettyInstall { + uint8_t dieId = 0; + uint16_t startJettyCtxId = 0; + std::vector ctxs; +}; + +struct TileXRCcuChannelInstall { + uint8_t dieId = 0; + uint32_t channelId = 0; + TileXRCcuChannelCtxDataV1 ctx; +}; + +struct TileXRCcuCkeClearInstall { + uint8_t dieId = 0; + uint32_t startCkeId = 0; + uint32_t count = 0; +}; + +struct TileXRCcuXnClearInstall { + uint8_t dieId = 0; + uint32_t startXnId = 0; + uint32_t count = 0; +}; + +struct TileXRCcuRemoteXnBindingProof { + uint8_t dieId = 0; + uint16_t channelId = 0; + uint16_t localXn = 0; + uint16_t remoteXn = 0; + uint16_t notifyCke = 0; + uint32_t peerRank = 0; + bool peerExchangeObserved = false; + uint16_t localWaitCke = 0; + bool endpointRouteVerified = false; + bool channelResourceOwnerVerified = false; + bool transportResourceExchangeVerified = false; +}; + +struct TileXRCcuLowerLayerInstallPlan { + std::vector msidTokens; + std::vector pfes; + std::vector jettys; + std::vector channels; + std::vector xnClears; + std::vector ckeClears; + std::vector remoteXnBindings; +}; + +enum class TileXRCcuInstallOrder : uint8_t { + RepositoryFirst = 0, + InstallLowerLayerFirst = 1, +}; + +struct TileXRCcuInstallRequest { + const TileXRCcuLaunchPackage* package = nullptr; + const TileXRCcuInstallManifest* manifest = nullptr; + uint32_t deviceId = 0; + uint32_t rank = 0; + std::string provider; + bool offlineOnly = true; + const TileXRCcuDriverAdapter* driverAdapter = nullptr; + TileXRCcuDeviceMemoryOps repositoryMemoryOps; + void* repositoryMemoryUserData = nullptr; + TileXRCcuRepositoryInstallOptions repositoryInstallOptions; + TileXRCcuRepositoryInstallReceipt* repositoryReceipt = nullptr; + TileXRCcuInstallOrder installOrder = TileXRCcuInstallOrder::RepositoryFirst; + const TileXRCcuLowerLayerInstallPlan* lowerLayerPlan = nullptr; +}; + +struct TileXRCcuInstallStepEvidence { + bool satisfied = false; + TileXRCcuEvidenceSource source; + std::string message; +}; + +struct TileXRCcuInstallProviderReport { + TileXRCcuInstallStepEvidence mission; + TileXRCcuInstallStepEvidence repository; + TileXRCcuInstallStepEvidence localXn; + TileXRCcuInstallStepEvidence remoteXn; + TileXRCcuInstallStepEvidence notifyCke; + TileXRCcuInstallStepEvidence channelBinding; + bool offlineOnly = true; + bool installAttempted = false; + bool installSucceeded = false; + uint32_t requiredInstallSurfaceCount = 0; + uint32_t publicVerifiedInstallSurfaceCount = 0; + uint32_t missingInstallSurfaceCount = 0; + uint64_t installAttemptReceiptId = 0; + std::string message; +}; + +int TileXRCcuBuildInstallEvidence( + const TileXRCcuInstallProviderReport& installReport, + TileXRCcuHardwareInstallEvidence* evidence); + +int TileXRCcuBuildInstallManifest( + const TileXRCcuLaunchPackage& package, + TileXRCcuInstallManifest* manifest, + TileXRCcuInstallManifestReport* report); + +int TileXRCcuInstallHardware( + const TileXRCcuInstallRequest& request, + TileXRCcuHardwareInstallEvidence* evidence, + TileXRCcuInstallProviderReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_INSTALL_PROVIDER_H diff --git a/src/comm/ccu/tilexr_ccu_launch_package.cpp b/src/comm/ccu/tilexr_ccu_launch_package.cpp new file mode 100644 index 00000000..fa1a5ec0 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_launch_package.cpp @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_launch_package.h" + +namespace TileXR { +namespace { + +constexpr uint64_t TILEXR_CCU_FNV_OFFSET = 1469598103934665603ULL; +constexpr uint64_t TILEXR_CCU_FNV_PRIME = 1099511628211ULL; + +void MixByte(uint8_t value, uint64_t* hash) +{ + *hash ^= value; + *hash *= TILEXR_CCU_FNV_PRIME; +} + +void MixU16(uint16_t value, uint64_t* hash) +{ + MixByte(static_cast(value & 0xffU), hash); + MixByte(static_cast((value >> 8U) & 0xffU), hash); +} + +void MixU32(uint32_t value, uint64_t* hash) +{ + MixU16(static_cast(value & 0xffffU), hash); + MixU16(static_cast((value >> 16U) & 0xffffU), hash); +} + +void MixU64(uint64_t value, uint64_t* hash) +{ + MixU32(static_cast(value & 0xffffffffULL), hash); + MixU32(static_cast((value >> 32ULL) & 0xffffffffULL), hash); +} + +void MixRange(const TileXRCcuRange& range, uint64_t* hash) +{ + MixByte(range.dieId, hash); + MixU16(range.startId, hash); + MixU16(range.num, hash); +} + +void MixMission(const TileXRCcuMission& mission, uint64_t* hash) +{ + MixByte(mission.dieId, hash); + MixByte(mission.missionId, hash); + MixU32(mission.key, hash); + MixByte(mission.installed ? 1U : 0U, hash); +} + +void MixInstructionWindow(const TileXRCcuInstructionWindow& window, uint64_t* hash) +{ + MixByte(window.dieId, hash); + MixU16(window.repositoryStartId, hash); + MixU16(window.repositoryCount, hash); + MixU16(window.missionStartId, hash); + MixU16(window.missionCount, hash); +} + +void MixSyncResource(const TileXRCcuSyncResource& resource, uint64_t* hash) +{ + MixByte(resource.dieId, hash); + MixU16(resource.localXn, hash); + MixU16(resource.remoteXn, hash); + MixU16(resource.notifyCke, hash); + MixU16(resource.channelId, hash); + MixU16(resource.bindingCount, hash); + MixU16(resource.localWaitCke, hash); + MixU16(resource.localWaitMask, hash); + MixU16(resource.remoteNotifyMask, hash); + MixU16(resource.sourceCke, hash); + MixU16(resource.sourceCkeMask, hash); +} + +void MixTaskWindow(const TileXRCcuTaskWindow& window, uint64_t* hash) +{ + MixByte(window.dieId, hash); + MixU16(window.instStartId, hash); + MixU16(window.instCnt, hash); + MixU32(window.argSize, hash); + MixU64(static_cast(window.args.size()), hash); + for (uint64_t arg : window.args) { + MixU64(arg, hash); + } +} + +void MixInstr(const TileXRCcuInstr& instr, uint64_t* hash) +{ + for (uint64_t word : instr.words) { + MixU64(word, hash); + } +} + +void MixRepository(const TileXRCcuRepositoryImage& repository, uint64_t* hash) +{ + MixByte(repository.dieId, hash); + MixU16(repository.repositoryStartId, hash); + MixU16(repository.repositoryCount, hash); + MixU16(repository.missionStartId, hash); + MixU16(repository.missionCount, hash); + MixU16(repository.missionOffset, hash); + MixU16(repository.sqeLoadOffset, hash); + MixU16(repository.sqeLoadCount, hash); + MixU16(repository.syncOffset, hash); + MixU16(repository.syncCount, hash); + MixU64(static_cast(repository.instructions.size()), hash); + for (const auto& instr : repository.instructions) { + MixInstr(instr, hash); + } +} + +void MixTask(const TileXRCcuTask& task, uint64_t* hash) +{ + MixByte(task.dieId, hash); + MixByte(task.missionId, hash); + MixU16(task.timeout, hash); + MixU16(task.instStartId, hash); + MixU16(task.instCnt, hash); + MixU32(task.key, hash); + MixU32(task.argSize, hash); + for (uint64_t arg : task.args) { + MixU64(arg, hash); + } +} + +void ResetReport(TileXRCcuLaunchPackageReport* report) +{ + if (report == nullptr) { + return; + } + report->taskCount = 0; + report->repositoryCount = 0; + report->installedInstructionCount = 0; + report->message.clear(); +} + +void ClearPackage(TileXRCcuLaunchPackage* package) +{ + if (package == nullptr) { + return; + } + package->plan = TileXRCcuProducerPlan{}; + package->program = TileXRCcuProgram{}; + package->repository = TileXRCcuRepositoryImage{}; + package->tasks.clear(); + package->installScope = TileXRCcuLaunchInstallScope{}; + package->requiresHardwareInstall = true; +} + +int Fail(TileXRCcuLaunchPackageReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +void FillReport(const TileXRCcuLaunchPackage& package, TileXRCcuLaunchPackageReport* report) +{ + if (report == nullptr) { + return; + } + report->taskCount = static_cast(package.tasks.size()); + report->repositoryCount = package.repository.repositoryCount; + report->installedInstructionCount = + static_cast(package.repository.sqeLoadCount + package.repository.syncCount); + report->message = "ok"; +} + +} // namespace + +int TileXRCcuBuildLaunchPackage( + const TileXRCcuProducerPlan& plan, + TileXRCcuLaunchPackage* package, + TileXRCcuLaunchPackageReport* report) +{ + ResetReport(report); + if (package == nullptr) { + return Fail(report, "missing output CCU launch package"); + } + ClearPackage(package); + + TileXRCcuProducerPlanReport planReport; + TileXRCcuProgram program; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + ClearPackage(package); + return Fail(report, planReport.message); + } + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + ClearPackage(package); + return Fail(report, repositoryReport.message); + } + + std::vector tasks; + if (TileXRCcuBuildTasks(plan, &tasks, &planReport) != TILEXR_SUCCESS) { + ClearPackage(package); + return Fail(report, planReport.message); + } + + package->plan = plan; + package->program = program; + package->repository = repository; + package->tasks = tasks; + package->installScope = TileXRCcuLaunchInstallScope{}; + package->requiresHardwareInstall = true; + FillReport(*package, report); + return TILEXR_SUCCESS; +} + +uint64_t TileXRCcuComputeLaunchPackageFingerprint(const TileXRCcuLaunchPackage& package) +{ + uint64_t hash = TILEXR_CCU_FNV_OFFSET; + MixU64(0x54494c4558524343ULL, &hash); // "TILEXRCC" + MixMission(package.plan.mission, &hash); + MixRange(package.plan.kernelLocalXn, &hash); + MixRange(package.plan.kernelLocalGsa, &hash); + MixRange(package.plan.kernelLocalCke, &hash); + MixRange(package.plan.kernelLocalMission, &hash); + MixU32(static_cast(package.plan.barrierMode), &hash); + MixInstructionWindow(package.plan.instructionWindow, &hash); + MixU64(static_cast(package.plan.syncResources.size()), &hash); + for (const auto& resource : package.plan.syncResources) { + MixSyncResource(resource, &hash); + } + MixU64(static_cast(package.plan.taskWindows.size()), &hash); + for (const auto& window : package.plan.taskWindows) { + MixTaskWindow(window, &hash); + } + MixU64(static_cast(package.program.sqeLoad.size()), &hash); + for (const auto& instr : package.program.sqeLoad) { + MixInstr(instr, &hash); + } + MixU64(static_cast(package.program.sync.size()), &hash); + for (const auto& instr : package.program.sync) { + MixInstr(instr, &hash); + } + MixRepository(package.repository, &hash); + MixU64(static_cast(package.tasks.size()), &hash); + for (const auto& task : package.tasks) { + MixTask(task, &hash); + } + MixByte(package.requiresHardwareInstall ? 1U : 0U, &hash); + return hash == 0 ? 1 : hash; +} + +int TileXRCcuBindLaunchPackageInstallScope( + TileXRCcuLaunchPackage* package, + uint32_t deviceId, + uint32_t rank, + const std::string& provider) +{ + if (package == nullptr || provider.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (package->tasks.empty() || package->repository.instructions.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + package->installScope.deviceId = deviceId; + package->installScope.rank = rank; + package->installScope.provider = provider; + package->installScope.packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(*package); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_launch_package.h b/src/comm/ccu/tilexr_ccu_launch_package.h new file mode 100644 index 00000000..891b4d4e --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_launch_package.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_LAUNCH_PACKAGE_H +#define TILEXR_CCU_LAUNCH_PACKAGE_H + +#include "ccu/tilexr_ccu_repository.h" + +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuLaunchInstallScope { + uint32_t deviceId = 0; + uint32_t rank = 0; + uint64_t packageFingerprint = 0; + std::string provider; +}; + +struct TileXRCcuLaunchPackage { + TileXRCcuProducerPlan plan; + TileXRCcuProgram program; + TileXRCcuRepositoryImage repository; + std::vector tasks; + TileXRCcuLaunchInstallScope installScope; + bool requiresHardwareInstall = true; +}; + +struct TileXRCcuLaunchPackageReport { + uint32_t taskCount = 0; + uint32_t repositoryCount = 0; + uint32_t installedInstructionCount = 0; + std::string message; +}; + +int TileXRCcuBuildLaunchPackage( + const TileXRCcuProducerPlan& plan, + TileXRCcuLaunchPackage* package, + TileXRCcuLaunchPackageReport* report); + +uint64_t TileXRCcuComputeLaunchPackageFingerprint(const TileXRCcuLaunchPackage& package); + +int TileXRCcuBindLaunchPackageInstallScope( + TileXRCcuLaunchPackage* package, + uint32_t deviceId, + uint32_t rank, + const std::string& provider); + +} // namespace TileXR + +#endif // TILEXR_CCU_LAUNCH_PACKAGE_H diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_payloads.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_payloads.cpp new file mode 100644 index 00000000..8256d6b4 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_lower_layer_payloads.cpp @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_lower_layer_payloads.h" + +#include +#include + +namespace TileXR { +namespace { + +constexpr uint32_t TOKEN_VALUE_VALID = 1; +constexpr uint32_t DOORBELL_ADDR_TYPE_VA = 1; +constexpr uint32_t DOORBELL_TOKEN_VALUE_VALID = 1; +constexpr uint32_t CCU_WQE_NUM_PER_SQE = 4; + +void ResetReport(TileXRCcuLowerLayerPayloadReport* report) +{ + if (report != nullptr) { + report->message.clear(); + } +} + +int Fail(TileXRCcuLowerLayerPayloadReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +void Store16(uint8_t* raw, uint32_t offset, uint16_t value) +{ + raw[offset] = static_cast(value & 0xffU); + raw[offset + 1U] = static_cast((value >> 8U) & 0xffU); +} + +uint16_t Log2PowerOfTwo(uint32_t value) +{ + uint16_t log2 = 0; + while (value > 1U) { + value >>= 1U; + ++log2; + } + return log2; +} + +bool IsPowerOfTwo(uint32_t value) +{ + return value != 0 && (value & (value - 1U)) == 0; +} + +bool IsEidEmpty(const std::array& eid) +{ + return std::all_of(eid.begin(), eid.end(), [](uint8_t byte) { return byte == 0; }); +} + +} // namespace + +int TileXRCcuBuildPfeCtx( + const TileXRCcuPfeCtxSpec& spec, + TileXRCcuPfeCtx* ctx, + TileXRCcuLowerLayerPayloadReport* report) +{ + ResetReport(report); + if (ctx == nullptr) { + return Fail(report, "missing output CCU PFE context"); + } + std::memset(ctx->raw, 0, sizeof(ctx->raw)); + if (spec.startJettyId == 0 || spec.jettyCount == 0 || spec.jettyCount > 128U || + spec.startLocalJettyCtxId >= 128U) { + return Fail(report, "invalid CCU PFE context spec"); + } + + Store16(ctx->raw, 0, spec.startJettyId); + const uint16_t word = static_cast( + ((spec.jettyCount - 1U) & 0x7fU) | + ((static_cast(spec.startLocalJettyCtxId) & 0x7fU) << 7U)); + Store16(ctx->raw, 2, word); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildLocalJettyCtx( + const TileXRCcuLocalJettyCtxSpec& spec, + TileXRCcuLocalJettyCtxData* ctx, + TileXRCcuLowerLayerPayloadReport* report) +{ + ResetReport(report); + if (ctx == nullptr) { + return Fail(report, "missing output CCU local jetty context"); + } + std::memset(ctx->raw, 0, sizeof(ctx->raw)); + const uint32_t wqeBasicBlocks = spec.sqDepth * CCU_WQE_NUM_PER_SQE; + if (spec.pfeId > 0xfU || spec.dieId > 1U || spec.doorbellVa == 0 || + spec.sqDepth == 0 || !IsPowerOfTwo(wqeBasicBlocks)) { + return Fail(report, "invalid CCU local jetty context spec"); + } + + Store16(ctx->raw, 0, static_cast(spec.doorbellVa & 0xffffU)); + Store16(ctx->raw, 2, static_cast((spec.doorbellVa >> 16U) & 0xffffU)); + Store16(ctx->raw, 4, static_cast((spec.doorbellVa >> 32U) & 0xffffU)); + Store16(ctx->raw, 6, static_cast((spec.doorbellVa >> 48U) & 0xffffU)); + + Store16(ctx->raw, 8, static_cast( + (spec.pfeId & 0xfU) | + ((static_cast(spec.dieId) & 0x1U) << 4U) | + (DOORBELL_ADDR_TYPE_VA << 5U) | + (DOORBELL_TOKEN_VALUE_VALID << 6U) | + ((spec.doorbellTokenId & 0xffU) << 8U))); + Store16(ctx->raw, 10, static_cast( + ((spec.doorbellTokenId >> 8U) & 0xfffU) | + ((spec.doorbellTokenValue & 0xfU) << 12U))); + Store16(ctx->raw, 12, static_cast((spec.doorbellTokenValue >> 4U) & 0xffffU)); + Store16(ctx->raw, 14, static_cast( + ((spec.doorbellTokenValue >> 20U) & 0xfffU) | + ((static_cast(Log2PowerOfTwo(wqeBasicBlocks)) & 0xfU) << 12U))); + Store16(ctx->raw, 22, static_cast( + (static_cast(spec.wqeBasicBlockStartId) & 0xfU) << 12U)); + Store16(ctx->raw, 24, static_cast((spec.wqeBasicBlockStartId >> 4U) & 0xffU)); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildChannelCtxV1( + const TileXRCcuChannelCtxV1Spec& spec, + TileXRCcuChannelCtxDataV1* ctx, + TileXRCcuLowerLayerPayloadReport* report) +{ + ResetReport(report); + if (ctx == nullptr) { + return Fail(report, "missing output CCU channel context v1"); + } + std::memset(ctx->raw, 0, sizeof(ctx->raw)); + if (IsEidEmpty(spec.remoteEid) || spec.sourcePfeId > 0xfU || spec.startJettyId == 0 || + spec.jettyCount == 0 || spec.jettyCount > 128U || spec.dieId > 1U || + spec.remoteCcuVa == 0) { + return Fail(report, "invalid CCU channel context v1 spec"); + } + + std::copy(spec.remoteEid.begin(), spec.remoteEid.end(), ctx->raw); + Store16(ctx->raw, 16, static_cast(spec.tpn & 0xffffU)); + Store16(ctx->raw, 18, static_cast( + ((spec.tpn >> 16U) & 0xffU) | + ((spec.sourcePfeId & 0xfU) << 8U) | + ((static_cast(spec.startJettyId) & 0xfU) << 12U))); + const uint32_t jettyNumMinusOne = spec.jettyCount - 1U; + Store16(ctx->raw, 20, static_cast( + ((static_cast(spec.startJettyId) >> 4U) & 0xfffU) | + ((jettyNumMinusOne & 0xfU) << 12U))); + Store16(ctx->raw, 22, static_cast( + ((jettyNumMinusOne >> 4U) & 0x7U) | + ((static_cast(spec.dieId) & 0x1U) << 3U) | + ((spec.memoryTokenId & 0xfffU) << 4U))); + Store16(ctx->raw, 24, static_cast( + ((spec.memoryTokenId >> 12U) & 0xffU) | + ((spec.memoryTokenValue & 0xffU) << 8U))); + Store16(ctx->raw, 26, static_cast((spec.memoryTokenValue >> 8U) & 0xffffU)); + + const uint64_t dstVa = spec.remoteCcuVa >> TILEXR_CCU_REMOTE_CCU_VA_SHIFT; + Store16(ctx->raw, 28, static_cast( + ((spec.memoryTokenValue >> 24U) & 0xffU) | + ((dstVa & 0xffU) << 8U))); + Store16(ctx->raw, 30, static_cast((dstVa >> 8U) & 0xffffU)); + Store16(ctx->raw, 32, static_cast((dstVa >> 24U) & 0xffffU)); + Store16(ctx->raw, 34, static_cast( + ((dstVa >> 40U) & 0x1U) | + (TOKEN_VALUE_VALID << 1U))); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_payloads.h b/src/comm/ccu/tilexr_ccu_lower_layer_payloads.h new file mode 100644 index 00000000..30fb33fa --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_lower_layer_payloads.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_LOWER_LAYER_PAYLOADS_H +#define TILEXR_CCU_LOWER_LAYER_PAYLOADS_H + +#include "ccu/tilexr_ccu_abi_constants.h" +#include "ccu/tilexr_ccu_driver_adapter.h" + +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuPfeCtxSpec { + uint16_t startJettyId = 0; + uint16_t jettyCount = 0; + uint16_t startLocalJettyCtxId = 0; +}; + +struct TileXRCcuLocalJettyCtxSpec { + uint8_t dieId = 0; + uint32_t pfeId = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + uint16_t wqeBasicBlockStartId = 0; +}; + +struct TileXRCcuChannelCtxV1Spec { + std::array remoteEid {}; + uint32_t tpn = 0; + uint32_t sourcePfeId = 0; + uint16_t startJettyId = 0; + uint16_t jettyCount = 0; + uint8_t dieId = 0; + uint32_t memoryTokenId = 0; + uint32_t memoryTokenValue = 0; + uint64_t remoteCcuVa = 0; +}; + +struct TileXRCcuLowerLayerPayloadReport { + std::string message; +}; + +int TileXRCcuBuildPfeCtx( + const TileXRCcuPfeCtxSpec& spec, + TileXRCcuPfeCtx* ctx, + TileXRCcuLowerLayerPayloadReport* report); + +int TileXRCcuBuildLocalJettyCtx( + const TileXRCcuLocalJettyCtxSpec& spec, + TileXRCcuLocalJettyCtxData* ctx, + TileXRCcuLowerLayerPayloadReport* report); + +int TileXRCcuBuildChannelCtxV1( + const TileXRCcuChannelCtxV1Spec& spec, + TileXRCcuChannelCtxDataV1* ctx, + TileXRCcuLowerLayerPayloadReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_LOWER_LAYER_PAYLOADS_H diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp new file mode 100644 index 00000000..095c38aa --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -0,0 +1,671 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + +#include +#include +#include +#include + +namespace TileXR { +namespace { + +constexpr uint16_t TILEXR_CCU_DEFAULT_START_JETTY_ID = 1024; +constexpr uint16_t TILEXR_CCU_DEFAULT_START_LOCAL_JETTY_CTX_ID = 0; +constexpr uint16_t TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE = 4; +constexpr uint16_t TILEXR_CCU_HCOMM_WQE_BASIC_BLOCKS_PER_ROUTE = 256; +constexpr uint32_t TILEXR_CCU_HCOMM_PER_DIE_PFE_RESERVED_NUM = 16; +constexpr uint16_t TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM = 23; +constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID = 92; +constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM = 36; +constexpr uint32_t TILEXR_CCU_HCOMM_MAX_INNER_FE_ID = 7; + +void ResetReport(TileXRCcuLowerLayerPlanBuilderReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuLowerLayerPlanBuilderReport{}; + } +} + +int Fail( + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + const std::string& message) +{ + if (plan != nullptr) { + *plan = TileXRCcuLowerLayerInstallPlan{}; + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int FailPayload( + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + const TileXRCcuLowerLayerPayloadReport& payloadReport) +{ + return Fail(plan, report, payloadReport.message.empty() ? "invalid lower-layer CCU payload spec" : + payloadReport.message); +} + +uint16_t CheckedU16(uint32_t value) +{ + return static_cast(std::min(value, std::numeric_limits::max())); +} + +bool AddOverflowsU16(uint16_t start, uint32_t count) +{ + return static_cast(start) + count > + static_cast(std::numeric_limits::max()) + 1U; +} + +TileXRCcuRange EffectiveLocalWaitCkeRange(const TileXRCcuResourceAllocation& allocation) +{ + TileXRCcuRange local = allocation.localWaitCke.num == 0 ? allocation.notifyCke : allocation.localWaitCke; + if (allocation.sourceCke.num == 0) { + return local; + } + if (local.dieId == allocation.sourceCke.dieId && + static_cast(local.startId) + local.num == allocation.sourceCke.startId) { + local.num = CheckedU16(static_cast(local.num) + allocation.sourceCke.num); + } + return local; +} + +TileXRCcuRange EffectiveRemoteNotifyCkeRange(const TileXRCcuResourceAllocation& allocation) +{ + return allocation.remoteNotifyCke.num == 0 ? allocation.notifyCke : allocation.remoteNotifyCke; +} + +uint16_t SelectLowerLayerWqeBasicBlockStride() +{ + const char* value = std::getenv("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE"); + if (value != nullptr && std::strcmp(value, "hcomm_cap") == 0) { + return TILEXR_CCU_HCOMM_WQE_BASIC_BLOCKS_PER_ROUTE; + } + return TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE; +} + +bool LowerLayerEnvEquals(const char* name, const char* expected) +{ + const char* value = std::getenv(name); + return value != nullptr && std::strcmp(value, expected) == 0; +} + +bool IsEmptyEndpointEid(const std::array& eid) +{ + return std::all_of(eid.begin(), eid.end(), [](uint8_t value) { + return value == 0; + }); +} + +bool HasCompleteVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute& route) +{ + return route.endpointRouteVerified && + !IsEmptyEndpointEid(route.remoteEid) && + route.doorbellVa != 0 && + route.doorbellTokenId != 0 && + route.sqDepth != 0; +} + +bool RangeContains(uint32_t start, uint32_t count, uint32_t value) +{ + return count != 0 && value >= start && value < start + count; +} + +uint32_t SelectLowerLayerPfeOffset(uint8_t dieId, uint32_t pfeId) +{ + if (LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE", "hcomm_die")) { + return static_cast(dieId) * TILEXR_CCU_HCOMM_PER_DIE_PFE_RESERVED_NUM + pfeId; + } + return pfeId; +} + +void ApplyHcommFeIdPfePartition(uint32_t pfeId, TileXRCcuLowerLayerTransportSnapshot* snapshot) +{ + if (snapshot == nullptr) { + return; + } + if (pfeId > TILEXR_CCU_HCOMM_MAX_INNER_FE_ID) { + snapshot->startLocalJettyCtxId = TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID; + snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM; + } else { + snapshot->startLocalJettyCtxId = CheckedU16(pfeId * TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM); + snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + } + snapshot->startJettyId = CheckedU16(TILEXR_CCU_DEFAULT_START_JETTY_ID + snapshot->startLocalJettyCtxId); +} + +void ApplyHcommOrderedPfePartition(TileXRCcuLowerLayerTransportSnapshot* snapshot) +{ + if (snapshot == nullptr) { + return; + } + snapshot->startLocalJettyCtxId = 0; + snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + snapshot->startJettyId = TILEXR_CCU_DEFAULT_START_JETTY_ID; +} + +void ApplyLowerLayerPfePartition(uint32_t pfeId, TileXRCcuLowerLayerTransportSnapshot* snapshot) +{ + if (LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", "hcomm_fe_id")) { + ApplyHcommFeIdPfePartition(pfeId, snapshot); + return; + } + if (LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", "hcomm_ordered") || + LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", "hcomm")) { + ApplyHcommOrderedPfePartition(snapshot); + } +} + +int ValidateSpec( + const TileXRCcuLowerLayerPlanSpec& spec, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + if (!spec.msidToken.valid) { + return Fail(plan, report, "missing lower-layer CCU MSID token"); + } + if (spec.jettys.empty()) { + return Fail(plan, report, "missing lower-layer CCU local jetty contexts"); + } + if (spec.channels.empty()) { + return Fail(plan, report, "missing lower-layer CCU channel contexts"); + } + if (!spec.xnClear.valid || spec.xnClear.count == 0) { + return Fail(plan, report, "missing lower-layer CCU local XN clear range"); + } + if (!spec.ckeClear.valid || spec.ckeClear.count == 0) { + return Fail(plan, report, "missing lower-layer CCU CKE clear range"); + } + if (spec.jettys.size() > std::numeric_limits::max()) { + return Fail(plan, report, "too many lower-layer CCU local jetty contexts"); + } + if (spec.pfe.jettyCount != 0 && spec.pfe.jettyCount > 128U) { + return Fail(plan, report, "lower-layer CCU PFE jetty count is out of range"); + } + if (spec.pfe.jettyCount != 0 && spec.pfe.jettyCount < spec.jettys.size()) { + return Fail(plan, report, "lower-layer CCU PFE jetty window is smaller than local jetty contexts"); + } + if (AddOverflowsU16(spec.pfe.startLocalJettyCtxId, static_cast(spec.jettys.size()))) { + return Fail(plan, report, "lower-layer CCU local jetty context range overflows"); + } + return TILEXR_SUCCESS; +} + +void FillReport(const TileXRCcuLowerLayerInstallPlan& plan, TileXRCcuLowerLayerPlanBuilderReport* report) +{ + if (report == nullptr) { + return; + } + report->msidTokenCount = static_cast(plan.msidTokens.size()); + report->pfeCount = static_cast(plan.pfes.size()); + report->jettyCount = static_cast(plan.jettys.size()); + report->localJettyCtxCount = 0; + for (const auto& jetty : plan.jettys) { + report->localJettyCtxCount += static_cast(jetty.ctxs.size()); + } + report->channelCount = static_cast(plan.channels.size()); + report->ckeClearCount = static_cast(plan.ckeClears.size()); + report->message = "ok"; +} + +void FillTemplateReport( + const TileXRCcuLowerLayerTransportSnapshot& snapshot, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + if (report == nullptr) { + return; + } + report->msidTokenCount = snapshot.msidToken.valid ? 1U : 0U; + report->pfeCount = snapshot.routes.empty() ? 0U : 1U; + report->jettyCount = static_cast(snapshot.routes.size()); + report->localJettyCtxCount = static_cast(snapshot.routes.size()); + report->channelCount = static_cast(snapshot.routes.size()); + report->ckeClearCount = snapshot.ckeCount == 0 ? 0U : 1U; + report->message = "ok"; +} + +void AppendRemoteNotifyCkeClears( + const TileXRCcuLowerLayerTransportSnapshot& snapshot, + TileXRCcuLowerLayerInstallPlan* plan) +{ + if (plan == nullptr) { + return; + } + std::vector notifyCkes; + notifyCkes.reserve(snapshot.routes.size()); + for (const auto& route : snapshot.routes) { + if (route.remoteNotifyCke == 0 || + RangeContains(snapshot.ckeStartId, snapshot.ckeCount, route.remoteNotifyCke)) { + continue; + } + notifyCkes.push_back(route.remoteNotifyCke); + } + if (notifyCkes.empty()) { + return; + } + + std::sort(notifyCkes.begin(), notifyCkes.end()); + notifyCkes.erase(std::unique(notifyCkes.begin(), notifyCkes.end()), notifyCkes.end()); + + uint32_t rangeStart = notifyCkes.front(); + uint32_t previous = rangeStart; + for (size_t i = 1; i <= notifyCkes.size(); ++i) { + if (i < notifyCkes.size() && notifyCkes[i] == previous + 1U) { + previous = notifyCkes[i]; + continue; + } + plan->ckeClears.push_back({ + snapshot.dieId, + rangeStart, + previous - rangeStart + 1U, + }); + if (i < notifyCkes.size()) { + rangeStart = notifyCkes[i]; + previous = rangeStart; + } + } +} + +void AppendRemoteXnClears( + const TileXRCcuLowerLayerTransportSnapshot& snapshot, + TileXRCcuLowerLayerInstallPlan* plan) +{ + if (plan == nullptr) { + return; + } + std::vector remoteXns; + remoteXns.reserve(snapshot.routes.size()); + for (const auto& route : snapshot.routes) { + if (route.remoteXnId == 0 || + RangeContains(snapshot.xnStartId, snapshot.xnCount, route.remoteXnId)) { + continue; + } + remoteXns.push_back(route.remoteXnId); + } + if (remoteXns.empty()) { + return; + } + + std::sort(remoteXns.begin(), remoteXns.end()); + remoteXns.erase(std::unique(remoteXns.begin(), remoteXns.end()), remoteXns.end()); + + uint32_t rangeStart = remoteXns.front(); + uint32_t previous = rangeStart; + for (size_t i = 1; i <= remoteXns.size(); ++i) { + if (i < remoteXns.size() && remoteXns[i] == previous + 1U) { + previous = remoteXns[i]; + continue; + } + plan->xnClears.push_back({ + snapshot.dieId, + rangeStart, + previous - rangeStart + 1U, + }); + if (i < remoteXns.size()) { + rangeStart = remoteXns[i]; + previous = rangeStart; + } + } +} + +} // namespace + +int TileXRCcuBuildLowerLayerInstallPlan( + const TileXRCcuLowerLayerPlanSpec& spec, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + ResetReport(report); + if (plan == nullptr) { + return Fail(nullptr, report, "missing output lower-layer CCU install plan"); + } + *plan = TileXRCcuLowerLayerInstallPlan{}; + + if (ValidateSpec(spec, plan, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuLowerLayerInstallPlan result; + result.msidTokens.push_back({ + spec.msidToken.dieId, + spec.msidToken.msId, + spec.msidToken.tokenId, + spec.msidToken.tokenValue, + }); + + TileXRCcuLowerLayerPayloadReport payloadReport; + TileXRCcuPfeInstall pfe; + pfe.dieId = spec.pfe.dieId; + pfe.pfeOffset = spec.pfe.pfeOffset; + TileXRCcuPfeCtxSpec pfeCtxSpec; + pfeCtxSpec.startJettyId = spec.pfe.startJettyId; + pfeCtxSpec.jettyCount = spec.pfe.jettyCount == 0 ? + CheckedU16(static_cast(spec.jettys.size())) : + spec.pfe.jettyCount; + pfeCtxSpec.startLocalJettyCtxId = spec.pfe.startLocalJettyCtxId; + if (TileXRCcuBuildPfeCtx(pfeCtxSpec, &pfe.ctx, &payloadReport) != TILEXR_SUCCESS) { + return FailPayload(plan, report, payloadReport); + } + result.pfes.push_back(pfe); + + TileXRCcuJettyInstall jettyInstall; + jettyInstall.dieId = spec.pfe.dieId; + jettyInstall.startJettyCtxId = spec.pfe.startLocalJettyCtxId; + for (const auto& jettySpec : spec.jettys) { + if (jettySpec.startJettyCtxId != 0 && jettySpec.startJettyCtxId != jettyInstall.startJettyCtxId + + jettyInstall.ctxs.size()) { + return Fail(plan, report, "lower-layer CCU local jetty contexts must be contiguous"); + } + TileXRCcuLocalJettyCtxData ctx; + TileXRCcuLocalJettyCtxSpec ctxSpec; + ctxSpec.dieId = jettySpec.dieId; + ctxSpec.pfeId = jettySpec.pfeId; + ctxSpec.doorbellVa = jettySpec.doorbellVa; + ctxSpec.doorbellTokenId = jettySpec.doorbellTokenId; + ctxSpec.doorbellTokenValue = jettySpec.doorbellTokenValue; + ctxSpec.sqDepth = jettySpec.sqDepth; + ctxSpec.wqeBasicBlockStartId = jettySpec.wqeBasicBlockStartId; + if (TileXRCcuBuildLocalJettyCtx(ctxSpec, &ctx, &payloadReport) != TILEXR_SUCCESS) { + return FailPayload(plan, report, payloadReport); + } + jettyInstall.ctxs.push_back(ctx); + } + result.jettys.push_back(jettyInstall); + + for (const auto& channelSpec : spec.channels) { + TileXRCcuChannelInstall channel; + channel.dieId = channelSpec.dieId; + channel.channelId = channelSpec.channelId; + TileXRCcuChannelCtxV1Spec ctxSpec; + ctxSpec.remoteEid = channelSpec.remoteEid; + ctxSpec.tpn = channelSpec.tpn; + ctxSpec.sourcePfeId = channelSpec.sourcePfeId; + ctxSpec.startJettyId = channelSpec.startJettyId; + ctxSpec.jettyCount = channelSpec.jettyCount == 0 ? + CheckedU16(static_cast(spec.jettys.size())) : + channelSpec.jettyCount; + ctxSpec.dieId = channelSpec.dieId; + ctxSpec.memoryTokenId = channelSpec.memoryTokenId; + ctxSpec.memoryTokenValue = channelSpec.memoryTokenValue; + ctxSpec.remoteCcuVa = channelSpec.remoteCcuVa; + if (TileXRCcuBuildChannelCtxV1(ctxSpec, &channel.ctx, &payloadReport) != TILEXR_SUCCESS) { + return FailPayload(plan, report, payloadReport); + } + result.channels.push_back(channel); + } + + result.xnClears.push_back({ + spec.xnClear.dieId, + spec.xnClear.startXnId, + spec.xnClear.count, + }); + + result.ckeClears.push_back({ + spec.ckeClear.dieId, + spec.ckeClear.startCkeId, + spec.ckeClear.count, + }); + result.remoteXnBindings = spec.remoteXnBindings; + + *plan = result; + FillReport(*plan, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildLowerLayerTransportTemplate( + const TileXRCcuBasicInfo& basicInfo, + const TileXRCcuResourceAllocation& allocation, + const std::vector& remoteCcuBuffers, + TileXRCcuLowerLayerTransportSnapshot* snapshot, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + ResetReport(report); + if (snapshot == nullptr) { + return Fail(nullptr, report, "missing output lower-layer CCU transport template"); + } + *snapshot = TileXRCcuLowerLayerTransportSnapshot{}; + + const TileXRCcuRange localWaitCke = EffectiveLocalWaitCkeRange(allocation); + const TileXRCcuRange remoteNotifyCke = EffectiveRemoteNotifyCkeRange(allocation); + if (allocation.channels.num == 0 || allocation.localXn.num == 0 || localWaitCke.num == 0 || + remoteNotifyCke.num == 0 || allocation.remoteXn.num == 0) { + return Fail(nullptr, report, "missing lower-layer CCU allocated resources"); + } + if (remoteCcuBuffers.empty() || remoteCcuBuffers.size() != allocation.remoteXn.num) { + return Fail(nullptr, report, "remote CCU buffer template count does not match remote XN allocation"); + } + if (allocation.channels.num < remoteCcuBuffers.size()) { + return Fail(nullptr, report, "channel allocation count does not match lower-layer route count"); + } + if (remoteCcuBuffers.size() > std::numeric_limits::max()) { + return Fail(nullptr, report, "too many lower-layer CCU remote routes"); + } + + TileXRCcuLowerLayerTransportSnapshot result; + result.msidToken.dieId = basicInfo.dieId; + result.msidToken.msId = basicInfo.msId; + if (basicInfo.msidToken.valid && basicInfo.msidToken.tokenId != 0) { + result.msidToken.tokenId = basicInfo.msidToken.tokenId; + result.msidToken.tokenValue = basicInfo.msidToken.tokenValue; + result.msidToken.valid = true; + } + result.dieId = basicInfo.dieId; + result.pfeId = allocation.channels.startId; + result.pfeOffset = SelectLowerLayerPfeOffset(basicInfo.dieId, result.pfeId); + result.startJettyId = TILEXR_CCU_DEFAULT_START_JETTY_ID; + result.startLocalJettyCtxId = TILEXR_CCU_DEFAULT_START_LOCAL_JETTY_CTX_ID; + ApplyLowerLayerPfePartition(result.pfeId, &result); + result.xnStartId = allocation.localXn.startId; + result.xnCount = allocation.localXn.num; + result.ckeStartId = localWaitCke.startId; + result.ckeCount = localWaitCke.num; + result.routes.reserve(remoteCcuBuffers.size()); + + const uint16_t wqeBasicBlockStride = SelectLowerLayerWqeBasicBlockStride(); + for (uint32_t i = 0; i < remoteCcuBuffers.size(); ++i) { + const auto& remoteCcuBuffer = remoteCcuBuffers[i]; + if (remoteCcuBuffer.remoteCcuVa == 0) { + return Fail(nullptr, report, "missing remote CCU VA for lower-layer route"); + } + TileXRCcuLowerLayerTransportRoute route; + route.channelId = allocation.channels.startId + i; + route.peerRank = remoteCcuBuffer.peerRank == TILEXR_CCU_REMOTE_PEER_RANK_UNKNOWN ? + i : + remoteCcuBuffer.peerRank; + route.remoteXnId = remoteCcuBuffer.remoteXnId == 0 ? + static_cast(allocation.remoteXn.startId + i) : + remoteCcuBuffer.remoteXnId; + route.remoteNotifyCke = remoteCcuBuffer.remoteNotifyCke == 0 ? + static_cast(remoteNotifyCke.startId + i) : + remoteCcuBuffer.remoteNotifyCke; + if (i > std::numeric_limits::max() / wqeBasicBlockStride) { + return Fail(nullptr, report, "lower-layer CCU WQE basic block start overflows"); + } + route.wqeBasicBlockStartId = static_cast(i * wqeBasicBlockStride); + route.remoteCcuVa = remoteCcuBuffer.remoteCcuVa; + route.memoryTokenId = remoteCcuBuffer.memoryTokenId; + route.memoryTokenValue = remoteCcuBuffer.memoryTokenValue; + route.channelResourceOwnerVerified = remoteCcuBuffer.channelResourceOwnerVerified; + route.transportResourceExchangeVerified = remoteCcuBuffer.transportResourceExchangeVerified; + if (remoteCcuBuffer.endpointRouteVerified && + !IsEmptyEndpointEid(remoteCcuBuffer.remoteEid) && + remoteCcuBuffer.doorbellVa != 0 && + remoteCcuBuffer.doorbellTokenId != 0 && + remoteCcuBuffer.sqDepth != 0) { + route.remoteEid = remoteCcuBuffer.remoteEid; + route.tpn = remoteCcuBuffer.tpn; + route.doorbellVa = remoteCcuBuffer.doorbellVa; + route.doorbellTokenId = remoteCcuBuffer.doorbellTokenId; + route.doorbellTokenValue = remoteCcuBuffer.doorbellTokenValue; + route.sqDepth = remoteCcuBuffer.sqDepth; + route.localDoorbellVa = remoteCcuBuffer.localDoorbellVa; + route.localDoorbellTokenId = remoteCcuBuffer.localDoorbellTokenId; + route.localDoorbellTokenValue = remoteCcuBuffer.localDoorbellTokenValue; + route.localSqDepth = remoteCcuBuffer.localSqDepth; + route.endpointRouteVerified = true; + } + result.routes.push_back(route); + } + + *snapshot = result; + FillTemplateReport(*snapshot, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuOverlayVerifiedEndpointRoutes( + const std::vector& verifiedRoutes, + TileXRCcuLowerLayerTransportSnapshot* snapshot, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + ResetReport(report); + if (snapshot == nullptr) { + return Fail(nullptr, report, "missing lower-layer CCU transport snapshot for endpoint route overlay"); + } + if (verifiedRoutes.empty()) { + FillTemplateReport(*snapshot, report); + return TILEXR_SUCCESS; + } + + for (const auto& verified : verifiedRoutes) { + if (!HasCompleteVerifiedEndpointRoute(verified)) { + return Fail(nullptr, report, "verified endpoint route is incomplete"); + } + auto routeIt = std::find_if( + snapshot->routes.begin(), + snapshot->routes.end(), + [&verified](const TileXRCcuLowerLayerTransportRoute& route) { + return route.channelId == verified.channelId; + }); + if (routeIt == snapshot->routes.end()) { + return Fail(nullptr, report, "verified endpoint route does not match an allocated channel"); + } + + routeIt->remoteEid = verified.remoteEid; + routeIt->tpn = verified.tpn; + routeIt->doorbellVa = verified.doorbellVa; + routeIt->doorbellTokenId = verified.doorbellTokenId; + routeIt->doorbellTokenValue = verified.doorbellTokenValue; + routeIt->sqDepth = verified.sqDepth; + routeIt->localDoorbellVa = verified.localDoorbellVa; + routeIt->localDoorbellTokenId = verified.localDoorbellTokenId; + routeIt->localDoorbellTokenValue = verified.localDoorbellTokenValue; + routeIt->localSqDepth = verified.localSqDepth; + routeIt->endpointRouteVerified = true; + routeIt->channelResourceOwnerVerified = verified.channelResourceOwnerVerified; + routeIt->transportResourceExchangeVerified = verified.transportResourceExchangeVerified; + } + + FillTemplateReport(*snapshot, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot& snapshot, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report) +{ + ResetReport(report); + if (plan == nullptr) { + return Fail(nullptr, report, "missing output lower-layer CCU install plan"); + } + *plan = TileXRCcuLowerLayerInstallPlan{}; + + if (snapshot.routes.empty()) { + return Fail(plan, report, "missing lower-layer CCU transport routes"); + } + if (snapshot.routes.size() > std::numeric_limits::max()) { + return Fail(plan, report, "too many lower-layer CCU transport routes"); + } + + TileXRCcuLowerLayerPlanSpec spec; + spec.msidToken = snapshot.msidToken; + spec.pfe.dieId = snapshot.dieId; + spec.pfe.pfeOffset = snapshot.pfeOffset; + spec.pfe.startJettyId = snapshot.startJettyId; + spec.pfe.jettyCount = snapshot.pfeJettyCount; + spec.pfe.startLocalJettyCtxId = snapshot.startLocalJettyCtxId; + spec.xnClear.dieId = snapshot.dieId; + spec.xnClear.startXnId = snapshot.xnStartId; + spec.xnClear.count = snapshot.xnCount; + spec.xnClear.valid = snapshot.xnCount != 0; + spec.ckeClear.dieId = snapshot.dieId; + spec.ckeClear.startCkeId = snapshot.ckeStartId; + spec.ckeClear.count = snapshot.ckeCount; + spec.ckeClear.valid = snapshot.ckeCount != 0; + + uint32_t routeIndex = 0; + for (const auto& route : snapshot.routes) { + TileXRCcuLowerLayerJettySpec jetty; + jetty.dieId = snapshot.dieId; + jetty.pfeId = snapshot.pfeId; + jetty.startJettyCtxId = static_cast(snapshot.startLocalJettyCtxId + routeIndex); + jetty.doorbellVa = route.localDoorbellVa == 0 ? route.doorbellVa : route.localDoorbellVa; + jetty.doorbellTokenId = route.localDoorbellTokenId == 0 ? + route.doorbellTokenId : + route.localDoorbellTokenId; + jetty.doorbellTokenValue = route.localDoorbellVa == 0 ? + route.doorbellTokenValue : + route.localDoorbellTokenValue; + jetty.sqDepth = route.localSqDepth == 0 ? route.sqDepth : route.localSqDepth; + jetty.wqeBasicBlockStartId = route.wqeBasicBlockStartId; + spec.jettys.push_back(jetty); + + TileXRCcuLowerLayerChannelSpec channel; + channel.dieId = snapshot.dieId; + channel.channelId = route.channelId; + channel.remoteEid = route.remoteEid; + channel.tpn = route.tpn; + channel.sourcePfeId = snapshot.pfeId; + channel.startJettyId = route.startJettyId == 0 ? + static_cast(snapshot.startJettyId + routeIndex) : + route.startJettyId; + channel.jettyCount = 1; + channel.memoryTokenId = route.memoryTokenId; + channel.memoryTokenValue = route.memoryTokenValue; + channel.remoteCcuVa = route.remoteCcuVa; + const auto channelIt = std::find_if( + spec.channels.begin(), + spec.channels.end(), + [&channel](const TileXRCcuLowerLayerChannelSpec& existing) { + return existing.dieId == channel.dieId && existing.channelId == channel.channelId; + }); + if (channelIt == spec.channels.end()) { + spec.channels.push_back(channel); + } + + TileXRCcuRemoteXnBindingProof remoteXn; + remoteXn.dieId = snapshot.dieId; + remoteXn.channelId = static_cast(route.channelId); + remoteXn.localXn = static_cast(snapshot.xnStartId + routeIndex); + remoteXn.remoteXn = route.remoteXnId; + remoteXn.notifyCke = route.remoteNotifyCke == 0 ? + static_cast(snapshot.ckeStartId + routeIndex) : + route.remoteNotifyCke; + remoteXn.peerRank = route.peerRank; + remoteXn.peerExchangeObserved = route.remoteXnId != 0; + remoteXn.localWaitCke = static_cast(snapshot.ckeStartId + routeIndex); + remoteXn.endpointRouteVerified = route.endpointRouteVerified; + remoteXn.channelResourceOwnerVerified = route.channelResourceOwnerVerified; + remoteXn.transportResourceExchangeVerified = route.transportResourceExchangeVerified; + spec.remoteXnBindings.push_back(remoteXn); + ++routeIndex; + } + + const int ret = TileXRCcuBuildLowerLayerInstallPlan(spec, plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + AppendRemoteXnClears(snapshot, plan); + AppendRemoteNotifyCkeClears(snapshot, plan); + FillReport(*plan, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h new file mode 100644 index 00000000..26ff8c7f --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h @@ -0,0 +1,182 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_LOWER_LAYER_PLAN_BUILDER_H +#define TILEXR_CCU_LOWER_LAYER_PLAN_BUILDER_H + +#include "ccu/tilexr_ccu_install_provider.h" +#include "ccu/tilexr_ccu_lower_layer_payloads.h" + +#include +#include +#include +#include + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_REMOTE_PEER_RANK_UNKNOWN = 0xffffffffU; + +struct TileXRCcuLowerLayerMsidTokenSpec { + uint8_t dieId = 0; + uint32_t msId = 0; + uint32_t tokenId = 0; + uint32_t tokenValue = 0; + bool valid = false; +}; + +struct TileXRCcuLowerLayerPfeSpec { + uint8_t dieId = 0; + uint32_t pfeOffset = 0; + uint16_t startJettyId = 0; + uint16_t jettyCount = 0; + uint16_t startLocalJettyCtxId = 0; +}; + +struct TileXRCcuLowerLayerJettySpec { + uint8_t dieId = 0; + uint32_t pfeId = 0; + uint16_t startJettyCtxId = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + uint16_t wqeBasicBlockStartId = 0; +}; + +struct TileXRCcuLowerLayerChannelSpec { + uint8_t dieId = 0; + uint32_t channelId = 0; + std::array remoteEid {}; + uint32_t tpn = 0; + uint32_t sourcePfeId = 0; + uint16_t startJettyId = 0; + uint16_t jettyCount = 0; + uint32_t memoryTokenId = 0; + uint32_t memoryTokenValue = 0; + uint64_t remoteCcuVa = 0; +}; + +struct TileXRCcuLowerLayerCkeClearSpec { + uint8_t dieId = 0; + uint32_t startCkeId = 0; + uint32_t count = 0; + bool valid = false; +}; + +struct TileXRCcuLowerLayerXnClearSpec { + uint8_t dieId = 0; + uint32_t startXnId = 0; + uint32_t count = 0; + bool valid = false; +}; + +struct TileXRCcuLowerLayerPlanSpec { + TileXRCcuLowerLayerMsidTokenSpec msidToken; + TileXRCcuLowerLayerPfeSpec pfe; + std::vector jettys; + std::vector channels; + TileXRCcuLowerLayerXnClearSpec xnClear; + TileXRCcuLowerLayerCkeClearSpec ckeClear; + std::vector remoteXnBindings; +}; + +struct TileXRCcuLowerLayerPlanBuilderReport { + uint32_t msidTokenCount = 0; + uint32_t pfeCount = 0; + uint32_t jettyCount = 0; + uint32_t localJettyCtxCount = 0; + uint32_t channelCount = 0; + uint32_t ckeClearCount = 0; + std::string message; +}; + +struct TileXRCcuRemoteCcuBufferInfo { + uint64_t remoteCcuVa = 0; + uint32_t peerRank = TILEXR_CCU_REMOTE_PEER_RANK_UNKNOWN; + uint32_t memoryTokenId = 0; + uint32_t rawMemoryTokenId = 0; + uint32_t memoryTokenValue = 0; + uint16_t remoteXnId = 0; + uint16_t remoteNotifyCke = 0; + std::array remoteEid {}; + uint32_t tpn = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + uint64_t localDoorbellVa = 0; + uint32_t localDoorbellTokenId = 0; + uint32_t localDoorbellTokenValue = 0; + uint32_t localSqDepth = 0; + bool endpointRouteVerified = false; + bool channelResourceOwnerVerified = false; + bool transportResourceExchangeVerified = false; +}; + +struct TileXRCcuLowerLayerTransportRoute { + uint32_t channelId = 0; + uint32_t peerRank = 0; + uint16_t remoteXnId = 0; + uint16_t remoteNotifyCke = 0; + std::array remoteEid {}; + uint32_t tpn = 0; + uint64_t doorbellVa = 0; + uint32_t doorbellTokenId = 0; + uint32_t doorbellTokenValue = 0; + uint32_t sqDepth = 0; + uint64_t localDoorbellVa = 0; + uint32_t localDoorbellTokenId = 0; + uint32_t localDoorbellTokenValue = 0; + uint32_t localSqDepth = 0; + uint16_t startJettyId = 0; + uint16_t wqeBasicBlockStartId = 0; + uint32_t memoryTokenId = 0; + uint32_t memoryTokenValue = 0; + uint64_t remoteCcuVa = 0; + bool endpointRouteVerified = false; + bool channelResourceOwnerVerified = false; + bool transportResourceExchangeVerified = false; +}; + +struct TileXRCcuLowerLayerTransportSnapshot { + TileXRCcuLowerLayerMsidTokenSpec msidToken; + uint8_t dieId = 0; + uint32_t pfeOffset = 0; + uint32_t pfeId = 0; + uint16_t startJettyId = 0; + uint16_t pfeJettyCount = 0; + uint16_t startLocalJettyCtxId = 0; + uint32_t xnStartId = 0; + uint32_t xnCount = 0; + uint32_t ckeStartId = 0; + uint32_t ckeCount = 0; + std::vector routes; +}; + +int TileXRCcuBuildLowerLayerInstallPlan( + const TileXRCcuLowerLayerPlanSpec& spec, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report); + +int TileXRCcuBuildLowerLayerTransportTemplate( + const TileXRCcuBasicInfo& basicInfo, + const TileXRCcuResourceAllocation& allocation, + const std::vector& remoteCcuBuffers, + TileXRCcuLowerLayerTransportSnapshot* snapshot, + TileXRCcuLowerLayerPlanBuilderReport* report); + +int TileXRCcuOverlayVerifiedEndpointRoutes( + const std::vector& verifiedRoutes, + TileXRCcuLowerLayerTransportSnapshot* snapshot, + TileXRCcuLowerLayerPlanBuilderReport* report); + +int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot& snapshot, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_LOWER_LAYER_PLAN_BUILDER_H diff --git a/src/comm/ccu/tilexr_ccu_memory_program.cpp b/src/comm/ccu/tilexr_ccu_memory_program.cpp new file mode 100644 index 00000000..a1559abf --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_memory_program.cpp @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_memory_program.h" + +namespace TileXR { +namespace { + +constexpr uint64_t TILEXR_CCU_TOKEN_VALID_SHIFT = 52ULL; +constexpr uint64_t TILEXR_CCU_TOKEN_ID_SHIFT = 32ULL; +constexpr uint64_t TILEXR_CCU_TOKEN_ID_MASK = 0xfffffULL; +constexpr uint64_t TILEXR_CCU_TOKEN_VALUE_MASK = 0xffffffffULL; + +void ResetReport(TileXRCcuMemoryProgramReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuMemoryProgramReport{}; + } +} + +int Fail( + std::vector* program, + TileXRCcuMemoryProgramReport* report, + const std::string& message) +{ + if (program != nullptr) { + program->clear(); + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int ValidateSpec( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report) +{ + if (program == nullptr) { + return Fail(program, report, "missing output CCU memory copy program"); + } + if (spec.localGsa == 0 || spec.localXn == 0 || spec.remoteGsa == 0 || spec.remoteXn == 0 || + spec.lengthXn == 0) { + return Fail(program, report, "missing CCU memory copy GSA/XN resources"); + } + if (spec.localAddr == 0 || spec.localToken == 0 || spec.remoteAddr == 0 || spec.remoteToken == 0) { + return Fail(program, report, "missing CCU memory copy address/token inputs"); + } + if (spec.lengthBytes == 0) { + return Fail(program, report, "missing CCU memory copy length"); + } + if (spec.channelId == 0) { + return Fail(program, report, "missing CCU memory copy channel"); + } + if (spec.completionCke == 0 || spec.completionMask == 0) { + return Fail(program, report, "missing CCU memory copy completion CKE"); + } + if (spec.reduceDataType > 0xfU || spec.reduceOpCode > 0xfU) { + return Fail(program, report, "CCU memory copy reduce fields exceed v1 encoding width"); + } + return TILEXR_SUCCESS; +} + +int AppendLoadImmediates( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToGsa(spec.localGsa, spec.localAddr, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode local memory address GSA load"); + } + program->push_back(instr); + + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, spec.localToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode local memory token XN load"); + } + program->push_back(instr); + + if (TileXRCcuEncodeLoadImdToGsa(spec.remoteGsa, spec.remoteAddr, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode remote memory address GSA load"); + } + program->push_back(instr); + + if (TileXRCcuEncodeLoadImdToXn(spec.remoteXn, spec.remoteToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode remote memory token XN load"); + } + program->push_back(instr); + + if (TileXRCcuEncodeLoadImdToXn(spec.lengthXn, spec.lengthBytes, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode memory copy length XN load"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendTransfer( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report) +{ + TileXRCcuMemTransferSpec transfer; + transfer.localGsa = spec.localGsa; + transfer.localXn = spec.localXn; + transfer.remoteGsa = spec.remoteGsa; + transfer.remoteXn = spec.remoteXn; + transfer.lengthXn = spec.lengthXn; + transfer.channelId = spec.channelId; + transfer.reduceDataType = spec.reduceDataType; + transfer.reduceOpCode = spec.reduceOpCode; + transfer.setCkeId = spec.completionCke; + transfer.setCkeMask = spec.completionMask; + transfer.clearWait = true; + transfer.lengthFromXn = true; + transfer.reduceEnabled = spec.reduceEnabled; + + TileXRCcuInstr instr; + const int ret = spec.direction == TileXRCcuMemoryCopyDirection::RemoteToLocal ? + TileXRCcuEncodeTransRmtMemToLocMem(transfer, &instr) : + TileXRCcuEncodeTransLocMemToRmtMem(transfer, &instr); + if (ret != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU memory transfer instruction"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendCompletionWait( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report) +{ + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.completionCke; + wait.waitMask = spec.completionMask; + wait.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode CCU memory copy completion wait"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +void FillReport(const std::vector& program, TileXRCcuMemoryProgramReport* report) +{ + if (report == nullptr) { + return; + } + report->loadInstructionCount = 5; + report->transferInstructionCount = 1; + report->waitInstructionCount = 1; + report->totalInstructionCount = static_cast(program.size()); + report->message = "ok"; +} + +} // namespace + +uint64_t TileXRCcuPackMemoryToken(uint32_t tokenId, uint32_t tokenValue, bool valid) +{ + const uint64_t validBits = valid ? 1ULL : 0ULL; + return (validBits << TILEXR_CCU_TOKEN_VALID_SHIFT) | + ((static_cast(tokenId) & TILEXR_CCU_TOKEN_ID_MASK) << TILEXR_CCU_TOKEN_ID_SHIFT) | + (static_cast(tokenValue) & TILEXR_CCU_TOKEN_VALUE_MASK); +} + +int TileXRCcuBuildMemoryCopyProgram( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report) +{ + ResetReport(report); + if (program != nullptr) { + program->clear(); + } + int ret = ValidateSpec(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + program->reserve(7); + ret = AppendLoadImmediates(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = AppendTransfer(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = AppendCompletionWait(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + FillReport(*program, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_memory_program.h b/src/comm/ccu/tilexr_ccu_memory_program.h new file mode 100644 index 00000000..19640b5e --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_memory_program.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_MEMORY_PROGRAM_H +#define TILEXR_CCU_MEMORY_PROGRAM_H + +#include "ccu/tilexr_ccu_microcode.h" + +#include +#include +#include + +namespace TileXR { + +enum class TileXRCcuMemoryCopyDirection { + RemoteToLocal = 0, + LocalToRemote = 1, +}; + +struct TileXRCcuMemoryCopySpec { + TileXRCcuMemoryCopyDirection direction = TileXRCcuMemoryCopyDirection::RemoteToLocal; + uint16_t localGsa = 0; + uint16_t localXn = 0; + uint16_t remoteGsa = 0; + uint16_t remoteXn = 0; + uint16_t lengthXn = 0; + uint64_t localAddr = 0; + uint64_t localToken = 0; + uint64_t remoteAddr = 0; + uint64_t remoteToken = 0; + uint64_t lengthBytes = 0; + uint16_t channelId = 0; + uint16_t completionCke = 0; + uint16_t completionMask = 0; + uint16_t reduceDataType = 0; + uint16_t reduceOpCode = 0; + bool reduceEnabled = false; +}; + +struct TileXRCcuMemoryProgramReport { + uint32_t loadInstructionCount = 0; + uint32_t transferInstructionCount = 0; + uint32_t waitInstructionCount = 0; + uint32_t totalInstructionCount = 0; + std::string message; +}; + +uint64_t TileXRCcuPackMemoryToken(uint32_t tokenId, uint32_t tokenValue, bool valid); + +int TileXRCcuBuildMemoryCopyProgram( + const TileXRCcuMemoryCopySpec& spec, + std::vector* program, + TileXRCcuMemoryProgramReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_MEMORY_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_microcode.cpp b/src/comm/ccu/tilexr_ccu_microcode.cpp new file mode 100644 index 00000000..44abfdbe --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_microcode.cpp @@ -0,0 +1,298 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_microcode.h" + +namespace TileXR { +namespace { + +constexpr uint64_t TILEXR_CCU_LOAD_SQE_ARGS_TO_X_HEADER = 0x0001U; +constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_GSA_HEADER = 0x0002U; +constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_XN_HEADER = 0x0003U; +constexpr uint64_t TILEXR_CCU_SET_CKE_HEADER = 0x0802U; +constexpr uint64_t TILEXR_CCU_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint64_t TILEXR_CCU_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint64_t TILEXR_CCU_SYNC_CKE_HEADER = 0x100bU; +constexpr uint64_t TILEXR_CCU_SYNC_XN_HEADER = 0x100dU; +constexpr uint64_t TILEXR_CCU_SYNC_XN_TRACE_FLAG = 0x0001000000000000ULL; + +void ClearInstr(TileXRCcuInstr* instr) +{ + for (auto& word : instr->words) { + word = 0; + } +} + +int ValidateInstrOutput(TileXRCcuInstr* instr) +{ + if (instr == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + ClearInstr(instr); + return TILEXR_SUCCESS; +} + +uint64_t PackSlots(uint16_t slot0, uint16_t slot1, uint16_t slot2, uint16_t slot3) +{ + return static_cast(slot0) | + (static_cast(slot1) << 16U) | + (static_cast(slot2) << 32U) | + (static_cast(slot3) << 48U); +} + +uint16_t ClearTypeBit(bool clearWait) +{ + return clearWait ? 1U : 0U; +} + +uint16_t TransferControlSlot(const TileXRCcuMemTransferSpec& spec) +{ + constexpr uint16_t udfType = 0; + return static_cast(udfType | + (static_cast(spec.reduceDataType) << 8U) | + (static_cast(spec.reduceOpCode) << 12U)); +} + +uint16_t TransferFlagSlot(const TileXRCcuMemTransferSpec& spec) +{ + return static_cast( + (spec.clearWait ? 1U : 0U) | + (spec.lengthFromXn ? 2U : 0U) | + (spec.reduceEnabled ? 4U : 0U)); +} + +int ValidateTransferSpec(const TileXRCcuMemTransferSpec& spec) +{ + if (spec.localGsa == 0 || spec.localXn == 0 || spec.remoteGsa == 0 || spec.remoteXn == 0 || + spec.lengthXn == 0 || spec.channelId == 0 || spec.reduceDataType > 0xfU || spec.reduceOpCode > 0xfU) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if ((spec.setCkeId == 0) != (spec.setCkeMask == 0) || + (spec.waitCkeId == 0) != (spec.waitCkeMask == 0)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +void WriteLe16(uint8_t* bytes, size_t offset, uint16_t value) +{ + bytes[offset] = static_cast(value & 0xffU); + bytes[offset + 1U] = static_cast((value >> 8U) & 0xffU); +} + +void WriteLe64(uint8_t* bytes, size_t offset, uint64_t value) +{ + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + bytes[offset + i] = static_cast((value >> (i * 8U)) & 0xffU); + } +} + +uint64_t ReadLe64(const uint8_t* bytes, size_t offset) +{ + uint64_t value = 0; + for (size_t i = 0; i < sizeof(uint64_t); ++i) { + value |= static_cast(bytes[offset + i]) << (i * 8U); + } + return value; +} + +} // namespace + +int TileXRCcuEncodeLoadSqeArgsToX(uint16_t xnId, uint32_t sqeArgId, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (xnId == 0 || sqeArgId >= TILEXR_CCU_SQE_ARGS_LEN) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots(static_cast(TILEXR_CCU_LOAD_SQE_ARGS_TO_X_HEADER), xnId, + static_cast(sqeArgId), 0); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeLoadImdToXn(uint16_t xnId, uint64_t immediate, uint16_t secFlag, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (xnId == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + uint8_t bytes[sizeof(TileXRCcuInstr)] = {}; + WriteLe16(bytes, 0, static_cast(TILEXR_CCU_LOAD_IMD_TO_XN_HEADER)); + WriteLe16(bytes, 2, xnId); + WriteLe64(bytes, 4, immediate); + WriteLe16(bytes, 12, secFlag); + for (size_t i = 0; i < 4U; ++i) { + instr->words[i] = ReadLe64(bytes, i * sizeof(uint64_t)); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeLoadImdToGsa(uint16_t gsaId, uint64_t immediate, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (gsaId == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + uint8_t bytes[sizeof(TileXRCcuInstr)] = {}; + WriteLe16(bytes, 0, static_cast(TILEXR_CCU_LOAD_IMD_TO_GSA_HEADER)); + WriteLe16(bytes, 2, gsaId); + WriteLe64(bytes, 4, immediate); + for (size_t i = 0; i < 4U; ++i) { + instr->words[i] = ReadLe64(bytes, i * sizeof(uint64_t)); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeSyncXn(const TileXRCcuSyncXnSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (spec.remoteXn == 0 || spec.localXn == 0 || spec.channelId == 0 || spec.notifyCke == 0 || + spec.notifyMask == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots(static_cast(TILEXR_CCU_SYNC_XN_HEADER), spec.remoteXn, spec.localXn, 0); + instr->words[1] = PackSlots(spec.channelId, spec.notifyCke, spec.notifyMask, 0); + instr->words[2] = spec.clearWait ? TILEXR_CCU_SYNC_XN_TRACE_FLAG : 0; + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeSyncCke(const TileXRCcuSyncCkeSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (spec.remoteCke == 0 || spec.localCke == 0 || spec.localCkeMask == 0 || spec.channelId == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_SYNC_CKE_HEADER), + spec.remoteCke, + spec.localCke, + spec.localCkeMask); + instr->words[1] = PackSlots(spec.channelId, 0, 0, 0); + instr->words[2] = PackSlots(0, 0, 0, ClearTypeBit(spec.clearWait)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeSetCke(const TileXRCcuCkeSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if ((spec.ckeId == 0 || spec.mask == 0) && (spec.waitCkeId == 0 || spec.waitMask == 0)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots(static_cast(TILEXR_CCU_SET_CKE_HEADER), ClearTypeBit(spec.clearWait), + spec.ckeId, spec.mask); + instr->words[1] = PackSlots(spec.waitCkeId, spec.waitMask, 0, 0); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeClearCke(const TileXRCcuCkeSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if ((spec.ckeId == 0 || spec.mask == 0) && (spec.waitCkeId == 0 || spec.waitMask == 0)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots(static_cast(TILEXR_CCU_CLEAR_CKE_HEADER), ClearTypeBit(spec.clearWait), + spec.ckeId, spec.mask); + instr->words[1] = PackSlots(spec.waitCkeId, spec.waitMask, 0, 0); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeTransRmtMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_RMT_MEM_TO_LOC_MEM_HEADER), + spec.localGsa, + spec.localXn, + spec.remoteGsa); + instr->words[1] = PackSlots(spec.remoteXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); + instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MEM_TO_RMT_MEM_HEADER), + spec.remoteGsa, + spec.remoteXn, + spec.localGsa); + instr->words[1] = PackSlots(spec.localXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); + instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildSqeLoadProgram(uint16_t firstXnId, uint32_t argCount, std::vector* program) +{ + if (program == nullptr || firstXnId == 0 || argCount == 0 || argCount > TILEXR_CCU_SQE_ARGS_LEN) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + program->clear(); + program->reserve(argCount); + for (uint32_t argId = 0; argId < argCount; ++argId) { + TileXRCcuInstr instr; + const uint32_t xnId = static_cast(firstXnId) + argId; + if (xnId > UINT16_MAX || TileXRCcuEncodeLoadSqeArgsToX(static_cast(xnId), argId, &instr) != + TILEXR_SUCCESS) { + program->clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program->push_back(instr); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildSyncProgram(const std::vector& specs, std::vector* program) +{ + if (program == nullptr || specs.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + program->clear(); + program->reserve(specs.size()); + for (const auto& spec : specs) { + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncXn(spec, &instr) != TILEXR_SUCCESS) { + program->clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program->push_back(instr); + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_microcode.h b/src/comm/ccu/tilexr_ccu_microcode.h new file mode 100644 index 00000000..fe8c0328 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_microcode.h @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_MICROCODE_H +#define TILEXR_CCU_MICROCODE_H + +#include "ccu/tilexr_ccu_runtime.h" + +#include +#include + +namespace TileXR { + +struct TileXRCcuInstr { + uint64_t words[4] = {}; +}; + +struct TileXRCcuSyncXnSpec { + uint16_t remoteXn = 0; + uint16_t localXn = 0; + uint16_t channelId = 0; + uint16_t notifyCke = 0; + uint16_t notifyMask = 0; + uint16_t setCkeId = 0; + uint16_t setCkeMask = 0; + uint16_t waitCkeId = 0; + uint16_t waitCkeMask = 0; + bool clearWait = true; +}; + +struct TileXRCcuSyncCkeSpec { + uint16_t remoteCke = 0; + uint16_t localCke = 0; + uint16_t localCkeMask = 0; + uint16_t channelId = 0; + uint16_t setCkeId = 0; + uint16_t setCkeMask = 0; + uint16_t waitCkeId = 0; + uint16_t waitCkeMask = 0; + bool clearWait = true; +}; + +struct TileXRCcuCkeSpec { + uint16_t ckeId = 0; + uint16_t mask = 0; + uint16_t waitCkeId = 0; + uint16_t waitMask = 0; + bool clearWait = true; +}; + +struct TileXRCcuMemTransferSpec { + uint16_t localGsa = 0; + uint16_t localXn = 0; + uint16_t remoteGsa = 0; + uint16_t remoteXn = 0; + uint16_t lengthXn = 0; + uint16_t channelId = 0; + uint16_t reduceDataType = 0; + uint16_t reduceOpCode = 0; + uint16_t setCkeId = 0; + uint16_t setCkeMask = 0; + uint16_t waitCkeId = 0; + uint16_t waitCkeMask = 0; + bool clearWait = true; + bool lengthFromXn = true; + bool reduceEnabled = false; +}; + +int TileXRCcuEncodeLoadSqeArgsToX(uint16_t xnId, uint32_t sqeArgId, TileXRCcuInstr* instr); + +int TileXRCcuEncodeLoadImdToXn(uint16_t xnId, uint64_t immediate, uint16_t secFlag, TileXRCcuInstr* instr); + +int TileXRCcuEncodeLoadImdToGsa(uint16_t gsaId, uint64_t immediate, TileXRCcuInstr* instr); + +int TileXRCcuEncodeSyncXn(const TileXRCcuSyncXnSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeSyncCke(const TileXRCcuSyncCkeSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeSetCke(const TileXRCcuCkeSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeClearCke(const TileXRCcuCkeSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeTransRmtMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuBuildSqeLoadProgram( + uint16_t firstXnId, + uint32_t argCount, + std::vector* program); + +int TileXRCcuBuildSyncProgram( + const std::vector& specs, + std::vector* program); + +} // namespace TileXR + +#endif // TILEXR_CCU_MICROCODE_H diff --git a/src/comm/ccu/tilexr_ccu_producer_plan.cpp b/src/comm/ccu/tilexr_ccu_producer_plan.cpp new file mode 100644 index 00000000..d87cd503 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_producer_plan.cpp @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_producer_plan.h" + +#include +#include +#include + +namespace TileXR { +namespace { + +constexpr uint32_t TILEXR_CCU_HCOMM_TASK1_PRELUDE_LOAD_ARG_COUNT = 2U; + +void ResetReport(TileXRCcuProducerPlanReport* report) +{ + if (report == nullptr) { + return; + } + report->syncResourceCount = 0; + report->taskCount = 0; + report->instructionCount = 0; + report->message.clear(); +} + +int Fail(TileXRCcuProducerPlanReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool ContainsId(const TileXRCcuRange& range, uint8_t dieId, uint16_t id) +{ + if (range.dieId != dieId || range.num == 0) { + return false; + } + const uint32_t begin = range.startId; + const uint32_t end = begin + range.num; + return id >= begin && id < end; +} + +bool InstructionWindowContains(const TileXRCcuInstructionWindow& window, uint8_t dieId, uint16_t start, uint16_t count) +{ + if (window.dieId != dieId || window.missionCount == 0 || count == 0) { + return false; + } + const uint32_t begin = window.missionStartId; + const uint32_t end = begin + window.missionCount; + const uint32_t taskBegin = start; + const uint32_t taskEnd = taskBegin + count; + return taskBegin >= begin && taskEnd <= end; +} + +uint16_t EffectiveLocalWaitCke(const TileXRCcuSyncResource& resource) +{ + return resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; +} + +uint16_t EffectiveRemoteNotifyMask(const TileXRCcuSyncResource& resource) +{ + return resource.remoteNotifyMask == 0 ? 1U : resource.remoteNotifyMask; +} + +uint16_t EffectiveLocalWaitMask(const TileXRCcuSyncResource& resource) +{ + return resource.localWaitMask == 0 ? 1U : resource.localWaitMask; +} + +uint16_t EffectiveSourceCkeMask(const TileXRCcuSyncResource& resource) +{ + return resource.sourceCkeMask == 0 ? 0xffffU : resource.sourceCkeMask; +} + +bool SyncCkeMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncCke || + mode == TileXRCcuBarrierMode::SyncCkeSetWait || + mode == TileXRCcuBarrierMode::SyncCkePostOnly; +} + +bool SyncXnMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXn || + mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +bool RequiresHcommStyleTask1Prelude(const TileXRCcuProducerPlan& plan) +{ + return plan.taskWindows.size() > 1 && SyncXnMode(plan.barrierMode); +} + +uint32_t SqeLoadXnOffset(uint32_t argId) +{ + return argId; +} + +uint32_t HcommStyleTask1PreludeLoadXnOffset(uint32_t argId) +{ + return argId; +} + +uint16_t HcommStylePreludeReserveXn(const TileXRCcuProducerPlan& plan) +{ + return static_cast( + static_cast(plan.kernelLocalXn.startId) + TILEXR_CCU_SQE_ARGS_LEN); +} + +bool HasKernelLocalGsa(const TileXRCcuProducerPlan& plan) +{ + return plan.kernelLocalGsa.dieId == plan.mission.dieId && plan.kernelLocalGsa.startId != 0 && + plan.kernelLocalGsa.num != 0; +} + +int AppendSqeLoadProgram( + const TileXRCcuProducerPlan& plan, + uint32_t argCount, + std::vector* program, + TileXRCcuProducerPlanReport* report) +{ + if (program == nullptr || plan.kernelLocalXn.startId == 0 || argCount == 0 || argCount > TILEXR_CCU_SQE_ARGS_LEN) { + return Fail(report, "invalid SQE argument load microcode request"); + } + + program->clear(); + program->reserve(argCount); + for (uint32_t argId = 0; argId < argCount; ++argId) { + const uint32_t xnId = static_cast(plan.kernelLocalXn.startId) + SqeLoadXnOffset(argId); + if (xnId > std::numeric_limits::max() || + !ContainsId(plan.kernelLocalXn, plan.mission.dieId, static_cast(xnId))) { + program->clear(); + return Fail(report, "SQE argument load XN is outside the kernel-local XN repository range"); + } + + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadSqeArgsToX(static_cast(xnId), argId, &instr) != TILEXR_SUCCESS) { + program->clear(); + return Fail(report, "failed to encode SQE argument load microcode"); + } + program->push_back(instr); + } + return TILEXR_SUCCESS; +} + +int ValidateMission(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (!plan.mission.installed || plan.mission.key == 0) { + return Fail(report, "missing installed mission/key for CCU producer plan"); + } + if (!ContainsId(plan.kernelLocalMission, plan.mission.dieId, plan.mission.missionId)) { + return Fail(report, "mission id is outside the kernel-local mission repository range"); + } + return TILEXR_SUCCESS; +} + +int ValidateInstructionWindow(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (plan.instructionWindow.repositoryCount == 0 || plan.instructionWindow.missionCount == 0) { + return Fail(report, "missing instruction repository window for CCU producer plan"); + } + if (plan.instructionWindow.dieId != plan.mission.dieId) { + return Fail(report, "instruction repository die does not match mission die"); + } + const uint32_t repositoryEnd = plan.instructionWindow.repositoryStartId + plan.instructionWindow.repositoryCount; + const uint32_t missionEnd = plan.instructionWindow.missionStartId + plan.instructionWindow.missionCount; + if (plan.instructionWindow.missionStartId < plan.instructionWindow.repositoryStartId || missionEnd > repositoryEnd) { + return Fail(report, "mission instruction window is outside the instruction repository range"); + } + return TILEXR_SUCCESS; +} + +int ValidateKernelLocalRepositories(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (plan.kernelLocalXn.dieId != plan.mission.dieId || plan.kernelLocalXn.num == 0) { + return Fail(report, "missing kernel-local XN repository range"); + } + if (plan.kernelLocalCke.dieId != plan.mission.dieId || plan.kernelLocalCke.num == 0) { + return Fail(report, "missing kernel-local CKE repository range"); + } + return TILEXR_SUCCESS; +} + +int ValidateSyncResources(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (plan.syncResources.empty()) { + return Fail(report, "missing CCU sync resources"); + } + std::set channelIds; + for (const auto& resource : plan.syncResources) { + if (resource.dieId != plan.mission.dieId) { + return Fail(report, "sync resource die does not match mission die"); + } + if (resource.channelId == 0) { + return Fail(report, "missing channel id for sync resource"); + } + if (!channelIds.insert(resource.channelId).second) { + return Fail(report, "duplicate channel id for sync resource"); + } + if (!ContainsId(plan.kernelLocalXn, resource.dieId, resource.localXn)) { + return Fail(report, "local XN is outside the kernel-local XN repository range"); + } + if (resource.remoteXn == 0) { + return Fail(report, "missing channel-bound remote XN"); + } + if (resource.notifyCke == 0) { + return Fail(report, "missing remote notify CKE resource"); + } + if (EffectiveRemoteNotifyMask(resource) == 0) { + return Fail(report, "missing remote notify CKE mask"); + } + const uint16_t localWaitCke = EffectiveLocalWaitCke(resource); + if (localWaitCke == 0) { + return Fail(report, "missing local wait CKE resource"); + } + if (EffectiveLocalWaitMask(resource) == 0) { + return Fail(report, "missing local wait CKE mask"); + } + if (resource.localWaitCke != 0 && !ContainsId(plan.kernelLocalCke, resource.dieId, localWaitCke)) { + return Fail(report, "local wait CKE is outside the kernel-local CKE repository range"); + } + if (SyncCkeMode(plan.barrierMode)) { + if (resource.sourceCke == 0) { + return Fail(report, "missing source CKE resource for SyncCKE barrier"); + } + if (EffectiveSourceCkeMask(resource) == 0) { + return Fail(report, "missing source CKE mask for SyncCKE barrier"); + } + if (!ContainsId(plan.kernelLocalCke, resource.dieId, resource.sourceCke)) { + return Fail(report, "source CKE is outside the kernel-local CKE repository range"); + } + } + if (resource.bindingCount == 0) { + return Fail(report, "missing channel variable binding for sync resource"); + } + } + return TILEXR_SUCCESS; +} + +int ValidateTasks(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (plan.taskWindows.empty()) { + return Fail(report, "missing CCU task windows"); + } + for (const auto& window : plan.taskWindows) { + if (window.dieId != plan.mission.dieId) { + return Fail(report, "task die does not match mission die"); + } + if (!InstructionWindowContains(plan.instructionWindow, window.dieId, window.instStartId, window.instCnt)) { + return Fail(report, "task instruction range is outside the loaded mission instruction window"); + } + if (window.argSize != 1 && window.argSize != TILEXR_CCU_SQE_ARGS_LEN) { + return Fail(report, "task argSize must match a supported CCU SQE payload shape"); + } + if (window.args.size() > TILEXR_CCU_SQE_ARGS_LEN) { + return Fail(report, "task args exceed the CCU SQE payload capacity"); + } + } + return TILEXR_SUCCESS; +} + +void FillReport(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + if (report == nullptr) { + return; + } + report->syncResourceCount = static_cast(plan.syncResources.size()); + report->taskCount = static_cast(plan.taskWindows.size()); + report->instructionCount = plan.instructionWindow.repositoryCount; + report->message = "ok"; +} + +int AppendHcommStyleTask1Prelude( + const TileXRCcuProducerPlan& plan, + std::vector* program, + TileXRCcuProducerPlanReport* report) +{ + if (!RequiresHcommStyleTask1Prelude(plan)) { + return TILEXR_SUCCESS; + } + + const uint16_t reserveXn = HcommStylePreludeReserveXn(plan); + if (!ContainsId(plan.kernelLocalXn, plan.mission.dieId, reserveXn)) { + return Fail(report, "missing reserve XN for hcomm-style task1 prelude"); + } + + for (uint32_t argId = 0; argId < TILEXR_CCU_HCOMM_TASK1_PRELUDE_LOAD_ARG_COUNT; ++argId) { + const uint32_t xnId = + static_cast(plan.kernelLocalXn.startId) + HcommStyleTask1PreludeLoadXnOffset(argId); + if (xnId > std::numeric_limits::max() || + !ContainsId(plan.kernelLocalXn, plan.mission.dieId, static_cast(xnId))) { + return Fail(report, "missing load-arg XN for hcomm-style task1 prelude"); + } + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadSqeArgsToX(static_cast(xnId), argId, &instr) != TILEXR_SUCCESS) { + return Fail(report, "failed to encode hcomm-style task1 load-arg prelude"); + } + program->push_back(instr); + } + + TileXRCcuInstr reserveXnInstr; + if (TileXRCcuEncodeLoadImdToXn(reserveXn, 0, 0, &reserveXnInstr) != TILEXR_SUCCESS) { + return Fail(report, "failed to encode hcomm-style task1 reserve XN prelude"); + } + program->push_back(reserveXnInstr); + + TileXRCcuInstr reserveAddrInstr; + if (HasKernelLocalGsa(plan)) { + if (TileXRCcuEncodeLoadImdToGsa(plan.kernelLocalGsa.startId, 0, &reserveAddrInstr) != TILEXR_SUCCESS) { + return Fail(report, "failed to encode hcomm-style task1 reserve GSA prelude"); + } + } else if (TileXRCcuEncodeLoadImdToXn(reserveXn, 0, 0, &reserveAddrInstr) != TILEXR_SUCCESS) { + return Fail(report, "failed to encode hcomm-style task1 nop prelude"); + } + program->push_back(reserveAddrInstr); + + const TileXRCcuSyncResource& firstResource = plan.syncResources.front(); + TileXRCcuCkeSpec notifyInit; + notifyInit.ckeId = firstResource.notifyCke; + notifyInit.mask = EffectiveRemoteNotifyMask(firstResource); + notifyInit.clearWait = false; + TileXRCcuInstr notifyInstr; + if (TileXRCcuEncodeSetCke(notifyInit, ¬ifyInstr) != TILEXR_SUCCESS) { + return Fail(report, "failed to encode hcomm-style task1 notify CKE prelude"); + } + program->push_back(notifyInstr); + return TILEXR_SUCCESS; +} + +} // namespace + +int TileXRCcuValidateProducerPlan(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report) +{ + ResetReport(report); + + int ret = ValidateMission(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = ValidateInstructionWindow(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = ValidateKernelLocalRepositories(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = ValidateSyncResources(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = ValidateTasks(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + FillReport(plan, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildTasks( + const TileXRCcuProducerPlan& plan, + std::vector* tasks, + TileXRCcuProducerPlanReport* report) +{ + if (tasks == nullptr) { + ResetReport(report); + return Fail(report, "missing output task vector"); + } + + int ret = TileXRCcuValidateProducerPlan(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + tasks->clear(); + tasks->reserve(plan.taskWindows.size()); + for (const auto& window : plan.taskWindows) { + TileXRCcuTask task; + task.dieId = window.dieId; + task.missionId = plan.mission.missionId; + task.timeout = TILEXR_CCU_DEFAULT_TASK_TIMEOUT_SEC; + task.instStartId = window.instStartId; + task.instCnt = window.instCnt; + task.key = plan.mission.key; + task.argSize = window.argSize; + std::copy(window.args.begin(), window.args.end(), task.args); + tasks->push_back(task); + } + + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildMicrocode( + const TileXRCcuProducerPlan& plan, + TileXRCcuProgram* program, + TileXRCcuProducerPlanReport* report) +{ + if (program == nullptr) { + ResetReport(report); + return Fail(report, "missing output CCU program"); + } + + int ret = TileXRCcuValidateProducerPlan(plan, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + program->sqeLoad.clear(); + program->sync.clear(); + if (plan.taskWindows.size() > 1) { + if (AppendSqeLoadProgram(plan, TILEXR_CCU_SQE_ARGS_LEN, &program->sqeLoad, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + + std::vector barrierSpecs; + barrierSpecs.reserve(plan.syncResources.size()); + for (const auto& resource : plan.syncResources) { + TileXRCcuBarrierSyncSpec spec; + spec.remoteXn = resource.remoteXn; + spec.localXn = resource.localXn; + spec.channelId = resource.channelId; + spec.remoteNotifyCke = resource.notifyCke; + spec.remoteNotifyMask = EffectiveRemoteNotifyMask(resource); + spec.localWaitCke = EffectiveLocalWaitCke(resource); + spec.localWaitMask = EffectiveLocalWaitMask(resource); + spec.sourceCke = resource.sourceCke; + spec.sourceCkeMask = EffectiveSourceCkeMask(resource); + spec.clearLocalWait = true; + barrierSpecs.push_back(spec); + } + + TileXRCcuBarrierProgramReport barrierReport; + if (AppendHcommStyleTask1Prelude(plan, &program->sync, report) != TILEXR_SUCCESS) { + program->sqeLoad.clear(); + program->sync.clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector barrierProgram; + if (TileXRCcuBuildBarrierProgram(barrierSpecs, &barrierProgram, &barrierReport, plan.barrierMode) != + TILEXR_SUCCESS) { + program->sqeLoad.clear(); + program->sync.clear(); + return Fail(report, barrierReport.message.empty() ? "failed to build sync microcode" : barrierReport.message); + } + program->sync.insert(program->sync.end(), barrierProgram.begin(), barrierProgram.end()); + + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_producer_plan.h b/src/comm/ccu/tilexr_ccu_producer_plan.h new file mode 100644 index 00000000..af4403c2 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_producer_plan.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_PRODUCER_PLAN_H +#define TILEXR_CCU_PRODUCER_PLAN_H + +#include "ccu/tilexr_ccu_barrier_program.h" +#include "ccu/tilexr_ccu_runtime.h" + +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuRange { + uint8_t dieId = 0; + uint16_t startId = 0; + uint16_t num = 0; +}; + +struct TileXRCcuMission { + uint8_t dieId = 0; + uint8_t missionId = 0; + uint32_t key = 0; + bool installed = false; +}; + +struct TileXRCcuInstructionWindow { + uint8_t dieId = 0; + uint16_t repositoryStartId = 0; + uint16_t repositoryCount = 0; + uint16_t missionStartId = 0; + uint16_t missionCount = 0; +}; + +struct TileXRCcuSyncResource { + uint8_t dieId = 0; + uint16_t localXn = 0; + uint16_t remoteXn = 0; + uint16_t notifyCke = 0; + uint16_t channelId = 0; + uint16_t bindingCount = 0; + uint16_t localWaitCke = 0; + uint16_t localWaitMask = 1; + uint16_t remoteNotifyMask = 1; + uint16_t sourceCke = 0; + uint16_t sourceCkeMask = 0xffff; +}; + +struct TileXRCcuTaskWindow { + uint8_t dieId = 0; + uint16_t instStartId = 0; + uint16_t instCnt = 0; + uint32_t argSize = 0; + std::vector args; +}; + +struct TileXRCcuProducerPlan { + TileXRCcuMission mission; + TileXRCcuRange kernelLocalXn; + TileXRCcuRange kernelLocalGsa; + TileXRCcuRange kernelLocalCke; + TileXRCcuRange kernelLocalMission; + TileXRCcuInstructionWindow instructionWindow; + std::vector syncResources; + std::vector taskWindows; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; +}; + +struct TileXRCcuProgram { + std::vector sqeLoad; + std::vector sync; +}; + +struct TileXRCcuProducerPlanReport { + uint32_t syncResourceCount = 0; + uint32_t taskCount = 0; + uint32_t instructionCount = 0; + std::string message; +}; + +int TileXRCcuValidateProducerPlan(const TileXRCcuProducerPlan& plan, TileXRCcuProducerPlanReport* report); + +int TileXRCcuBuildTasks( + const TileXRCcuProducerPlan& plan, + std::vector* tasks, + TileXRCcuProducerPlanReport* report); + +int TileXRCcuBuildMicrocode( + const TileXRCcuProducerPlan& plan, + TileXRCcuProgram* program, + TileXRCcuProducerPlanReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_PRODUCER_PLAN_H diff --git a/src/comm/ccu/tilexr_ccu_provider.cpp b/src/comm/ccu/tilexr_ccu_provider.cpp new file mode 100644 index 00000000..d42265b6 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_provider.cpp @@ -0,0 +1,401 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_provider.h" + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuProviderReport* report) +{ + if (report == nullptr) { + return; + } + report->taskCount = 0; + report->repositoryCount = 0; + report->installedInstructionCount = 0; + report->evidenceBitCount = 0; + report->publicVerifiedEvidenceCount = 0; + report->legacyEvidenceCount = 0; + report->publicCandidateEvidenceCount = 0; + report->privateObservedEvidenceCount = 0; + report->missingEvidenceCount = 0; + report->submitReady = false; + report->message.clear(); +} + +int Fail(TileXRCcuProviderReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +const char* EvidenceKindName(TileXRCcuEvidenceKind kind) +{ + switch (kind) { + case TileXRCcuEvidenceKind::LegacyBoolean: + return "legacy"; + case TileXRCcuEvidenceKind::PublicVerified: + return "public verified"; + case TileXRCcuEvidenceKind::PublicCandidate: + return "public candidate"; + case TileXRCcuEvidenceKind::PrivateObserved: + return "private observed"; + case TileXRCcuEvidenceKind::Missing: + return "missing"; + default: + return "unknown"; + } +} + +const char* EvidenceSurfaceName(TileXRCcuEvidenceSurface surface) +{ + switch (surface) { + case TileXRCcuEvidenceSurface::Unspecified: + return "unspecified surface"; + case TileXRCcuEvidenceSurface::PublicInstallProvider: + return "public install provider"; + case TileXRCcuEvidenceSurface::LowerLayerResourceHelper: + return "lower-layer resource helper"; + case TileXRCcuEvidenceSurface::PrivateProducerObservation: + return "private producer observation"; + default: + return "unknown surface"; + } +} + +std::string EvidenceMessage( + const std::string& label, + const TileXRCcuEvidenceSource& source, + const std::string& suffix) +{ + std::string message = label + " " + suffix; + if (source.kind != TileXRCcuEvidenceKind::LegacyBoolean) { + message += " (" + std::string(EvidenceKindName(source.kind)); + message += ", " + std::string(EvidenceSurfaceName(source.surface)); + if (!source.source.empty()) { + message += ": " + source.source; + } + if (!source.detail.empty()) { + message += "; " + source.detail; + } + message += ")"; + } + return message; +} + +int ValidateEvidenceSource( + const std::string& label, + bool installed, + const TileXRCcuEvidenceSource& source, + TileXRCcuProviderReport* report) +{ + if (!installed || source.kind == TileXRCcuEvidenceKind::Missing) { + return Fail(report, EvidenceMessage(label, source, "hardware install evidence is missing")); + } + if (source.kind == TileXRCcuEvidenceKind::LegacyBoolean || + source.kind == TileXRCcuEvidenceKind::PublicVerified) { + return TILEXR_SUCCESS; + } + if (source.kind == TileXRCcuEvidenceKind::PublicCandidate) { + return Fail(report, EvidenceMessage(label, source, "evidence is only a public candidate")); + } + if (source.kind == TileXRCcuEvidenceKind::PrivateObserved) { + return Fail(report, EvidenceMessage(label, source, "evidence is private observed")); + } + return Fail(report, EvidenceMessage(label, source, "evidence kind is not accepted")); +} + +void CountEvidenceKind(bool installed, const TileXRCcuEvidenceSource& source, TileXRCcuProviderReport* report) +{ + if (report == nullptr) { + return; + } + ++report->evidenceBitCount; + if (!installed || source.kind == TileXRCcuEvidenceKind::Missing) { + ++report->missingEvidenceCount; + return; + } + switch (source.kind) { + case TileXRCcuEvidenceKind::LegacyBoolean: + ++report->legacyEvidenceCount; + break; + case TileXRCcuEvidenceKind::PublicVerified: + ++report->publicVerifiedEvidenceCount; + break; + case TileXRCcuEvidenceKind::PublicCandidate: + ++report->publicCandidateEvidenceCount; + break; + case TileXRCcuEvidenceKind::PrivateObserved: + ++report->privateObservedEvidenceCount; + break; + case TileXRCcuEvidenceKind::Missing: + ++report->missingEvidenceCount; + break; + default: + ++report->missingEvidenceCount; + break; + } +} + +void CountInstallEvidence(const TileXRCcuHardwareInstallEvidence& evidence, TileXRCcuProviderReport* report) +{ + CountEvidenceKind(evidence.missionInstalled, evidence.missionSource, report); + CountEvidenceKind(evidence.repositoryInstalled, evidence.repositorySource, report); + CountEvidenceKind(evidence.localXnInstalled, evidence.localXnSource, report); + CountEvidenceKind(evidence.remoteXnBound, evidence.remoteXnSource, report); + CountEvidenceKind(evidence.notifyCkeInstalled, evidence.notifyCkeSource, report); + CountEvidenceKind(evidence.channelBindingsInstalled, evidence.channelBindingSource, report); +} + +void FillReadyReport(const TileXRCcuLaunchPackage& package, TileXRCcuProviderReport* report) +{ + if (report == nullptr) { + return; + } + report->taskCount = static_cast(package.tasks.size()); + report->repositoryCount = package.repository.repositoryCount; + report->installedInstructionCount = + static_cast(package.repository.sqeLoadCount + package.repository.syncCount); + report->submitReady = true; + report->message = "ok"; +} + +int ValidatePackageShape(const TileXRCcuLaunchPackage& package, TileXRCcuProviderReport* report) +{ + if (package.tasks.empty()) { + return Fail(report, "missing CCU launch tasks in provider package"); + } + if (package.repository.instructions.empty()) { + return Fail(report, "missing CCU repository image in provider package"); + } + if (package.program.sync.empty()) { + return Fail(report, "missing generated CCU microcode in provider package"); + } + return TILEXR_SUCCESS; +} + +bool RequiresRemoteXnBindingEvidence(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXn || + mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +bool RequiresChannelBindingEvidence(TileXRCcuBarrierMode mode) +{ + return mode != TileXRCcuBarrierMode::LocalCke && + mode != TileXRCcuBarrierMode::LocalCkePostOnly; +} + +int ValidateInstallEvidence( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + TileXRCcuProviderReport* report) +{ + CountInstallEvidence(evidence, report); + if (ValidateEvidenceSource("mission/key", evidence.missionInstalled, evidence.missionSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateEvidenceSource("repository", evidence.repositoryInstalled, evidence.repositorySource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateEvidenceSource("local XN", evidence.localXnInstalled, evidence.localXnSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode) && + ValidateEvidenceSource("remote XN binding", evidence.remoteXnBound, evidence.remoteXnSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateEvidenceSource("notify CKE install", evidence.notifyCkeInstalled, evidence.notifyCkeSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresChannelBindingEvidence(package.plan.barrierMode) && + ValidateEvidenceSource( + "channel binding", evidence.channelBindingsInstalled, evidence.channelBindingSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int ValidateSubmitEvidenceSource( + const std::string& label, + const TileXRCcuLaunchPackage& package, + bool installed, + const TileXRCcuEvidenceSource& source, + bool endpointRouteRequired, + TileXRCcuProviderReport* report) +{ + if (!installed || source.kind != TileXRCcuEvidenceKind::PublicVerified) { + return Fail(report, EvidenceMessage(label, source, "submit requires public verified evidence")); + } + if (source.source.empty() || source.detail.empty()) { + return Fail(report, EvidenceMessage(label, source, "public verified evidence source/detail required")); + } + if (source.surface != TileXRCcuEvidenceSurface::PublicInstallProvider) { + return Fail(report, EvidenceMessage( + label, source, "submit requires public install provider evidence")); + } + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (package.installScope.packageFingerprint == 0 || + package.installScope.packageFingerprint != packageFingerprint || + package.installScope.provider.empty()) { + return Fail(report, EvidenceMessage(label, source, "launch install scope is not bound")); + } + if (source.packageFingerprint == 0 || source.packageFingerprint != packageFingerprint) { + return Fail(report, EvidenceMessage(label, source, "package fingerprint mismatch")); + } + if (source.deviceId != package.installScope.deviceId) { + return Fail(report, EvidenceMessage(label, source, "device scope mismatch")); + } + if (source.rank != package.installScope.rank) { + return Fail(report, EvidenceMessage(label, source, "rank scope mismatch")); + } + if (source.provider.empty() || source.provider != package.installScope.provider) { + return Fail(report, EvidenceMessage(label, source, "provider scope mismatch")); + } + if (source.installAttemptReceiptId == 0) { + return Fail(report, EvidenceMessage(label, source, "install attempt receipt is missing")); + } + if (endpointRouteRequired && !source.endpointRouteVerified) { + return Fail(report, EvidenceMessage(label, source, "submit requires verified endpoint route evidence")); + } + return TILEXR_SUCCESS; +} + +int ValidateSameInstallAttemptReceipt( + const TileXRCcuEvidenceSource& expected, + const std::string& label, + const TileXRCcuEvidenceSource& source, + TileXRCcuProviderReport* report) +{ + if (source.installAttemptReceiptId != expected.installAttemptReceiptId) { + return Fail(report, EvidenceMessage(label, source, "install attempt receipt mismatch")); + } + return TILEXR_SUCCESS; +} + +int ValidateSubmitEvidence( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + TileXRCcuProviderReport* report) +{ + if (ValidateSubmitEvidenceSource( + "mission/key", package, evidence.missionInstalled, evidence.missionSource, false, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateSubmitEvidenceSource( + "repository", package, evidence.repositoryInstalled, evidence.repositorySource, false, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateSubmitEvidenceSource( + "local XN", package, evidence.localXnInstalled, evidence.localXnSource, false, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode) && + ValidateSubmitEvidenceSource( + "remote XN binding", package, evidence.remoteXnBound, evidence.remoteXnSource, true, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateSubmitEvidenceSource( + "notify CKE install", package, evidence.notifyCkeInstalled, evidence.notifyCkeSource, false, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresChannelBindingEvidence(package.plan.barrierMode) && + ValidateSubmitEvidenceSource( + "channel binding", package, evidence.channelBindingsInstalled, evidence.channelBindingSource, true, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const TileXRCcuEvidenceSource& expected = evidence.missionSource; + if (ValidateSameInstallAttemptReceipt(expected, "repository", evidence.repositorySource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateSameInstallAttemptReceipt(expected, "local XN", evidence.localXnSource, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresRemoteXnBindingEvidence(package.plan.barrierMode) && + ValidateSameInstallAttemptReceipt(expected, "remote XN binding", evidence.remoteXnSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateSameInstallAttemptReceipt(expected, "notify CKE install", evidence.notifyCkeSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (RequiresChannelBindingEvidence(package.plan.barrierMode) && + ValidateSameInstallAttemptReceipt(expected, "channel binding", evidence.channelBindingSource, report) != + TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +} // namespace + +int TileXRCcuValidateHardwareInstall( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + TileXRCcuProviderReport* report) +{ + ResetReport(report); + if (ValidatePackageShape(package, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateInstallEvidence(package, evidence, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (report != nullptr) { + report->taskCount = static_cast(package.tasks.size()); + report->repositoryCount = package.repository.repositoryCount; + report->installedInstructionCount = + static_cast(package.repository.sqeLoadCount + package.repository.syncCount); + report->submitReady = false; + report->message = "hardware install evidence is validate-compatible"; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuPrepareSubmitTasks( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + std::vector* submitTasks, + TileXRCcuProviderReport* report) +{ + ResetReport(report); + if (submitTasks == nullptr) { + return Fail(report, "missing output submit task vector"); + } + submitTasks->clear(); + + if (TileXRCcuValidateHardwareInstall(package, evidence, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (report != nullptr) { + report->submitReady = false; + } + if (ValidateSubmitEvidence(package, evidence, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *submitTasks = package.tasks; + FillReadyReport(package, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_provider.h b/src/comm/ccu/tilexr_ccu_provider.h new file mode 100644 index 00000000..2a96436f --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_provider.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_PROVIDER_H +#define TILEXR_CCU_PROVIDER_H + +#include "ccu/tilexr_ccu_launch_package.h" + +#include +#include +#include + +namespace TileXR { + +enum class TileXRCcuEvidenceKind : uint8_t { + LegacyBoolean = 0, + PublicVerified = 1, + PublicCandidate = 2, + PrivateObserved = 3, + Missing = 4, +}; + +enum class TileXRCcuEvidenceSurface : uint8_t { + Unspecified = 0, + PublicInstallProvider = 1, + LowerLayerResourceHelper = 2, + PrivateProducerObservation = 3, +}; + +struct TileXRCcuEvidenceSource { + TileXRCcuEvidenceKind kind = TileXRCcuEvidenceKind::LegacyBoolean; + TileXRCcuEvidenceSurface surface = TileXRCcuEvidenceSurface::Unspecified; + uint64_t packageFingerprint = 0; + uint32_t deviceId = 0; + uint32_t rank = 0; + std::string provider; + uint64_t installAttemptReceiptId = 0; + bool endpointRouteVerified = false; + std::string source; + std::string detail; +}; + +struct TileXRCcuHardwareInstallEvidence { + bool missionInstalled = false; + bool repositoryInstalled = false; + bool localXnInstalled = false; + bool remoteXnBound = false; + bool notifyCkeInstalled = false; + bool channelBindingsInstalled = false; + TileXRCcuEvidenceSource missionSource; + TileXRCcuEvidenceSource repositorySource; + TileXRCcuEvidenceSource localXnSource; + TileXRCcuEvidenceSource remoteXnSource; + TileXRCcuEvidenceSource notifyCkeSource; + TileXRCcuEvidenceSource channelBindingSource; +}; + +struct TileXRCcuProviderReport { + uint32_t taskCount = 0; + uint32_t repositoryCount = 0; + uint32_t installedInstructionCount = 0; + uint32_t evidenceBitCount = 0; + uint32_t publicVerifiedEvidenceCount = 0; + uint32_t legacyEvidenceCount = 0; + uint32_t publicCandidateEvidenceCount = 0; + uint32_t privateObservedEvidenceCount = 0; + uint32_t missingEvidenceCount = 0; + bool submitReady = false; + std::string message; +}; + +int TileXRCcuValidateHardwareInstall( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + TileXRCcuProviderReport* report); + +int TileXRCcuPrepareSubmitTasks( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + std::vector* submitTasks, + TileXRCcuProviderReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_PROVIDER_H diff --git a/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp new file mode 100644 index 00000000..f364290d --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_ra_custom_channel_provider.h" + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuRaCustomChannelProviderReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuRaCustomChannelProviderReport{}; + } +} + +int Fail(TileXRCcuRaCustomChannelProviderReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +void FillReport( + uint32_t devicePhyId, + bool initialized, + const std::string& message, + TileXRCcuRaCustomChannelProviderReport* report) +{ + if (report == nullptr) { + return; + } + report->devicePhyId = devicePhyId; + report->initialized = initialized; + report->message = message; +} + +} // namespace + +int TileXRCcuRaCustomChannelProvider::Init( + uint32_t devicePhyId, + TileXRCcuRaCustomChannelFunc raCustomChannel, + TileXRCcuRaCustomChannelProviderReport* report) +{ + TileXRCcuRaCustomChannelInvoker invoker; + if (raCustomChannel != nullptr) { + invoker = + [raCustomChannel]( + TileXRCcuRaInfo info, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out) -> int { + return raCustomChannel( + info, + const_cast(&in), + out); + }; + } + return InitCallable(devicePhyId, invoker, report); +} + +int TileXRCcuRaCustomChannelProvider::InitCallable( + uint32_t devicePhyId, + TileXRCcuRaCustomChannelInvoker raCustomChannel, + TileXRCcuRaCustomChannelProviderReport* report) +{ + ResetReport(report); + if (!raCustomChannel) { + initialized_ = false; + return Fail(report, "missing RA custom channel function"); + } + devicePhyId_ = devicePhyId; + raCustomChannel_ = raCustomChannel; + initialized_ = true; + FillReport(devicePhyId_, initialized_, "ok", report); + return TILEXR_SUCCESS; +} + +int TileXRCcuRaCustomChannelProvider::CreateAdapter( + TileXRCcuDriverAdapter* adapter, + TileXRCcuDriverAdapterReport* report) +{ + if (adapter == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!initialized_ || !raCustomChannel_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + return adapter->Init(devicePhyId_, &TileXRCcuRaCustomChannelProvider::AdapterCallback, this, report); +} + +int TileXRCcuRaCustomChannelProvider::AdapterCallback( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) +{ + auto* provider = static_cast(userData); + if (provider == nullptr || !provider->raCustomChannel_ || out == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRCcuRaInfo info {}; + info.mode = TILEXR_CCU_NETWORK_OFFLINE; + info.phyId = devicePhyId; + return provider->raCustomChannel_( + info, + in, + out); +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.h b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.h new file mode 100644 index 00000000..48f9984d --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_RA_CUSTOM_CHANNEL_PROVIDER_H +#define TILEXR_CCU_RA_CUSTOM_CHANNEL_PROVIDER_H + +#include "ccu/tilexr_ccu_driver_adapter.h" +#include "ccu/tilexr_ccu_hccp_types.h" + +#include +#include +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuRaCustomChannelProviderReport { + uint32_t devicePhyId = 0; + bool initialized = false; + std::string message; +}; + +class TileXRCcuRaCustomChannelProvider { +private: + using TileXRCcuRaCustomChannelInvoker = std::function; + +public: + int Init( + uint32_t devicePhyId, + TileXRCcuRaCustomChannelFunc raCustomChannel, + TileXRCcuRaCustomChannelProviderReport* report); + + template + int Init( + uint32_t devicePhyId, + int (*raCustomChannel)(RaInfoT, void*, void*), + TileXRCcuRaCustomChannelProviderReport* report) + { + if (raCustomChannel == nullptr) { + return InitCallable(devicePhyId, TileXRCcuRaCustomChannelInvoker {}, report); + } + TileXRCcuRaCustomChannelInvoker invoker = + [raCustomChannel]( + TileXRCcuRaInfo info, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out) -> int { + RaInfoT compatInfo {}; + const size_t copyBytes = sizeof(compatInfo) < sizeof(info) ? sizeof(compatInfo) : sizeof(info); + std::memcpy(&compatInfo, &info, copyBytes); + return raCustomChannel(compatInfo, const_cast(&in), out); + }; + return InitCallable(devicePhyId, invoker, report); + } + + int CreateAdapter(TileXRCcuDriverAdapter* adapter, TileXRCcuDriverAdapterReport* report); + +private: + int InitCallable( + uint32_t devicePhyId, + TileXRCcuRaCustomChannelInvoker raCustomChannel, + TileXRCcuRaCustomChannelProviderReport* report); + + static int AdapterCallback( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData); + + uint32_t devicePhyId_ = 0; + TileXRCcuRaCustomChannelInvoker raCustomChannel_; + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_RA_CUSTOM_CHANNEL_PROVIDER_H diff --git a/src/comm/ccu/tilexr_ccu_repository.cpp b/src/comm/ccu/tilexr_ccu_repository.cpp new file mode 100644 index 00000000..4b9b6bcf --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_repository.cpp @@ -0,0 +1,894 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_repository.h" + +#if defined(__has_include) +#if __has_include() +#define TILEXR_CCU_HAVE_ACL_RT_HEADER 1 +#include +#else +#define TILEXR_CCU_HAVE_ACL_RT_HEADER 0 +#endif +#else +#define TILEXR_CCU_HAVE_ACL_RT_HEADER 1 +#include +#endif + +#if defined(__has_include) +#if __has_include() +#define TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER 1 +#include +#else +#define TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER 0 +#endif +#else +#define TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER 1 +#include +#endif + +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +constexpr uint16_t TILEXR_CCU_ACL_MODULE3_ID = 3U; +constexpr uint16_t TILEXR_CCU_RT_HBM_MODULE_ID = 0U; + +struct TileXRCcuUploadReadbackDiagnostic { + bool attempted = false; + bool ok = false; + int ret = 0; + uint64_t bytes = 0; + uint64_t fnv1a64 = 0; + uint32_t mismatchCount = 0; + std::string firstInstructionWords; + std::string lastInstructionWords; +}; + +void ResetReport(TileXRCcuRepositoryReport* report) +{ + if (report == nullptr) { + return; + } + report->repositoryCount = 0; + report->installedInstructionCount = 0; + report->sqeLoadOffset = 0; + report->syncOffset = 0; + report->repositoryUploaded = false; + report->repositoryInstalled = false; + report->message.clear(); +} + +int Fail( + TileXRCcuRepositoryReport* report, + const std::string& message, + int code = TILEXR_ERROR_PARA_CHECK_FAIL) +{ + if (report != nullptr) { + report->message = message; + } + return code; +} + +bool ContainsWindow(uint16_t outerStart, uint16_t outerCount, uint16_t innerStart, uint16_t innerCount) +{ + if (outerCount == 0 || innerCount == 0) { + return false; + } + const uint32_t outerBegin = outerStart; + const uint32_t outerEnd = outerBegin + outerCount; + const uint32_t innerBegin = innerStart; + const uint32_t innerEnd = innerBegin + innerCount; + return innerBegin >= outerBegin && innerEnd <= outerEnd; +} + +uint16_t OffsetFrom(uint16_t base, uint16_t id) +{ + return static_cast(static_cast(id) - base); +} + +int ValidateProgramShape( + const TileXRCcuProducerPlan& plan, + const TileXRCcuProgram& program, + TileXRCcuRepositoryReport* report) +{ + if (program.sync.empty()) { + return Fail(report, "missing sync microcode for repository image"); + } + if (program.sqeLoad.empty()) { + if (plan.taskWindows.size() != 1) { + return Fail(report, "pure sync repository image requires exactly one task window"); + } + const auto& syncTask = plan.taskWindows[0]; + if (syncTask.instCnt < program.sync.size()) { + return Fail(report, "sync task window is too small for generated sync microcode"); + } + return TILEXR_SUCCESS; + } + if (plan.taskWindows.size() < 2) { + return Fail(report, "repository image requires SQE and sync task windows"); + } + + const auto& sqeTask = plan.taskWindows[0]; + const auto& syncTask = plan.taskWindows[1]; + if (sqeTask.instCnt != program.sqeLoad.size()) { + return Fail(report, "SQE task window does not match generated SQE microcode"); + } + if (syncTask.instStartId != static_cast(sqeTask.instStartId) + sqeTask.instCnt) { + return Fail(report, "sync task window must start immediately after SQE microcode"); + } + if (syncTask.instCnt < program.sync.size()) { + return Fail(report, "sync task window is too small for generated sync microcode"); + } + return TILEXR_SUCCESS; +} + +int ValidateRepositoryWindow(const TileXRCcuProducerPlan& plan, TileXRCcuRepositoryReport* report) +{ + const auto& window = plan.instructionWindow; + if (!ContainsWindow( + window.repositoryStartId, window.repositoryCount, window.missionStartId, window.missionCount)) { + return Fail(report, "mission instruction window is outside the repository image"); + } + for (const auto& task : plan.taskWindows) { + if (task.dieId != window.dieId) { + return Fail(report, "task die does not match repository image die"); + } + if (!ContainsWindow(window.missionStartId, window.missionCount, task.instStartId, task.instCnt)) { + return Fail(report, "task instruction window is outside the mission image"); + } + if (!ContainsWindow(window.repositoryStartId, window.repositoryCount, task.instStartId, task.instCnt)) { + return Fail(report, "task instruction window is outside the repository image"); + } + } + return TILEXR_SUCCESS; +} + +void FillImageHeader(const TileXRCcuInstructionWindow& window, TileXRCcuRepositoryImage* image) +{ + image->dieId = window.dieId; + image->repositoryStartId = window.repositoryStartId; + image->repositoryCount = window.repositoryCount; + image->missionStartId = window.missionStartId; + image->missionCount = window.missionCount; + image->missionOffset = OffsetFrom(window.repositoryStartId, window.missionStartId); +} + +void FillReport(const TileXRCcuRepositoryImage& image, TileXRCcuRepositoryReport* report) +{ + if (report == nullptr) { + return; + } + report->repositoryCount = image.repositoryCount; + report->installedInstructionCount = static_cast(image.sqeLoadCount + image.syncCount); + report->sqeLoadOffset = image.sqeLoadOffset; + report->syncOffset = image.syncOffset; + report->repositoryUploaded = false; + report->repositoryInstalled = false; + report->message = "ok"; +} + +int BuildRepositoryPaddingInstruction(const TileXRCcuProducerPlan& plan, TileXRCcuInstr* instr, + TileXRCcuRepositoryReport* report) +{ + if (plan.kernelLocalXn.startId == 0 || + TileXRCcuEncodeLoadImdToXn(plan.kernelLocalXn.startId, 0, 0, instr) != TILEXR_SUCCESS) { + return Fail(report, "failed to build valid repository padding instruction"); + } + return TILEXR_SUCCESS; +} + +int ValidateMemoryOps(const TileXRCcuDeviceMemoryOps& memoryOps, TileXRCcuRepositoryReport* report) +{ + if (memoryOps.alloc == nullptr) { + return Fail(report, "missing CCU repository device allocation hook"); + } + if (memoryOps.copyHostToDevice == nullptr) { + return Fail(report, "missing CCU repository host-to-device copy hook"); + } + if (memoryOps.free == nullptr) { + return Fail(report, "missing CCU repository device free hook"); + } + return TILEXR_SUCCESS; +} + +int ValidateInstallImage(const TileXRCcuRepositoryImage& image, TileXRCcuRepositoryReport* report) +{ + if (image.instructions.empty()) { + return Fail(report, "missing CCU repository instruction image"); + } + if (image.missionCount == 0) { + return Fail(report, "missing CCU repository mission instruction window"); + } + if (image.missionOffset >= image.instructions.size() || + static_cast(image.missionOffset) + image.missionCount > image.instructions.size()) { + return Fail(report, "CCU repository mission window is outside instruction image"); + } + if (image.missionStartId == 0) { + return Fail(report, "missing CCU repository mission start instruction id"); + } + return TILEXR_SUCCESS; +} + +void FillInstallReport(const TileXRCcuRepositoryInstallReceipt& receipt, TileXRCcuRepositoryReport* report) +{ + if (report == nullptr) { + return; + } + report->repositoryCount = receipt.instructionCount; + report->installedInstructionCount = receipt.instructionCount; + report->sqeLoadOffset = 0; + report->syncOffset = 0; + report->repositoryUploaded = receipt.uploaded; + report->repositoryInstalled = receipt.installed; + report->message = "ok"; +} + +bool FitsSizeT(uint64_t bytes) +{ + return bytes <= static_cast(std::numeric_limits::max()); +} + +uint16_t InstallStartId(const TileXRCcuRepositoryImage& image, const TileXRCcuRepositoryInstallOptions& options) +{ + return options.window == TileXRCcuRepositoryInstallWindow::FullRepository ? + image.repositoryStartId : + image.missionStartId; +} + +uint16_t InstallInstructionCount( + const TileXRCcuRepositoryImage& image, + const TileXRCcuRepositoryInstallOptions& options) +{ + return options.window == TileXRCcuRepositoryInstallWindow::FullRepository ? + image.repositoryCount : + image.missionCount; +} + +uint16_t InstallImageOffset(const TileXRCcuRepositoryImage& image, const TileXRCcuRepositoryInstallOptions& options) +{ + return options.window == TileXRCcuRepositoryInstallWindow::FullRepository ? 0 : image.missionOffset; +} + +uint32_t InstallCustomChannelDataLen( + uint64_t instructionBytes, + const TileXRCcuRepositoryInstallOptions& options) +{ + return options.dataLenMode == TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes ? + static_cast(sizeof(TileXRCcuInstrInfo)) : + static_cast(instructionBytes); +} + +const char* InstallWindowText(TileXRCcuRepositoryInstallWindow window) +{ + return window == TileXRCcuRepositoryInstallWindow::FullRepository ? "full_repository" : "mission"; +} + +const char* InstallDataLenModeText(TileXRCcuRepositoryInstallDataLenMode mode) +{ + return mode == TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes ? + "descriptor_bytes" : + "instruction_bytes"; +} + +uint64_t MixFnv1aByte(uint64_t hash, uint8_t value) +{ + constexpr uint64_t prime = 1099511628211ULL; + hash ^= value; + hash *= prime; + return hash; +} + +uint64_t BuildInstructionFnv1a64(const TileXRCcuInstr* instructions, uint16_t count) +{ + uint64_t hash = 1469598103934665603ULL; + for (uint16_t i = 0; i < count; ++i) { + for (uint64_t word : instructions[i].words) { + for (uint32_t byte = 0; byte < 8U; ++byte) { + hash = MixFnv1aByte(hash, static_cast((word >> (byte * 8U)) & 0xffU)); + } + } + } + return hash; +} + +std::string FormatInstructionWords(const TileXRCcuInstr& instr) +{ + std::ostringstream out; + out << std::hex << std::setfill('0'); + for (uint32_t i = 0; i < 4U; ++i) { + if (i != 0) { + out << ","; + } + out << "0x" << std::setw(16) << instr.words[i]; + } + return out.str(); +} + +uint32_t CountInstructionMismatches( + const TileXRCcuInstr* expected, + const TileXRCcuInstr* actual, + uint16_t count) +{ + uint32_t mismatches = 0; + for (uint16_t i = 0; i < count; ++i) { + if (expected[i].words[0] != actual[i].words[0] || + expected[i].words[1] != actual[i].words[1] || + expected[i].words[2] != actual[i].words[2] || + expected[i].words[3] != actual[i].words[3]) { + ++mismatches; + } + } + return mismatches; +} + +const char* UploadReadbackStatusText(const TileXRCcuUploadReadbackDiagnostic& diagnostic) +{ + if (!diagnostic.attempted) { + return "skipped"; + } + if (diagnostic.ret != 0) { + return "failed"; + } + if (!diagnostic.ok) { + return "mismatch"; + } + return "ok"; +} + +void FillSuccessfulReadbackDiagnostic( + const std::vector& readback, + const TileXRCcuInstr* expectedInstructions, + uint16_t instructionCount, + TileXRCcuUploadReadbackDiagnostic* diagnostic) +{ + if (diagnostic == nullptr || readback.empty() || expectedInstructions == nullptr || instructionCount == 0) { + return; + } + diagnostic->fnv1a64 = BuildInstructionFnv1a64(readback.data(), instructionCount); + diagnostic->firstInstructionWords = FormatInstructionWords(readback.front()); + diagnostic->lastInstructionWords = FormatInstructionWords(readback.back()); + diagnostic->mismatchCount = CountInstructionMismatches(expectedInstructions, readback.data(), instructionCount); + diagnostic->ok = diagnostic->mismatchCount == 0; +} + +void BuildUploadReadbackDiagnostic( + const TileXRCcuDeviceMemoryOps& memoryOps, + void* devicePtr, + uint64_t instructionBytes, + const TileXRCcuInstr* expectedInstructions, + uint16_t instructionCount, + void* memoryUserData, + TileXRCcuUploadReadbackDiagnostic* diagnostic) +{ + if (diagnostic == nullptr) { + return; + } + *diagnostic = TileXRCcuUploadReadbackDiagnostic {}; + if (memoryOps.copyDeviceToHost == nullptr) { + return; + } + diagnostic->attempted = true; + diagnostic->bytes = instructionBytes; + if (devicePtr == nullptr || expectedInstructions == nullptr || instructionCount == 0 || + !FitsSizeT(instructionBytes)) { + diagnostic->ret = TILEXR_ERROR_PARA_CHECK_FAIL; + return; + } + + std::vector readback(instructionCount); + diagnostic->ret = memoryOps.copyDeviceToHost( + readback.data(), + instructionBytes, + devicePtr, + instructionBytes, + memoryUserData); + if (diagnostic->ret != 0) { + return; + } + + FillSuccessfulReadbackDiagnostic(readback, expectedInstructions, instructionCount, diagnostic); +} + +void BuildDriverInstructionReadbackDiagnostic( + const TileXRCcuDriverAdapter& adapter, + uint8_t dieId, + uint16_t instructionStartId, + uint16_t instructionCount, + uint64_t instructionBytes, + const TileXRCcuInstr* expectedInstructions, + TileXRCcuUploadReadbackDiagnostic* diagnostic) +{ + if (diagnostic == nullptr) { + return; + } + *diagnostic = TileXRCcuUploadReadbackDiagnostic {}; + diagnostic->attempted = true; + diagnostic->bytes = instructionBytes; + if (expectedInstructions == nullptr || instructionCount == 0 || !FitsSizeT(instructionBytes)) { + diagnostic->ret = TILEXR_ERROR_PARA_CHECK_FAIL; + return; + } + + std::vector readback(instructionCount); + uint16_t readOffset = 0; + while (readOffset < instructionCount) { + const uint32_t batch = std::min( + TILEXR_CCU_MAX_DATA_ARRAY_SIZE, + static_cast(instructionCount - readOffset)); + TileXRCcuDriverAdapterReport driverReport; + diagnostic->ret = adapter.ReadInstructions( + dieId, + static_cast(instructionStartId + readOffset), + readback.data() + readOffset, + batch, + batch * TILEXR_CCU_INSTRUCTION_BYTES, + &driverReport); + if (diagnostic->ret != TILEXR_SUCCESS) { + return; + } + readOffset = static_cast(readOffset + batch); + } + + FillSuccessfulReadbackDiagnostic(readback, expectedInstructions, instructionCount, diagnostic); +} + +std::string BuildInstallFailureDiagnostic( + const std::string& driverMessage, + const TileXRCcuRepositoryImage& image, + const TileXRCcuRepositoryInstallOptions& options, + uint16_t installOffset, + uint16_t installStartId, + uint16_t installCount, + uint64_t instructionBytes, + uint32_t customChannelDataLen, + uint64_t deviceInstructionAddr, + const TileXRCcuUploadReadbackDiagnostic& uploadReadback, + const TileXRCcuUploadReadbackDiagnostic& driverReadback) +{ + const TileXRCcuInstr* firstInstruction = image.instructions.data() + installOffset; + const TileXRCcuInstr* lastInstruction = firstInstruction + installCount - 1U; + const uint64_t hash = BuildInstructionFnv1a64(firstInstruction, installCount); + + std::ostringstream message; + message << "failed to install CCU repository instruction image: " << driverMessage + << " dieId=" << static_cast(image.dieId) + << " installStartId=" << installStartId + << " installCount=" << installCount + << " instructionBytes=" << instructionBytes + << " customChannelDataLen=" << customChannelDataLen + << " deviceInstructionAddr=0x" << std::hex << deviceInstructionAddr << std::dec + << " window=" << InstallWindowText(options.window) + << " dataLenMode=" << InstallDataLenModeText(options.dataLenMode) + << " firstInstructionWords=" << FormatInstructionWords(*firstInstruction) + << " lastInstructionWords=" << FormatInstructionWords(*lastInstruction) + << " instructionFnv1a64=0x" << std::hex << hash << std::dec + << " uploadReadback=" << UploadReadbackStatusText(uploadReadback); + if (uploadReadback.attempted) { + message << " uploadReadbackRet=" << uploadReadback.ret + << " uploadReadbackBytes=" << uploadReadback.bytes; + if (uploadReadback.ret == 0) { + message << " uploadReadbackFnv1a64=0x" << std::hex << uploadReadback.fnv1a64 << std::dec + << " uploadReadbackFirstInstructionWords=" << uploadReadback.firstInstructionWords + << " uploadReadbackLastInstructionWords=" << uploadReadback.lastInstructionWords + << " uploadReadbackMismatchCount=" << uploadReadback.mismatchCount; + } + } + message << " driverReadback=" << UploadReadbackStatusText(driverReadback); + if (driverReadback.attempted) { + message << " driverReadbackRet=" << driverReadback.ret + << " driverReadbackBytes=" << driverReadback.bytes; + if (driverReadback.ret == 0) { + message << " driverReadbackFnv1a64=0x" << std::hex << driverReadback.fnv1a64 << std::dec + << " driverReadbackFirstInstructionWords=" << driverReadback.firstInstructionWords + << " driverReadbackLastInstructionWords=" << driverReadback.lastInstructionWords + << " driverReadbackMismatchCount=" << driverReadback.mismatchCount; + } + } + return message.str(); +} + +int AclDeviceAlloc(uint64_t bytes, void** devicePtr, void*) +{ + if (devicePtr == nullptr || bytes == 0 || !FitsSizeT(bytes)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + return aclrtMalloc(devicePtr, static_cast(bytes), ACL_MEM_MALLOC_HUGE_FIRST); +#else + *devicePtr = nullptr; + return TILEXR_ERROR_MKIRT; +#endif +} + +int AclModule3DeviceAlloc(uint64_t bytes, void** devicePtr, void*) +{ + if (devicePtr == nullptr || bytes == 0 || !FitsSizeT(bytes)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + aclrtMallocAttrValue moduleIdValue {}; + moduleIdValue.moduleId = TILEXR_CCU_ACL_MODULE3_ID; + aclrtMallocAttribute attrs {ACL_RT_MEM_ATTR_MODULE_ID, moduleIdValue}; + aclrtMallocConfig cfg {&attrs, 1}; + return aclrtMallocWithCfg( + devicePtr, + static_cast(bytes), + static_cast(ACL_MEM_TYPE_HIGH_BAND_WIDTH | ACL_MEM_MALLOC_HUGE_FIRST), + &cfg); +#else + *devicePtr = nullptr; + return TILEXR_ERROR_MKIRT; +#endif +} + +int AclCopyHostToDevice(void* deviceDst, uint64_t deviceDstBytes, const void* hostSrc, uint64_t bytes, void*) +{ + if (deviceDst == nullptr || hostSrc == nullptr || bytes == 0 || bytes > deviceDstBytes || + !FitsSizeT(deviceDstBytes) || !FitsSizeT(bytes)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + return aclrtMemcpy( + deviceDst, + static_cast(deviceDstBytes), + hostSrc, + static_cast(bytes), + ACL_MEMCPY_HOST_TO_DEVICE); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +int AclCopyDeviceToHost(void* hostDst, uint64_t hostDstBytes, const void* deviceSrc, uint64_t bytes, void*) +{ + if (hostDst == nullptr || deviceSrc == nullptr || bytes == 0 || bytes > hostDstBytes || + !FitsSizeT(hostDstBytes) || !FitsSizeT(bytes)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + return aclrtMemcpy( + hostDst, + static_cast(hostDstBytes), + deviceSrc, + static_cast(bytes), + ACL_MEMCPY_DEVICE_TO_HOST); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +int AclDeviceFree(void* devicePtr, void*) +{ + if (devicePtr == nullptr) { + return TILEXR_SUCCESS; + } +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + return aclrtFree(devicePtr); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +int RtHbmDeviceAlloc(uint64_t bytes, void** devicePtr, void*) +{ + if (devicePtr == nullptr || bytes == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER + return rtMalloc(devicePtr, bytes, RT_MEMORY_HBM, TILEXR_CCU_RT_HBM_MODULE_ID); +#else + *devicePtr = nullptr; + return TILEXR_ERROR_MKIRT; +#endif +} + +int RtHbmCopyHostToDevice(void* deviceDst, uint64_t deviceDstBytes, const void* hostSrc, uint64_t bytes, void*) +{ + if (deviceDst == nullptr || hostSrc == nullptr || bytes == 0 || bytes > deviceDstBytes) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER + return rtMemcpy(deviceDst, deviceDstBytes, hostSrc, bytes, RT_MEMCPY_HOST_TO_DEVICE); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +int RtHbmCopyDeviceToHost(void* hostDst, uint64_t hostDstBytes, const void* deviceSrc, uint64_t bytes, void*) +{ + if (hostDst == nullptr || deviceSrc == nullptr || bytes == 0 || bytes > hostDstBytes) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } +#if TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER + return rtMemcpy(hostDst, hostDstBytes, deviceSrc, bytes, RT_MEMCPY_DEVICE_TO_HOST); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +int RtHbmDeviceFree(void* devicePtr, void*) +{ + if (devicePtr == nullptr) { + return TILEXR_SUCCESS; + } +#if TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER + return rtFree(devicePtr); +#else + return TILEXR_ERROR_MKIRT; +#endif +} + +} // namespace + +TileXRCcuDeviceMemoryOps TileXRCcuMakeAclDeviceMemoryOps() +{ + TileXRCcuDeviceMemoryOps ops; +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + ops.alloc = AclDeviceAlloc; + ops.copyHostToDevice = AclCopyHostToDevice; + ops.free = AclDeviceFree; + ops.copyDeviceToHost = AclCopyDeviceToHost; +#endif + return ops; +} + +TileXRCcuDeviceMemoryOps TileXRCcuMakeAclModule3DeviceMemoryOps() +{ + TileXRCcuDeviceMemoryOps ops; +#if TILEXR_CCU_HAVE_ACL_RT_HEADER + ops.alloc = AclModule3DeviceAlloc; + ops.copyHostToDevice = AclCopyHostToDevice; + ops.free = AclDeviceFree; + ops.copyDeviceToHost = AclCopyDeviceToHost; +#endif + return ops; +} + +TileXRCcuDeviceMemoryOps TileXRCcuMakeRtHbmDeviceMemoryOps() +{ + TileXRCcuDeviceMemoryOps ops; +#if TILEXR_CCU_HAVE_RUNTIME_MEM_HEADER + ops.alloc = RtHbmDeviceAlloc; + ops.copyHostToDevice = RtHbmCopyHostToDevice; + ops.free = RtHbmDeviceFree; + ops.copyDeviceToHost = RtHbmCopyDeviceToHost; +#endif + return ops; +} + +TileXRCcuDeviceMemoryOps TileXRCcuMakeRepositoryDeviceMemoryOps(TileXRCcuRepositoryMemoryAllocMode mode) +{ + if (mode == TileXRCcuRepositoryMemoryAllocMode::AclModule3) { + return TileXRCcuMakeAclModule3DeviceMemoryOps(); + } + if (mode == TileXRCcuRepositoryMemoryAllocMode::RtHbm) { + return TileXRCcuMakeRtHbmDeviceMemoryOps(); + } + return TileXRCcuMakeAclDeviceMemoryOps(); +} + +int TileXRCcuBuildRepositoryImage( + const TileXRCcuProducerPlan& plan, + const TileXRCcuProgram& program, + TileXRCcuRepositoryImage* image, + TileXRCcuRepositoryReport* report) +{ + ResetReport(report); + if (image == nullptr) { + return Fail(report, "missing output repository image"); + } + image->instructions.clear(); + + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + return Fail(report, planReport.message); + } + if (ValidateProgramShape(plan, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (ValidateRepositoryWindow(plan, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + FillImageHeader(plan.instructionWindow, image); + const size_t syncTaskIndex = program.sqeLoad.empty() ? 0U : 1U; + image->sqeLoadOffset = program.sqeLoad.empty() ? + 0 : + OffsetFrom(plan.instructionWindow.repositoryStartId, plan.taskWindows[0].instStartId); + image->sqeLoadCount = static_cast(program.sqeLoad.size()); + image->syncOffset = OffsetFrom(plan.instructionWindow.repositoryStartId, plan.taskWindows[syncTaskIndex].instStartId); + image->syncCount = static_cast(program.sync.size()); + + const uint32_t sqeEnd = static_cast(image->sqeLoadOffset) + image->sqeLoadCount; + const uint32_t syncEnd = static_cast(image->syncOffset) + image->syncCount; + if (sqeEnd > image->repositoryCount || syncEnd > image->repositoryCount) { + image->instructions.clear(); + return Fail(report, "generated microcode does not fit in repository image"); + } + + TileXRCcuInstr paddingInstr; + if (BuildRepositoryPaddingInstruction(plan, &paddingInstr, report) != TILEXR_SUCCESS) { + image->instructions.clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + image->instructions.assign(image->repositoryCount, paddingInstr); + std::copy(program.sqeLoad.begin(), program.sqeLoad.end(), image->instructions.begin() + image->sqeLoadOffset); + std::copy(program.sync.begin(), program.sync.end(), image->instructions.begin() + image->syncOffset); + + FillReport(*image, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuInstallRepositoryImage( + const TileXRCcuRepositoryImage& image, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + const TileXRCcuDriverAdapter& adapter, + TileXRCcuRepositoryInstallReceipt* receipt, + TileXRCcuRepositoryReport* report) +{ + TileXRCcuRepositoryInstallOptions options; + return TileXRCcuInstallRepositoryImageWithOptions( + image, + options, + memoryOps, + memoryUserData, + adapter, + receipt, + report); +} + +int TileXRCcuInstallRepositoryImageWithOptions( + const TileXRCcuRepositoryImage& image, + const TileXRCcuRepositoryInstallOptions& options, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + const TileXRCcuDriverAdapter& adapter, + TileXRCcuRepositoryInstallReceipt* receipt, + TileXRCcuRepositoryReport* report) +{ + ResetReport(report); + if (receipt == nullptr) { + return Fail(report, "missing output CCU repository install receipt"); + } + *receipt = TileXRCcuRepositoryInstallReceipt{}; + + if (ValidateMemoryOps(memoryOps, report) != TILEXR_SUCCESS || + ValidateInstallImage(image, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint16_t installOffset = InstallImageOffset(image, options); + const uint16_t installStartId = InstallStartId(image, options); + const uint16_t installCount = InstallInstructionCount(image, options); + if (installCount == 0 || + installOffset >= image.instructions.size() || + static_cast(installOffset) + installCount > image.instructions.size()) { + return Fail(report, "CCU repository selected install window is outside instruction image"); + } + + const uint64_t instructionBytes = static_cast(installCount) * sizeof(TileXRCcuInstr); + void* devicePtr = nullptr; + const int allocRet = memoryOps.alloc(instructionBytes, &devicePtr, memoryUserData); + if (allocRet != 0 || devicePtr == nullptr) { + return Fail(report, "failed to allocate CCU repository device instruction image", TILEXR_ERROR_MKIRT); + } + + const auto* firstInstruction = image.instructions.data() + installOffset; + const int copyRet = memoryOps.copyHostToDevice( + devicePtr, + instructionBytes, + firstInstruction, + instructionBytes, + memoryUserData); + if (copyRet != 0) { + (void)memoryOps.free(devicePtr, memoryUserData); + return Fail(report, "failed to copy CCU repository instruction image to device", TILEXR_ERROR_MKIRT); + } + + TileXRCcuRepositoryInstallReceipt result; + result.dieId = image.dieId; + result.instructionStartId = installStartId; + result.instructionCount = installCount; + result.instructionBytes = instructionBytes; + result.deviceInstructionAddr = reinterpret_cast(devicePtr); + result.deviceInstructionPtr = devicePtr; + result.uploaded = true; + + TileXRCcuUploadReadbackDiagnostic uploadReadback; + BuildUploadReadbackDiagnostic( + memoryOps, + devicePtr, + instructionBytes, + firstInstruction, + installCount, + memoryUserData, + &uploadReadback); + + const uint32_t customChannelDataLen = InstallCustomChannelDataLen(instructionBytes, options); + TileXRCcuDriverAdapterReport driverReport; + const int installRet = adapter.InstallInstructionsWithDataLen( + image.dieId, + installStartId, + installCount, + result.deviceInstructionAddr, + static_cast(instructionBytes), + customChannelDataLen, + &driverReport); + if (installRet != TILEXR_SUCCESS) { + TileXRCcuUploadReadbackDiagnostic driverReadback; + BuildDriverInstructionReadbackDiagnostic( + adapter, + image.dieId, + installStartId, + installCount, + instructionBytes, + firstInstruction, + &driverReadback); + (void)memoryOps.free(devicePtr, memoryUserData); + return Fail( + report, + BuildInstallFailureDiagnostic( + driverReport.message, + image, + options, + installOffset, + installStartId, + installCount, + instructionBytes, + customChannelDataLen, + result.deviceInstructionAddr, + uploadReadback, + driverReadback), + TILEXR_ERROR_MKIRT); + } + + result.installed = true; + *receipt = result; + FillInstallReport(*receipt, report); + if (report != nullptr && + (options.window != TileXRCcuRepositoryInstallWindow::Mission || + options.dataLenMode != TileXRCcuRepositoryInstallDataLenMode::InstructionBytes)) { + report->message = std::string("ok window=") + InstallWindowText(options.window) + + " dataLenMode=" + InstallDataLenModeText(options.dataLenMode); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuReleaseRepositoryInstallReceipt( + TileXRCcuRepositoryInstallReceipt& receipt, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + TileXRCcuRepositoryReport* report) +{ + ResetReport(report); + if (memoryOps.free == nullptr) { + return Fail(report, "missing CCU repository device free hook"); + } + if (receipt.deviceInstructionPtr == nullptr) { + receipt = TileXRCcuRepositoryInstallReceipt{}; + if (report != nullptr) { + report->message = "ok"; + } + return TILEXR_SUCCESS; + } + const int freeRet = memoryOps.free(receipt.deviceInstructionPtr, memoryUserData); + if (freeRet != 0) { + return Fail(report, "failed to release CCU repository device instruction image", TILEXR_ERROR_MKIRT); + } + receipt = TileXRCcuRepositoryInstallReceipt{}; + if (report != nullptr) { + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_repository.h b/src/comm/ccu/tilexr_ccu_repository.h new file mode 100644 index 00000000..5a7fdc02 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_repository.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_REPOSITORY_H +#define TILEXR_CCU_REPOSITORY_H + +#include "ccu/tilexr_ccu_driver_adapter.h" +#include "ccu/tilexr_ccu_producer_plan.h" + +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuRepositoryImage { + uint8_t dieId = 0; + uint16_t repositoryStartId = 0; + uint16_t repositoryCount = 0; + uint16_t missionStartId = 0; + uint16_t missionCount = 0; + uint16_t missionOffset = 0; + uint16_t sqeLoadOffset = 0; + uint16_t sqeLoadCount = 0; + uint16_t syncOffset = 0; + uint16_t syncCount = 0; + std::vector instructions; +}; + +struct TileXRCcuRepositoryReport { + uint16_t repositoryCount = 0; + uint16_t installedInstructionCount = 0; + uint16_t sqeLoadOffset = 0; + uint16_t syncOffset = 0; + bool repositoryUploaded = false; + bool repositoryInstalled = false; + std::string message; +}; + +using TileXRCcuDeviceAllocFn = int (*)(uint64_t bytes, void** devicePtr, void* userData); +using TileXRCcuCopyHostToDeviceFn = int (*)( + void* deviceDst, + uint64_t deviceDstBytes, + const void* hostSrc, + uint64_t bytes, + void* userData); +using TileXRCcuCopyDeviceToHostFn = int (*)( + void* hostDst, + uint64_t hostDstBytes, + const void* deviceSrc, + uint64_t bytes, + void* userData); +using TileXRCcuDeviceFreeFn = int (*)(void* devicePtr, void* userData); + +struct TileXRCcuDeviceMemoryOps { + TileXRCcuDeviceAllocFn alloc = nullptr; + TileXRCcuCopyHostToDeviceFn copyHostToDevice = nullptr; + TileXRCcuDeviceFreeFn free = nullptr; + TileXRCcuCopyDeviceToHostFn copyDeviceToHost = nullptr; +}; + +enum class TileXRCcuRepositoryMemoryAllocMode : uint8_t { + Acl = 0, + AclModule3 = 1, + RtHbm = 2, +}; + +struct TileXRCcuRepositoryInstallReceipt { + uint8_t dieId = 0; + uint16_t instructionStartId = 0; + uint16_t instructionCount = 0; + uint64_t instructionBytes = 0; + uint64_t deviceInstructionAddr = 0; + void* deviceInstructionPtr = nullptr; + bool uploaded = false; + bool installed = false; +}; + +enum class TileXRCcuRepositoryInstallWindow : uint8_t { + Mission = 0, + FullRepository = 1, +}; + +enum class TileXRCcuRepositoryInstallDataLenMode : uint8_t { + InstructionBytes = 0, + DescriptorBytes = 1, +}; + +struct TileXRCcuRepositoryInstallOptions { + TileXRCcuRepositoryInstallWindow window = TileXRCcuRepositoryInstallWindow::Mission; + TileXRCcuRepositoryInstallDataLenMode dataLenMode = TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; +}; + +TileXRCcuDeviceMemoryOps TileXRCcuMakeAclDeviceMemoryOps(); +TileXRCcuDeviceMemoryOps TileXRCcuMakeAclModule3DeviceMemoryOps(); +TileXRCcuDeviceMemoryOps TileXRCcuMakeRtHbmDeviceMemoryOps(); +TileXRCcuDeviceMemoryOps TileXRCcuMakeRepositoryDeviceMemoryOps(TileXRCcuRepositoryMemoryAllocMode mode); + +int TileXRCcuBuildRepositoryImage( + const TileXRCcuProducerPlan& plan, + const TileXRCcuProgram& program, + TileXRCcuRepositoryImage* image, + TileXRCcuRepositoryReport* report); + +int TileXRCcuInstallRepositoryImage( + const TileXRCcuRepositoryImage& image, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + const TileXRCcuDriverAdapter& adapter, + TileXRCcuRepositoryInstallReceipt* receipt, + TileXRCcuRepositoryReport* report); + +int TileXRCcuInstallRepositoryImageWithOptions( + const TileXRCcuRepositoryImage& image, + const TileXRCcuRepositoryInstallOptions& options, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + const TileXRCcuDriverAdapter& adapter, + TileXRCcuRepositoryInstallReceipt* receipt, + TileXRCcuRepositoryReport* report); + +int TileXRCcuReleaseRepositoryInstallReceipt( + TileXRCcuRepositoryInstallReceipt& receipt, + const TileXRCcuDeviceMemoryOps& memoryOps, + void* memoryUserData, + TileXRCcuRepositoryReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_REPOSITORY_H diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp new file mode 100644 index 00000000..e9bb407b --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp @@ -0,0 +1,377 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_resource_allocator.h" + +#include +#include + +namespace TileXR { +namespace { + +constexpr const char* TILEXR_CCU_HCOMM_DERIVED_PROVIDER = "tilexr-hcomm-derived-resource-allocator"; +constexpr uint32_t TILEXR_CCU_HCOMM_TASK1_PRELUDE_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_HCOMM_TASK1_PRELUDE_RESERVED_XN_COUNT = 1U; + +void ResetReport(TileXRCcuResourceAllocatorReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuResourceAllocatorReport{}; + } +} + +int Fail(TileXRCcuResourceAllocatorReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool AddWouldOverflow(uint16_t start, uint16_t count) +{ + return static_cast(start) + count > static_cast(std::numeric_limits::max()) + 1U; +} + +bool ContainsRange(uint16_t outerStart, uint16_t outerCount, uint16_t innerStart, uint32_t innerCount) +{ + if (outerCount == 0 || innerCount == 0) { + return false; + } + const uint32_t outerBegin = outerStart; + const uint32_t outerEnd = outerBegin + outerCount; + const uint32_t innerBegin = innerStart; + const uint32_t innerEnd = innerBegin + innerCount; + return innerBegin >= outerBegin && innerEnd <= outerEnd; +} + +TileXRCcuRange MakeRange(uint8_t dieId, uint16_t startId, uint16_t count) +{ + TileXRCcuRange range; + range.dieId = dieId; + range.startId = startId; + range.num = count; + return range; +} + +void FillReport( + const TileXRCcuResourceAllocation& allocation, + const TileXRCcuResourceRequest& request, + TileXRCcuResourceAllocatorReport* report) +{ + if (report == nullptr) { + return; + } + report->missionAllocated = allocation.mission.num; + report->repositoryAllocated = allocation.repository.num; + report->localXnAllocated = allocation.localXn.num; + report->localGsaAllocated = allocation.localGsa.num; + report->remoteXnAllocated = allocation.remoteXn.num; + report->notifyCkeAllocated = allocation.notifyCke.num; + report->channelBindingsAllocated = request.syncResourceCount * request.bindingsPerSyncResource; + report->localWaitCkeAllocated = allocation.localWaitCke.num; + report->remoteNotifyCkeAllocated = allocation.remoteNotifyCke.num; + report->sourceCkeAllocated = allocation.sourceCke.num; + report->message = "ok"; +} + +uint16_t CheckedU16(uint32_t value) +{ + return static_cast(std::min(value, std::numeric_limits::max())); +} + +bool SyncXnMode(TileXRCcuBarrierMode mode) +{ + return mode == TileXRCcuBarrierMode::SyncXn || + mode == TileXRCcuBarrierMode::SyncXnPostOnly || + mode == TileXRCcuBarrierMode::SyncXnLoadPostOnly; +} + +uint32_t RequiredSqeLoadXnCount(uint32_t sqeArgCount, bool hcommStyleTask1Prelude) +{ + if (sqeArgCount == 0) { + return 0; + } + + uint32_t count = hcommStyleTask1Prelude ? + std::max(sqeArgCount, TILEXR_CCU_SQE_ARGS_LEN) + + TILEXR_CCU_HCOMM_TASK1_PRELUDE_RESERVED_XN_COUNT : + sqeArgCount; + return count; +} + +} // namespace + +bool TileXRCcuResourceAllocator::HasCapacity(const Cursor& cursor, uint32_t count) const +{ + return count <= std::numeric_limits::max() && + static_cast(cursor.used) + count <= cursor.count; +} + +uint16_t TileXRCcuResourceAllocator::CursorNext(const Cursor& cursor) const +{ + return static_cast(cursor.start + cursor.used); +} + +int TileXRCcuResourceAllocator::Init(const TileXRCcuResourceSpec& spec) +{ + const uint16_t missionInstructionStart = + spec.missionInstructionStartId == 0 ? spec.instructionStartId : spec.missionInstructionStartId; + const uint16_t localWaitCkeStart = + spec.localWaitCkeCount == 0 ? spec.ckeStartId : spec.localWaitCkeStartId; + const uint16_t localWaitCkeCount = + spec.localWaitCkeCount == 0 ? spec.ckeCount : spec.localWaitCkeCount; + const uint16_t remoteNotifyCkeStart = + spec.remoteNotifyCkeCount == 0 ? spec.ckeStartId : spec.remoteNotifyCkeStartId; + const uint16_t remoteNotifyCkeCount = + spec.remoteNotifyCkeCount == 0 ? spec.ckeCount : spec.remoteNotifyCkeCount; + const bool splitRemoteXn = spec.remoteXnCount != 0; + + if (spec.missionKey == 0 || spec.missionCount == 0 || spec.instructionCount == 0 || + spec.xnCount == 0 || localWaitCkeCount == 0 || remoteNotifyCkeCount == 0 || + spec.channelCount == 0 || + AddWouldOverflow(spec.missionStartId, spec.missionCount) || + AddWouldOverflow(spec.instructionStartId, spec.instructionCount) || + !ContainsRange(spec.instructionStartId, spec.instructionCount, missionInstructionStart, 1) || + AddWouldOverflow(spec.xnStartId, spec.xnCount) || + (spec.gsaCount != 0 && (spec.gsaStartId == 0 || AddWouldOverflow(spec.gsaStartId, spec.gsaCount))) || + (splitRemoteXn && (spec.remoteXnStartId == 0 || AddWouldOverflow(spec.remoteXnStartId, spec.remoteXnCount))) || + AddWouldOverflow(localWaitCkeStart, localWaitCkeCount) || + AddWouldOverflow(remoteNotifyCkeStart, remoteNotifyCkeCount) || + AddWouldOverflow(spec.channelStartId, spec.channelCount)) { + initialized_ = false; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + spec_ = spec; + mission_ = {spec.missionStartId, spec.missionCount, 0}; + repository_ = {spec.instructionStartId, spec.instructionCount, 0}; + xn_ = {spec.xnStartId, spec.xnCount, 0}; + gsa_ = {spec.gsaStartId, spec.gsaCount, 0}; + remoteXn_ = splitRemoteXn ? Cursor{spec.remoteXnStartId, spec.remoteXnCount, 0} : Cursor{}; + localWaitCke_ = {localWaitCkeStart, localWaitCkeCount, 0}; + remoteNotifyCke_ = {remoteNotifyCkeStart, remoteNotifyCkeCount, 0}; + channel_ = {spec.channelStartId, spec.channelCount, 0}; + nextReceiptId_ = 1; + active_.clear(); + initialized_ = true; + return TILEXR_SUCCESS; +} + +int TileXRCcuResourceAllocator::Allocate( + const TileXRCcuResourceRequest& request, + TileXRCcuProducerPlan* plan, + TileXRCcuResourceAllocation* allocation, + TileXRCcuResourceAllocatorReport* report) +{ + ResetReport(report); + if (!initialized_) { + return Fail(report, "CCU resource allocator is not initialized"); + } + if (plan == nullptr || allocation == nullptr) { + return Fail(report, "missing CCU resource allocation output"); + } + *plan = TileXRCcuProducerPlan{}; + *allocation = TileXRCcuResourceAllocation{}; + + if (request.sqeArgCount > TILEXR_CCU_SQE_ARGS_LEN) { + return Fail(report, "invalid SQE argument count"); + } + if (request.syncResourceCount == 0 || request.syncInstructionCount == 0 || + request.bindingsPerSyncResource == 0) { + return Fail(report, "invalid CCU sync resource request"); + } + if (request.syncResourceCount > std::numeric_limits::max() / 2U) { + return Fail(report, "barrier sync instruction window is too small for CCU post and wait microcode"); + } + const bool syncCkeMode = request.barrierMode == TileXRCcuBarrierMode::SyncCke || + request.barrierMode == TileXRCcuBarrierMode::SyncCkeSetWait || + request.barrierMode == TileXRCcuBarrierMode::SyncCkePostOnly; + const bool hcommStyleTask1Prelude = request.sqeArgCount != 0 && SyncXnMode(request.barrierMode); + const bool postOnly = request.barrierMode == TileXRCcuBarrierMode::SyncXnPostOnly || + request.barrierMode == TileXRCcuBarrierMode::SyncCkePostOnly || + request.barrierMode == TileXRCcuBarrierMode::LocalCkePostOnly; + const uint32_t requiredPostWaitInstructionCount = + postOnly ? request.syncResourceCount : request.syncResourceCount * 2U; + const uint32_t sourceCkeInitCount = syncCkeMode ? 1U : 0U; + const uint32_t sourceCkeResourceCount = syncCkeMode ? 1U : 0U; + const uint32_t task1PreludeInstructionCount = + hcommStyleTask1Prelude ? TILEXR_CCU_HCOMM_TASK1_PRELUDE_INSTRUCTION_COUNT : 0U; + const uint32_t requiredBarrierInstructionCount = + requiredPostWaitInstructionCount + sourceCkeInitCount + task1PreludeInstructionCount; + if (request.syncInstructionCount < requiredBarrierInstructionCount) { + return Fail(report, + hcommStyleTask1Prelude ? + "barrier sync instruction window is too small for hcomm-style task1 prelude and CCU post/wait microcode" : + "barrier sync instruction window is too small for CCU post and wait microcode"); + } + + const uint32_t localSqeXnCount = RequiredSqeLoadXnCount(request.sqeArgCount, hcommStyleTask1Prelude); + const uint32_t localXnCount = std::max(localSqeXnCount, request.syncResourceCount); + const uint32_t remoteXnCount = request.syncResourceCount; + const uint32_t localGsaCount = hcommStyleTask1Prelude && spec_.gsaCount != 0 ? 1U : 0U; + const uint32_t totalXnCount = localXnCount + remoteXnCount; + const uint32_t localWaitCkeCount = request.syncResourceCount; + const uint32_t remoteNotifyCkeCount = request.syncResourceCount; + const uint32_t localCkeCount = localWaitCkeCount + sourceCkeResourceCount; + const uint16_t repositoryStart = CursorNext(repository_); + const uint16_t missionInstructionStart = + spec_.missionInstructionStartId == 0 ? repositoryStart : spec_.missionInstructionStartId; + if (!ContainsRange(repositoryStart, repository_.count, missionInstructionStart, 1)) { + return Fail(report, "mission instruction start is outside instruction repository resources"); + } + const uint32_t repositoryPrefixCount = + static_cast(missionInstructionStart) - static_cast(repositoryStart); + const uint32_t missionInstructionCount = request.sqeArgCount + request.syncInstructionCount; + const uint32_t repositoryCount = repositoryPrefixCount + missionInstructionCount; + const uint32_t channelCount = request.syncResourceCount; + + if (!HasCapacity(mission_, 1)) { + return Fail(report, "insufficient mission resources"); + } + if (!HasCapacity(repository_, repositoryCount)) { + return Fail(report, "insufficient instruction repository resources"); + } + const bool splitRemoteXn = remoteXn_.count != 0; + if (!HasCapacity(xn_, splitRemoteXn ? localXnCount : totalXnCount)) { + return Fail(report, "insufficient XN resources"); + } + if (localGsaCount != 0 && !HasCapacity(gsa_, localGsaCount)) { + return Fail(report, "insufficient GSA resources"); + } + if (splitRemoteXn && !HasCapacity(remoteXn_, remoteXnCount)) { + return Fail(report, "insufficient remote XN resources"); + } + if (!HasCapacity(localWaitCke_, localCkeCount)) { + return Fail(report, "insufficient CKE resources"); + } + if (!HasCapacity(remoteNotifyCke_, remoteNotifyCkeCount)) { + return Fail(report, "insufficient remote notify CKE resources"); + } + if (!HasCapacity(channel_, channelCount)) { + return Fail(report, "insufficient channel resources"); + } + + const uint16_t missionStart = CursorNext(mission_); + const uint16_t localXnStart = CursorNext(xn_); + const uint16_t localGsaStart = CursorNext(gsa_); + const uint16_t remoteXnStart = splitRemoteXn ? + CursorNext(remoteXn_) : + static_cast(localXnStart + localXnCount); + const uint16_t localWaitCkeStart = CursorNext(localWaitCke_); + const uint16_t sourceCkeStart = static_cast(localWaitCkeStart + localWaitCkeCount); + const uint16_t remoteNotifyCkeStart = CursorNext(remoteNotifyCke_); + const uint16_t channelStart = CursorNext(channel_); + + TileXRCcuResourceAllocation result; + result.receiptId = nextReceiptId_++; + result.packageProvider = TILEXR_CCU_HCOMM_DERIVED_PROVIDER; + result.mission = MakeRange(spec_.dieId, missionStart, 1); + result.repository = MakeRange(spec_.dieId, repositoryStart, CheckedU16(repositoryCount)); + result.localXn = MakeRange(spec_.dieId, localXnStart, CheckedU16(localXnCount)); + result.localGsa = MakeRange(spec_.dieId, localGsaStart, CheckedU16(localGsaCount)); + result.remoteXn = MakeRange(spec_.dieId, remoteXnStart, CheckedU16(remoteXnCount)); + result.notifyCke = MakeRange(spec_.dieId, remoteNotifyCkeStart, CheckedU16(remoteNotifyCkeCount)); + result.channels = MakeRange(spec_.dieId, channelStart, CheckedU16(channelCount)); + result.localWaitCke = MakeRange(spec_.dieId, localWaitCkeStart, CheckedU16(localWaitCkeCount)); + result.remoteNotifyCke = result.notifyCke; + result.sourceCke = MakeRange(spec_.dieId, sourceCkeStart, CheckedU16(sourceCkeResourceCount)); + + TileXRCcuProducerPlan generated; + generated.barrierMode = request.barrierMode; + generated.mission = {spec_.dieId, static_cast(missionStart), spec_.missionKey, true}; + generated.kernelLocalMission = result.mission; + generated.kernelLocalXn = result.localXn; + generated.kernelLocalGsa = result.localGsa; + generated.kernelLocalCke = MakeRange(spec_.dieId, localWaitCkeStart, CheckedU16(localCkeCount)); + generated.instructionWindow = { + spec_.dieId, + result.repository.startId, + result.repository.num, + missionInstructionStart, + CheckedU16(missionInstructionCount), + }; + + for (uint32_t i = 0; i < request.syncResourceCount; ++i) { + TileXRCcuSyncResource resource; + resource.dieId = spec_.dieId; + resource.localXn = static_cast(result.localXn.startId + i); + resource.remoteXn = static_cast(static_cast(result.remoteXn.startId) + i); + resource.notifyCke = static_cast(static_cast(result.remoteNotifyCke.startId) + i); + resource.channelId = static_cast(static_cast(result.channels.startId) + i); + resource.bindingCount = CheckedU16(request.bindingsPerSyncResource); + resource.localWaitCke = static_cast(result.localWaitCke.startId + i); + resource.localWaitMask = 1; + resource.remoteNotifyMask = 1; + if (syncCkeMode) { + resource.sourceCke = result.sourceCke.startId; + resource.sourceCkeMask = 0xffff; + } + generated.syncResources.push_back(resource); + } + + if (request.sqeArgCount != 0) { + TileXRCcuTaskWindow sqeLoadTask; + sqeLoadTask.dieId = spec_.dieId; + sqeLoadTask.instStartId = missionInstructionStart; + sqeLoadTask.instCnt = CheckedU16(request.sqeArgCount); + sqeLoadTask.argSize = TILEXR_CCU_SQE_ARGS_LEN; + generated.taskWindows.push_back(sqeLoadTask); + } + + TileXRCcuTaskWindow syncTask; + syncTask.dieId = spec_.dieId; + syncTask.instStartId = static_cast(missionInstructionStart + request.sqeArgCount); + syncTask.instCnt = CheckedU16(request.syncInstructionCount); + syncTask.argSize = TILEXR_CCU_SQE_ARGS_LEN; + generated.taskWindows.push_back(syncTask); + + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(generated, &planReport) != TILEXR_SUCCESS) { + return Fail(report, planReport.message); + } + + mission_.used = static_cast(mission_.used + result.mission.num); + repository_.used = static_cast(repository_.used + result.repository.num); + xn_.used = static_cast(xn_.used + result.localXn.num + (splitRemoteXn ? 0 : result.remoteXn.num)); + gsa_.used = static_cast(gsa_.used + result.localGsa.num); + if (splitRemoteXn) { + remoteXn_.used = static_cast(remoteXn_.used + result.remoteXn.num); + } + localWaitCke_.used = static_cast(localWaitCke_.used + result.localWaitCke.num + result.sourceCke.num); + remoteNotifyCke_.used = static_cast(remoteNotifyCke_.used + result.remoteNotifyCke.num); + channel_.used = static_cast(channel_.used + result.channels.num); + + ActiveAllocation active; + active.allocation = result; + active.missionUsed = result.mission.num; + active.repositoryUsed = result.repository.num; + active.localXnUsed = result.localXn.num; + active.localGsaUsed = result.localGsa.num; + active.remoteXnUsed = result.remoteXn.num; + active.notifyCkeUsed = result.notifyCke.num; + active.channelUsed = result.channels.num; + active.localWaitCkeUsed = result.localWaitCke.num; + active.remoteNotifyCkeUsed = result.remoteNotifyCke.num; + active.sourceCkeUsed = result.sourceCke.num; + active_[result.receiptId] = active; + + *plan = generated; + *allocation = result; + FillReport(result, request, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuResourceAllocator::Release(uint64_t receiptId) +{ + const auto it = active_.find(receiptId); + if (it == active_.end()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + active_.erase(it); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.h b/src/comm/ccu/tilexr_ccu_resource_allocator.h new file mode 100644 index 00000000..706ff2b4 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.h @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_RESOURCE_ALLOCATOR_H +#define TILEXR_CCU_RESOURCE_ALLOCATOR_H + +#include "ccu/tilexr_ccu_producer_plan.h" + +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuResourceSpec { + uint8_t dieId = 0; + uint32_t missionKey = 0; + uint16_t missionStartId = 0; + uint16_t missionCount = 0; + uint16_t instructionStartId = 0; + uint16_t missionInstructionStartId = 0; + uint16_t instructionCount = 0; + uint16_t xnStartId = 0; + uint16_t xnCount = 0; + uint16_t gsaStartId = 0; + uint16_t gsaCount = 0; + uint16_t remoteXnStartId = 0; + uint16_t remoteXnCount = 0; + uint16_t ckeStartId = 0; + uint16_t ckeCount = 0; + uint16_t channelStartId = 0; + uint16_t channelCount = 0; + uint16_t localWaitCkeStartId = 0; + uint16_t localWaitCkeCount = 0; + uint16_t remoteNotifyCkeStartId = 0; + uint16_t remoteNotifyCkeCount = 0; +}; + +struct TileXRCcuResourceRequest { + uint32_t sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + uint32_t syncResourceCount = 0; + uint32_t syncInstructionCount = 0; + uint32_t bindingsPerSyncResource = 1; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; +}; + +struct TileXRCcuResourceAllocation { + uint64_t receiptId = 0; + std::string packageProvider; + TileXRCcuRange mission; + TileXRCcuRange repository; + TileXRCcuRange localXn; + TileXRCcuRange localGsa; + TileXRCcuRange remoteXn; + TileXRCcuRange notifyCke; + TileXRCcuRange channels; + TileXRCcuRange localWaitCke; + TileXRCcuRange remoteNotifyCke; + TileXRCcuRange sourceCke; +}; + +struct TileXRCcuResourceAllocatorReport { + uint32_t missionAllocated = 0; + uint32_t repositoryAllocated = 0; + uint32_t localXnAllocated = 0; + uint32_t localGsaAllocated = 0; + uint32_t remoteXnAllocated = 0; + uint32_t notifyCkeAllocated = 0; + uint32_t channelBindingsAllocated = 0; + uint32_t localWaitCkeAllocated = 0; + uint32_t remoteNotifyCkeAllocated = 0; + uint32_t sourceCkeAllocated = 0; + std::string message; +}; + +class TileXRCcuResourceAllocator { +public: + int Init(const TileXRCcuResourceSpec& spec); + + int Allocate( + const TileXRCcuResourceRequest& request, + TileXRCcuProducerPlan* plan, + TileXRCcuResourceAllocation* allocation, + TileXRCcuResourceAllocatorReport* report); + + int Release(uint64_t receiptId); + +private: + struct Cursor { + uint16_t start = 0; + uint16_t count = 0; + uint16_t used = 0; + }; + + bool HasCapacity(const Cursor& cursor, uint32_t count) const; + uint16_t CursorNext(const Cursor& cursor) const; + + struct ActiveAllocation { + TileXRCcuResourceAllocation allocation; + uint16_t missionUsed = 0; + uint16_t repositoryUsed = 0; + uint16_t localXnUsed = 0; + uint16_t localGsaUsed = 0; + uint16_t remoteXnUsed = 0; + uint16_t notifyCkeUsed = 0; + uint16_t channelUsed = 0; + uint16_t localWaitCkeUsed = 0; + uint16_t remoteNotifyCkeUsed = 0; + uint16_t sourceCkeUsed = 0; + }; + + TileXRCcuResourceSpec spec_; + Cursor mission_; + Cursor repository_; + Cursor xn_; + Cursor gsa_; + Cursor remoteXn_; + Cursor localWaitCke_; + Cursor remoteNotifyCke_; + Cursor channel_; + uint64_t nextReceiptId_ = 1; + bool initialized_ = false; + std::map active_; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_RESOURCE_ALLOCATOR_H diff --git a/src/comm/ccu/tilexr_ccu_runtime.cpp b/src/comm/ccu/tilexr_ccu_runtime.cpp new file mode 100644 index 00000000..138b7cbd --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_runtime.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_runtime.h" + +#include + +#include + +namespace TileXR { + +static_assert(RT_CCU_SQE_ARGS_LEN == TILEXR_CCU_SQE_ARGS_LEN, "TileXR CCU SQE arg count must match CANN runtime"); + +namespace { + +TileXRCcuTask CopyRuntimeTask(const rtCcuTaskInfo_t& runtimeTask) +{ + TileXRCcuTask task {}; + task.dieId = runtimeTask.dieId; + task.missionId = runtimeTask.missionId; + task.timeout = runtimeTask.timeout; + task.instStartId = runtimeTask.instStartId; + task.instCnt = runtimeTask.instCnt; + task.key = runtimeTask.key; + task.argSize = runtimeTask.argSize; + std::memcpy(task.args, runtimeTask.args, sizeof(task.args)); + return task; +} + +} // namespace + +int TileXRCcuValidateTask(const TileXRCcuTask& task) +{ + if (task.instCnt == RT_CCU_INST_CNT_INVALID) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (task.instStartId >= RT_CCU_INST_START_MAX) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (static_cast(task.instStartId) + static_cast(task.instCnt) > RT_CCU_INST_START_MAX) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (task.argSize != 1 && task.argSize != TILEXR_CCU_SQE_ARGS_LEN) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuSubmitTaskWithReport( + const TileXRCcuTask& task, + void* stream, + TileXRCcuRuntimeSubmitReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuRuntimeSubmitReport{}; + } + if (stream == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + int ret = TileXRCcuValidateTask(task); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + rtCcuTaskInfo_t runtimeTask {}; + runtimeTask.dieId = task.dieId; + runtimeTask.missionId = task.missionId; + runtimeTask.timeout = task.timeout; + runtimeTask.instStartId = task.instStartId; + runtimeTask.instCnt = task.instCnt; + runtimeTask.key = task.key; + runtimeTask.argSize = task.argSize; + std::memcpy(runtimeTask.args, task.args, sizeof(runtimeTask.args)); + + const TileXRCcuTask finalTask = CopyRuntimeTask(runtimeTask); + if (report != nullptr) { + report->finalTaskCaptured = true; + report->finalTask = finalTask; + } + + rtError_t launchRet = rtCCULaunch(&runtimeTask, stream); + if (report != nullptr) { + report->runtimeLaunchAttempted = true; + report->runtimeRet = static_cast(launchRet); + } + return launchRet == RT_ERROR_NONE ? TILEXR_SUCCESS : TILEXR_ERROR_MKIRT; +} + +int TileXRCcuSubmitTask(const TileXRCcuTask& task, void* stream) +{ + return TileXRCcuSubmitTaskWithReport(task, stream, nullptr); +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_runtime.h b/src/comm/ccu/tilexr_ccu_runtime.h new file mode 100644 index 00000000..7d406990 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_runtime.h @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_RUNTIME_H +#define TILEXR_CCU_RUNTIME_H + +#include + +#include "tilexr_types.h" + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_SQE_ARGS_LEN = 13U; +constexpr uint16_t TILEXR_CCU_DEFAULT_TASK_TIMEOUT_SEC = 120U; + +struct TileXRCcuTask { + uint8_t dieId = 0; + uint8_t missionId = 0; + uint16_t timeout = 0; + uint16_t instStartId = 0; + uint16_t instCnt = 0; + uint32_t key = 0; + uint32_t argSize = 0; + uint64_t args[TILEXR_CCU_SQE_ARGS_LEN] = {}; +}; + +struct TileXRCcuRuntimeSubmitReport { + bool runtimeLaunchAttempted = false; + int32_t runtimeRet = 0; + bool finalTaskCaptured = false; + TileXRCcuTask finalTask; +}; + +int TileXRCcuValidateTask(const TileXRCcuTask& task); + +int TileXRCcuSubmitTaskWithReport( + const TileXRCcuTask& task, + void* stream, + TileXRCcuRuntimeSubmitReport* report); + +int TileXRCcuSubmitTask(const TileXRCcuTask& task, void* stream); + +} // namespace TileXR + +#endif // TILEXR_CCU_RUNTIME_H diff --git a/src/comm/ccu/tilexr_ccu_specs.cpp b/src/comm/ccu/tilexr_ccu_specs.cpp new file mode 100644 index 00000000..06cdcdec --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_specs.cpp @@ -0,0 +1,178 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_specs.h" + +#include + +namespace TileXR { +namespace { + +constexpr uint32_t TILEXR_CCU_MOVE_16_BITS = 16; +constexpr uint32_t TILEXR_CCU_MOVE_24_BITS = 24; + +void ResetReport(TileXRCcuSpecsReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRCcuSpecsReport{}; +} + +int Fail(TileXRCcuSpecsReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +uint32_t Low16PlusOne(uint32_t value) +{ + return (value & 0x0000FFFFU) + 1U; +} + +uint32_t High16PlusOne(uint32_t value) +{ + return ((value >> TILEXR_CCU_MOVE_16_BITS) & 0x0000FFFFU) + 1U; +} + +bool FitsU16(uint32_t value) +{ + return value <= std::numeric_limits::max(); +} + +bool WindowOverflows(uint16_t start, uint32_t count) +{ + return count == 0 || count > std::numeric_limits::max() || + static_cast(start) + count > + static_cast(std::numeric_limits::max()) + 1U; +} + +void FillReport(const TileXRCcuSpecInfo& info, TileXRCcuSpecsReport* report) +{ + if (report == nullptr) { + return; + } + report->instructionNum = info.instructionNum; + report->xnNum = info.xnNum; + report->ckeNum = info.ckeNum; + report->channelNum = info.channelNum; + report->missionNum = info.missionNum; + report->message = "ok"; +} + +} // namespace + +int TileXRCcuDecodeBasicInfo( + const TileXRCcuBasicInfo& basicInfo, + TileXRCcuSpecInfo* specInfo, + TileXRCcuSpecsReport* report) +{ + ResetReport(report); + if (specInfo == nullptr) { + return Fail(report, "missing output CCU spec info"); + } + *specInfo = TileXRCcuSpecInfo{}; + + if (basicInfo.missionKey == 0) { + return Fail(report, "missing CCU mission key in basic info"); + } + if (basicInfo.resourceAddr == 0) { + return Fail(report, "missing CCU resource address in basic info"); + } + if (basicInfo.resourceAddr > std::numeric_limits::max() - TILEXR_CCU_V1_XN_RESOURCE_OFFSET) { + return Fail(report, "CCU XN base address overflows"); + } + + TileXRCcuSpecInfo decoded; + decoded.dieId = basicInfo.dieId; + decoded.msId = basicInfo.msId; + decoded.missionKey = basicInfo.missionKey; + decoded.resourceAddr = basicInfo.resourceAddr; + decoded.xnBaseAddr = basicInfo.resourceAddr + TILEXR_CCU_V1_XN_RESOURCE_OFFSET; + + decoded.instructionNum = Low16PlusOne(basicInfo.caps.cap0); + decoded.xnNum = High16PlusOne(basicInfo.caps.cap1); + decoded.gsaNum = Low16PlusOne(basicInfo.caps.cap1); + decoded.msNum = High16PlusOne(basicInfo.caps.cap2); + decoded.ckeNum = Low16PlusOne(basicInfo.caps.cap2); + decoded.jettyNum = High16PlusOne(basicInfo.caps.cap3); + decoded.channelNum = Low16PlusOne(basicInfo.caps.cap3); + decoded.pfeNum = (basicInfo.caps.cap4 & 0x000000FFU) + 1U; + decoded.missionNum = ((basicInfo.caps.cap0 >> TILEXR_CCU_MOVE_16_BITS) & 0x000000FFU) + 1U; + decoded.loopEngineNum = ((basicInfo.caps.cap0 >> TILEXR_CCU_MOVE_24_BITS) & 0x000000FFU) + 1U; + + *specInfo = decoded; + FillReport(decoded, report); + return TILEXR_SUCCESS; +} + +int TileXRCcuBuildResourceSpec( + const TileXRCcuSpecInfo& specInfo, + uint16_t missionStartId, + uint16_t instructionStartId, + uint16_t xnStartId, + uint16_t ckeStartId, + uint16_t channelStartId, + TileXRCcuResourceSpec* resourceSpec, + TileXRCcuSpecsReport* report, + uint16_t gsaStartId) +{ + ResetReport(report); + if (resourceSpec == nullptr) { + return Fail(report, "missing output CCU resource spec"); + } + *resourceSpec = TileXRCcuResourceSpec{}; + + if (specInfo.missionKey == 0) { + return Fail(report, "missing CCU mission key in spec info"); + } + if (!FitsU16(specInfo.missionNum) || !FitsU16(specInfo.instructionNum) || !FitsU16(specInfo.xnNum) || + !FitsU16(specInfo.gsaNum) || + !FitsU16(specInfo.ckeNum) || !FitsU16(specInfo.channelNum)) { + return Fail(report, "CCU resource count exceeds TileXR resource window capacity"); + } + if (WindowOverflows(missionStartId, specInfo.missionNum)) { + return Fail(report, "mission resource window overflows"); + } + if (WindowOverflows(instructionStartId, specInfo.instructionNum)) { + return Fail(report, "instruction resource window overflows"); + } + if (WindowOverflows(xnStartId, specInfo.xnNum)) { + return Fail(report, "XN resource window overflows"); + } + if (gsaStartId != 0 && WindowOverflows(gsaStartId, specInfo.gsaNum)) { + return Fail(report, "GSA resource window overflows"); + } + if (WindowOverflows(ckeStartId, specInfo.ckeNum)) { + return Fail(report, "CKE resource window overflows"); + } + if (WindowOverflows(channelStartId, specInfo.channelNum)) { + return Fail(report, "channel resource window overflows"); + } + + TileXRCcuResourceSpec result; + result.dieId = specInfo.dieId; + result.missionKey = specInfo.missionKey; + result.missionStartId = missionStartId; + result.missionCount = static_cast(specInfo.missionNum); + result.instructionStartId = instructionStartId; + result.instructionCount = static_cast(specInfo.instructionNum); + result.xnStartId = xnStartId; + result.xnCount = static_cast(specInfo.xnNum); + result.gsaStartId = gsaStartId; + result.gsaCount = gsaStartId == 0 ? 0 : static_cast(specInfo.gsaNum); + result.ckeStartId = ckeStartId; + result.ckeCount = static_cast(specInfo.ckeNum); + result.channelStartId = channelStartId; + result.channelCount = static_cast(specInfo.channelNum); + + *resourceSpec = result; + FillReport(specInfo, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_specs.h b/src/comm/ccu/tilexr_ccu_specs.h new file mode 100644 index 00000000..75674e41 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_specs.h @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_SPECS_H +#define TILEXR_CCU_SPECS_H + +#include "ccu/tilexr_ccu_resource_allocator.h" + +#include +#include + +namespace TileXR { + +constexpr uint64_t TILEXR_CCU_V1_CCUM_OFFSET = 0x800000ULL; +constexpr uint64_t TILEXR_CCU_RESOURCE_INS_RESERVE_SIZE = 0x100000ULL; +constexpr uint64_t TILEXR_CCU_V1_RESOURCE_GSA_RESERVE_SIZE = 0x8000ULL; +constexpr uint64_t TILEXR_CCU_V1_XN_RESOURCE_OFFSET = + TILEXR_CCU_V1_CCUM_OFFSET + TILEXR_CCU_RESOURCE_INS_RESERVE_SIZE + TILEXR_CCU_V1_RESOURCE_GSA_RESERVE_SIZE; +constexpr uint64_t TILEXR_CCU_RESOURCE_WINDOW_BYTES = 72ULL * 1024ULL * 1024ULL; + +struct TileXRCcuCaps { + uint32_t cap0 = 0; + uint32_t cap1 = 0; + uint32_t cap2 = 0; + uint32_t cap3 = 0; + uint32_t cap4 = 0; +}; + +struct TileXRCcuMsidTokenInfo { + uint32_t tokenId = 0; + uint32_t tokenValue = 0; + bool valid = false; +}; + +struct TileXRCcuBasicInfo { + uint8_t dieId = 0; + uint32_t msId = 0; + TileXRCcuMsidTokenInfo msidToken; + uint32_t missionKey = 0; + uint64_t resourceAddr = 0; + TileXRCcuCaps caps; +}; + +struct TileXRCcuSpecInfo { + uint8_t dieId = 0; + uint32_t msId = 0; + uint32_t missionKey = 0; + uint64_t resourceAddr = 0; + uint64_t xnBaseAddr = 0; + uint32_t loopEngineNum = 0; + uint32_t missionNum = 0; + uint32_t instructionNum = 0; + uint32_t xnNum = 0; + uint32_t gsaNum = 0; + uint32_t msNum = 0; + uint32_t ckeNum = 0; + uint32_t jettyNum = 0; + uint32_t channelNum = 0; + uint32_t pfeNum = 0; +}; + +struct TileXRCcuSpecsReport { + uint32_t instructionNum = 0; + uint32_t xnNum = 0; + uint32_t ckeNum = 0; + uint32_t channelNum = 0; + uint32_t missionNum = 0; + std::string message; +}; + +int TileXRCcuDecodeBasicInfo( + const TileXRCcuBasicInfo& basicInfo, + TileXRCcuSpecInfo* specInfo, + TileXRCcuSpecsReport* report); + +int TileXRCcuBuildResourceSpec( + const TileXRCcuSpecInfo& specInfo, + uint16_t missionStartId, + uint16_t instructionStartId, + uint16_t xnStartId, + uint16_t ckeStartId, + uint16_t channelStartId, + TileXRCcuResourceSpec* resourceSpec, + TileXRCcuSpecsReport* report, + uint16_t gsaStartId = 0); + +} // namespace TileXR + +#endif // TILEXR_CCU_SPECS_H diff --git a/src/comm/comm_wrap.cpp b/src/comm/comm_wrap.cpp index 27b18716..4eb717dc 100644 --- a/src/comm/comm_wrap.cpp +++ b/src/comm/comm_wrap.cpp @@ -11,7 +11,10 @@ #include #include #include +#include #include +#include +#include #include #include "tilexr_comm.h" @@ -21,6 +24,180 @@ using namespace std; using namespace TileXR; +namespace { + +constexpr const char* TILEXR_DIRECT_CCU_PUBLIC_PROVIDER = "tilexr-public-direct-ccu"; +constexpr const char* TILEXR_DIRECT_CCU_BARRIER_MODE_ENV = "TILEXR_CCU_DIRECT_BARRIER_MODE"; +constexpr uint32_t TILEXR_DIRECT_CCU_MEMORY_COPY_INSTRUCTION_COUNT = 7U; + +struct TileXRDirectCcuPreparedTasks { + ~TileXRDirectCcuPreparedTasks() + { + (void)TileXRCcuReleaseDirectInstallAttemptResources(attempt); + } + + TileXRCcuDirectInstallAttempt attempt; +}; + +void CopyDirectCcuMessage(const std::string& message, char* output) +{ + if (output == nullptr) { + return; + } + std::memset(output, 0, TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES); + if (message.empty()) { + return; + } + std::strncpy(output, message.c_str(), TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES - 1); +} + +void FillPublicPrepareReport( + const TileXRCcuDirectInstallReport& source, + TileXRDirectCcuPrepareReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRDirectCcuPrepareReport {}; + report->pipelineBuilt = source.pipelineBuilt; + report->installAttempted = source.installAttempted; + report->installSucceeded = source.installSucceeded; + report->submitReady = source.submitReady; + report->requiredInstallSurfaceCount = source.requiredInstallSurfaceCount; + report->publicVerifiedInstallSurfaceCount = source.publicVerifiedInstallSurfaceCount; + report->missingInstallSurfaceCount = source.missingInstallSurfaceCount; + report->taskCount = source.taskCount; + report->submitTaskCount = source.submitTaskCount; + CopyDirectCcuMessage(source.message, report->message); +} + +void FillPublicSubmitReport( + const TileXRCcuDirectSubmitReport& source, + TileXRDirectCcuSubmitReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRDirectCcuSubmitReport {}; + report->submitted = source.submitted; + report->taskCount = source.taskCount; + report->submittedTaskCount = source.submittedTaskCount; + CopyDirectCcuMessage(source.message, report->message); +} + +void FillPublicInstructionReadbackReport( + uint32_t readInstructionCount, + const TileXRCcuDriverAdapterReport& source, + TileXRDirectCcuInstructionReadbackReport* report) +{ + if (report == nullptr) { + return; + } + *report = TileXRDirectCcuInstructionReadbackReport {}; + report->readbackAttempted = true; + report->readInstructionCount = readInstructionCount; + report->opcode = source.opcode; + report->driverRet = source.driverRet; + report->opRet = source.opRet; + CopyDirectCcuMessage(source.message, report->message); +} + +TileXRCcuRepositoryInstallWindow RepositoryInstallWindowFromPublic(uint32_t value) +{ + return value == TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY ? + TileXRCcuRepositoryInstallWindow::FullRepository : + TileXRCcuRepositoryInstallWindow::Mission; +} + +TileXRCcuRepositoryInstallDataLenMode RepositoryInstallDataLenModeFromPublic(uint32_t value) +{ + return value == TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES ? + TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes : + TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; +} + +TileXRCcuRepositoryMemoryAllocMode RepositoryMemoryAllocModeFromPublic(uint32_t value) +{ + if (value == TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM) { + return TileXRCcuRepositoryMemoryAllocMode::RtHbm; + } + return value == TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3 ? + TileXRCcuRepositoryMemoryAllocMode::AclModule3 : + TileXRCcuRepositoryMemoryAllocMode::Acl; +} + +TileXRCcuInstallOrder InstallOrderFromPublic(uint32_t value) +{ + return value == TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST ? + TileXRCcuInstallOrder::InstallLowerLayerFirst : + TileXRCcuInstallOrder::RepositoryFirst; +} + +TileXRCcuDirectInstallOptions MakeDirectCcuOptions(const TileXRDirectCcuPrepareOptions& publicOptions) +{ + TileXRCcuDirectInstallOptions options; + options.syncResourceCount = publicOptions.syncResourceCount; + options.sqeArgCount = publicOptions.sqeArgCount; + options.syncInstructionCount = publicOptions.syncInstructionCount; + options.bindingsPerSyncResource = publicOptions.bindingsPerSyncResource; + options.missionStartId = publicOptions.missionStartId; + options.instructionStartId = publicOptions.instructionStartId; + options.missionInstructionStartId = publicOptions.missionInstructionStartId; + options.xnStartId = publicOptions.xnStartId; + options.gsaStartId = publicOptions.gsaStartId; + options.remoteXnStartId = publicOptions.remoteXnStartId; + options.remoteXnCount = publicOptions.remoteXnCount; + options.ckeStartId = publicOptions.ckeStartId; + options.channelStartId = publicOptions.channelStartId; + options.localWaitCkeStartId = publicOptions.localWaitCkeStartId; + options.localWaitCkeCount = publicOptions.localWaitCkeCount; + options.remoteNotifyCkeStartId = publicOptions.remoteNotifyCkeStartId; + options.remoteNotifyCkeCount = publicOptions.remoteNotifyCkeCount; + options.repositoryInstallOptions.window = + RepositoryInstallWindowFromPublic(publicOptions.repositoryInstallWindow); + options.repositoryInstallOptions.dataLenMode = + RepositoryInstallDataLenModeFromPublic(publicOptions.repositoryInstallDataLenMode); + options.repositoryMemoryAllocMode = + RepositoryMemoryAllocModeFromPublic(publicOptions.repositoryMemoryAllocMode); + options.installOrder = InstallOrderFromPublic(publicOptions.installOrder); + options.deviceId = publicOptions.deviceId; + options.rank = publicOptions.rank; + const char* barrierMode = std::getenv(TILEXR_DIRECT_CCU_BARRIER_MODE_ENV); + if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke") == 0) { + options.barrierMode = TileXRCcuBarrierMode::SyncCke; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke_set_wait") == 0) { + options.barrierMode = TileXRCcuBarrierMode::SyncCkeSetWait; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke_post_only") == 0) { + options.barrierMode = TileXRCcuBarrierMode::SyncCkePostOnly; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "local_cke") == 0) { + options.barrierMode = TileXRCcuBarrierMode::LocalCke; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "local_cke_post_only") == 0) { + options.barrierMode = TileXRCcuBarrierMode::LocalCkePostOnly; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_xn_post_only") == 0) { + options.barrierMode = TileXRCcuBarrierMode::SyncXnPostOnly; + } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_xn_load_post_only") == 0) { + options.barrierMode = TileXRCcuBarrierMode::SyncXnLoadPostOnly; + } + options.provider = (publicOptions.provider == nullptr || publicOptions.provider[0] == '\0') ? + TILEXR_DIRECT_CCU_PUBLIC_PROVIDER : + publicOptions.provider; + return options; +} + +TileXRDirectCcuPreparedTasks* PreparedHandle(TileXRDirectCcuPreparedTasksPtr prepared) +{ + return static_cast(prepared); +} + +TileXRCcuMemoryCopyDirection MemoryCopyDirectionFromPublic(uint32_t direction) +{ + return direction == TILEXR_DIRECT_CCU_MEMORY_COPY_LOCAL_TO_REMOTE ? + TileXRCcuMemoryCopyDirection::LocalToRemote : + TileXRCcuMemoryCopyDirection::RemoteToLocal; +} + +} // namespace + int TileXRCommInitRankLocal(int rankSize, int rank, TileXRCommPtr *comm) { TILEXR_LOG(INFO) << "using tilexr c++ api! rank" << rank; @@ -115,6 +292,29 @@ int TileXRCommInitRankWithDomain(int commDomain, int rankSize, int rank, TileXRC return TileXRCommInitRankWithCustDomainSize(commDomain, minBufferSize, rankSize, rank, comm); } +int TileXRCommInitRankDirectCcuWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm) +{ + TILEXR_LOG(INFO) << "using tilexr direct CCU only api! rank" << rank; + if (comm == nullptr) { + TILEXR_LOG(ERROR) << "tilexr direct CCU only comm ptr is nullptr!"; + return TILEXR_ERROR_INTERNAL; + } + *comm = nullptr; + unique_ptr c(new (std::nothrow) TileXRComm(rank, rankSize, commDomain, TILEXR_COMM_BUFFER_SIZE)); + if (c == nullptr) { + TILEXR_LOG(ERROR) << "TileXRComm create failed for direct CCU only init. rank : " + << rank << ", rankSize : " << rankSize; + return TILEXR_ERROR_INTERNAL; + } + int ret = c->InitDirectCcuOnly(); + if (ret != TILEXR_SUCCESS) { + TILEXR_LOG(ERROR) << "tilexr direct CCU only init failed! ret: " << ret; + return ret; + } + *comm = c.release(); + return TILEXR_SUCCESS; +} + int TileXRGetCommArgsDev(TileXRCommPtr comm, GM_ADDR &commArgsPtr) { if (comm == nullptr) { @@ -212,6 +412,254 @@ int TileXRGetSDMAWorkspaceDev(TileXRCommPtr comm, GM_ADDR *workspace) return TILEXR_SUCCESS; } +int TileXRCommPrepareDirectCcu( + TileXRCommPtr comm, + const TileXRDirectCcuPrepareOptions *options, + TileXRDirectCcuPreparedTasksPtr *prepared, + TileXRDirectCcuPrepareReport *report) +{ + if (prepared != nullptr) { + *prepared = nullptr; + } + if (report != nullptr) { + *report = TileXRDirectCcuPrepareReport {}; + } + if (comm == nullptr || options == nullptr || prepared == nullptr) { + TILEXR_LOG(ERROR) << "TileXRCommPrepareDirectCcu invalid input"; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); + if (handle == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + + auto* tilexr = static_cast(comm); + TileXRCcuDirectInstallReport internalReport; + TileXRCcuDirectInstallOptions internalOptions = MakeDirectCcuOptions(*options); + const int ret = tilexr->PrepareDirectCcuInstallAttempt(internalOptions, &handle->attempt, &internalReport); + FillPublicPrepareReport(internalReport, report); + const bool installDiagnosticReady = + internalReport.pipelineBuilt && + internalReport.installAttempted && + internalReport.installSucceeded && + !internalReport.submitReady; + if (ret != TILEXR_SUCCESS && !installDiagnosticReady) { + return ret; + } + + *prepared = handle.release(); + return TILEXR_SUCCESS; +} + +int TileXRCommPrepareDirectCcuMemoryCopy( + TileXRCommPtr comm, + const TileXRDirectCcuMemoryCopyPrepareOptions *options, + TileXRDirectCcuPreparedTasksPtr *prepared, + TileXRDirectCcuPrepareReport *report) +{ + if (prepared != nullptr) { + *prepared = nullptr; + } + if (report != nullptr) { + *report = TileXRDirectCcuPrepareReport {}; + } + if (comm == nullptr || options == nullptr || prepared == nullptr) { + TILEXR_LOG(ERROR) << "TileXRCommPrepareDirectCcuMemoryCopy invalid input"; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); + if (handle == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + + TileXRCcuDirectInstallOptions internalOptions = MakeDirectCcuOptions(options->prepare); + internalOptions.sqeArgCount = 0; + internalOptions.syncResourceCount = 1; + internalOptions.syncInstructionCount = std::max( + internalOptions.syncInstructionCount, + TILEXR_DIRECT_CCU_MEMORY_COPY_INSTRUCTION_COUNT); + internalOptions.bindingsPerSyncResource = + internalOptions.bindingsPerSyncResource == 0 ? 1 : internalOptions.bindingsPerSyncResource; + const TileXRCcuMemoryCopyDirection direction = MemoryCopyDirectionFromPublic(options->direction); + + auto* tilexr = static_cast(comm); + TileXRCcuDirectInstallReport internalReport; + const int ret = tilexr->PrepareDirectCcuMemoryCopyInstallAttempt( + internalOptions, + options->localSourceAddr, + options->localDestinationAddr, + options->bytes, + options->peerRank, + direction, + &handle->attempt, + &internalReport); + FillPublicPrepareReport(internalReport, report); + const bool installDiagnosticReady = + internalReport.pipelineBuilt && + internalReport.installAttempted && + internalReport.installSucceeded && + !internalReport.submitReady; + if (ret != TILEXR_SUCCESS && !installDiagnosticReady) { + return ret; + } + + *prepared = handle.release(); + return TILEXR_SUCCESS; +} + +int TileXRDirectCcuGetPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t index, + TileXRDirectCcuTaskInfo *task) +{ + if (prepared == nullptr || task == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* handle = PreparedHandle(prepared); + if (index >= handle->attempt.submitTasks.size()) { + return TILEXR_ERROR_NOT_FOUND; + } + + const TileXRCcuTask& source = handle->attempt.submitTasks[index]; + *task = TileXRDirectCcuTaskInfo {}; + task->dieId = source.dieId; + task->missionId = source.missionId; + task->timeout = source.timeout; + task->instStartId = source.instStartId; + task->instCnt = source.instCnt; + task->key = source.key; + task->argSize = source.argSize; + std::memcpy(task->args, source.args, sizeof(task->args)); + return TILEXR_SUCCESS; +} + +int TileXRDirectCcuSubmitPrepared( + TileXRDirectCcuPreparedTasksPtr prepared, + void *stream, + TileXRDirectCcuSubmitReport *report) +{ + if (report != nullptr) { + *report = TileXRDirectCcuSubmitReport {}; + } + if (prepared == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* handle = PreparedHandle(prepared); + TileXRCcuDirectSubmitReport internalReport; + const int ret = TileXRCcuSubmitPreparedTasks( + handle->attempt.submitTasks, + stream, + nullptr, + nullptr, + &internalReport); + FillPublicSubmitReport(internalReport, report); + return ret; +} + +int TileXRDirectCcuSubmitPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t index, + void *stream, + TileXRDirectCcuSubmitReport *report) +{ + if (report != nullptr) { + *report = TileXRDirectCcuSubmitReport {}; + } + if (prepared == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* handle = PreparedHandle(prepared); + if (index >= handle->attempt.submitTasks.size()) { + if (report != nullptr) { + report->taskCount = static_cast(handle->attempt.submitTasks.size()); + CopyDirectCcuMessage("selected direct CCU submit task is missing", report->message); + } + return TILEXR_ERROR_NOT_FOUND; + } + + std::vector selectedTasks(1U); + selectedTasks[0] = handle->attempt.submitTasks[index]; + TileXRCcuDirectSubmitReport internalReport; + const int ret = TileXRCcuSubmitPreparedTasks( + selectedTasks, + stream, + nullptr, + nullptr, + &internalReport); + FillPublicSubmitReport(internalReport, report); + return ret; +} + +int TileXRCommReadDirectCcuInstructions( + TileXRCommPtr comm, + uint8_t dieId, + uint16_t instructionStartId, + uint32_t instructionCount, + TileXRDirectCcuInstructionWords *instructions, + TileXRDirectCcuInstructionReadbackReport *report) +{ + if (report != nullptr) { + *report = TileXRDirectCcuInstructionReadbackReport {}; + } + if (comm == nullptr || instructions == nullptr || instructionCount == 0 || + instructionCount > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* tilexr = static_cast(comm); + TileXRCcuDriverAdapterReport internalReport; + const uint32_t instructionBytes = + instructionCount * static_cast(sizeof(TileXRDirectCcuInstructionWords)); + const int ret = tilexr->ReadDirectCcuInstructionsForDebug( + dieId, + instructionStartId, + instructions, + instructionCount, + instructionBytes, + &internalReport); + FillPublicInstructionReadbackReport(instructionCount, internalReport, report); + return ret; +} + +#if defined(TILEXR_CCU_TESTING) +extern "C" TileXRDirectCcuPreparedTasksPtr TileXRDirectCcuCreatePreparedForTest( + const TileXRDirectCcuTaskInfo* tasks, + uint32_t taskCount) +{ + if (tasks == nullptr || taskCount == 0) { + return nullptr; + } + std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); + if (handle == nullptr) { + return nullptr; + } + handle->attempt.submitTasks.reserve(taskCount); + for (uint32_t i = 0; i < taskCount; ++i) { + TileXRCcuTask task {}; + task.dieId = tasks[i].dieId; + task.missionId = tasks[i].missionId; + task.timeout = tasks[i].timeout; + task.instStartId = tasks[i].instStartId; + task.instCnt = tasks[i].instCnt; + task.key = tasks[i].key; + task.argSize = tasks[i].argSize; + std::memcpy(task.args, tasks[i].args, sizeof(task.args)); + handle->attempt.submitTasks.push_back(task); + } + return handle.release(); +} +#endif + +int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared) +{ + if (prepared == nullptr) { + return TILEXR_INVALID_VALUE; + } + delete PreparedHandle(prepared); + return TILEXR_SUCCESS; +} + void TileXRPrintDFX2Log(TileXRCommPtr comm) { if (comm == nullptr) { diff --git a/src/comm/tilexr_comm.cpp b/src/comm/tilexr_comm.cpp index f627b9fd..7c01300d 100644 --- a/src/comm/tilexr_comm.cpp +++ b/src/comm/tilexr_comm.cpp @@ -9,6 +9,8 @@ */ #include "tilexr_comm.h" #include "tilexr_internal.h" +#include "ccu/tilexr_ccu_memory_program.h" +#include "ccu/tilexr_ccu_repository.h" #include "sdma/tilexr_sdma_transport.h" #include "udma/tilexr_udma_transport.h" @@ -22,13 +24,14 @@ #include #include #include +#include #include "tilexr_log.h" #include "tools/socket/tilexr_sock_exchange.h" -#include "runtime/kernel.h" #include "runtime/mem.h" #include "runtime/dev.h" +#include "runtime/rts/rts_device.h" #include "runtime/rt_ffts.h" enum TopologyType : int { @@ -47,14 +50,145 @@ using namespace chrono; namespace TileXR { constexpr int HCCL_IPC_PID_ARRAY_SIZE = 1; // 固定每次只传一个PID数据 constexpr int TILEXR_INIT_TIMEOUT = 600; +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; static map g_localPeerMemMap; static map g_devList; +struct TileXRThreadAllGatherState { + std::vector data[TILEXR_MAX_RANK_SIZE]; + uint64_t arrivals = 0; + uint64_t departures = 0; + size_t bytes = 0; +}; +static map g_directCcuAllGatherStates; static std::mutex g_mtx; static std::mutex g_udmaMtx; static bool g_udmaUnavailable = false; +static std::mutex g_ccuDirectRuntimeMtx; +static bool g_ccuDirectRuntimeUnavailable = false; +static std::string g_ccuDirectRuntimeUnavailableMessage; static std::mutex g_sdmaMtx; static bool g_sdmaUnavailable = false; +uint8_t SelectDirectCcuInstallDieId() +{ + const char *text = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); + if (text == nullptr || text[0] == '\0') { + return 0; + } + char *end = nullptr; + const unsigned long parsed = std::strtoul(text, &end, 0); + if (end == text || *end != '\0' || parsed > 1UL) { + return 0; + } + return static_cast(parsed); +} + +uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +{ + if (peerRouteCount == 0) { + return 0; + } + return static_cast(peerLocalIndex) + + static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); +} + +uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +{ + return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); +} + +uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) +{ + if (syncRouteCount == 0) { + return 0; + } + return syncRouteCount; +} + +uint16_t SelectDirectCcuChannelBoundRemoteXnId( + uint16_t remoteXnStartId, + size_t peerLocalIndex, + uint32_t syncIndex, + size_t peerRouteCount) +{ + return static_cast( + static_cast(remoteXnStartId) + + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); +} + +uint16_t SelectDirectCcuRemoteNotifyCkeId(uint16_t remoteNotifyCkeStartId, size_t routeIndex) +{ + return static_cast(static_cast(remoteNotifyCkeStartId) + routeIndex); +} + +std::string ProcessDirectCcuRuntimeUnavailableMessage() +{ + lock_guard lock(g_ccuDirectRuntimeMtx); + if (!g_ccuDirectRuntimeUnavailable) { + return {}; + } + return g_ccuDirectRuntimeUnavailableMessage.empty() ? + "direct CCU runtime unavailable after process-level init failure" : + "direct CCU runtime unavailable after process-level init failure: " + + g_ccuDirectRuntimeUnavailableMessage; +} + +struct DirectCcuMemoryCopyEndpoint { + uint64_t sourceAddr = 0; + uint64_t sourceToken = 0; + uint64_t destinationAddr = 0; + uint64_t destinationToken = 0; + uint64_t bytes = 0; + uint32_t rank = 0; + uint32_t valid = 0; +}; + +int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) +{ + if (addr == 0 || bytes == 0 || packedToken == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *packedToken = 0; + rtMemUbTokenInfo info {}; + info.va = addr; + info.size = bytes; + const rtError_t ret = rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info); + if (ret != RT_ERROR_NONE) { + return TILEXR_ERROR_MKIRT; + } + constexpr uint32_t tokenIdRightShift = 8U; + const uint32_t tokenId = info.tokenId >> tokenIdRightShift; + *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); + return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; +} + +int BuildDirectCcuLocalMemoryCopyEndpoint( + uint32_t rank, + uint64_t sourceAddr, + uint64_t destinationAddr, + uint64_t bytes, + DirectCcuMemoryCopyEndpoint *endpoint) +{ + if (endpoint == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *endpoint = DirectCcuMemoryCopyEndpoint {}; + endpoint->rank = rank; + endpoint->bytes = bytes; + endpoint->sourceAddr = sourceAddr; + endpoint->destinationAddr = destinationAddr; + int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + endpoint->valid = 1; + return TILEXR_SUCCESS; +} + // 如果是互联的链路，返回false； 对910B2C那些不互联的链路，返回true bool SkipUnusedChannel910B2C(int curRank, int peerRank, ChipName chipName) @@ -170,6 +304,63 @@ int TileXRComm::InitUDMA() return TILEXR_SUCCESS; } +int TileXRComm::InitDirectCcuRuntime() +{ + if (rankSize_ <= 1) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped for single-rank communicator"; + return TILEXR_SUCCESS; + } + + lock_guard lock(g_ccuDirectRuntimeMtx); + if (g_ccuDirectRuntimeUnavailable) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; + return TILEXR_SUCCESS; + } + + ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime()); + if (ccuDirectRuntime_ == nullptr) { + TILEXR_LOG(WARN) << "TileXRCcuDirectRuntime allocation failed, direct CCU disabled"; + return TILEXR_SUCCESS; + } + + TileXRCcuDirectRuntimeOptions options {}; + options.rank = rank_; + options.rankSize = rankSize_; + options.devId = devId_; + options.allGather = &TileXRComm::DirectCcuAllGatherCallback; + options.allGatherUserData = this; + TileXRCcuDirectRuntimeReport runtimeReport; + const int ret = ccuDirectRuntime_->Init(options, &runtimeReport); + if (ret != TILEXR_SUCCESS || !ccuDirectRuntime_->IsAvailable()) { + TILEXR_LOG(WARN) << "TileXR direct CCU runtime init failed: " << ret + << ", logicDevId " << runtimeReport.logicDevId + << ", devicePhyId " << runtimeReport.devicePhyId + << ", hdcType " << runtimeReport.hdcType + << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) + << ", " << runtimeReport.message << ", direct CCU disabled"; + g_ccuDirectRuntimeUnavailable = true; + g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; + ResetDirectCcuBasicInfo(); + ccuDirectRuntime_.reset(); + return TILEXR_SUCCESS; + } + + const int ccuInfoRet = RefreshDirectCcuBasicInfo(0); + if (ccuInfoRet != TILEXR_SUCCESS && ccuInfoRet != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU basic info refresh failed after runtime init: " << ccuInfoRet + << ", " << directCcuBasicInfoReport_.message; + } + + TILEXR_LOG(INFO) << "InitDirectCcuRuntime success, rank " << rank_ << "/" << rankSize_ + << " logicDevId " << runtimeReport.logicDevId + << " devicePhyId " << runtimeReport.devicePhyId + << " hdcType " << runtimeReport.hdcType + << " raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << " ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0); + return TILEXR_SUCCESS; +} + int TileXRComm::InitSDMA() { { @@ -315,6 +506,23 @@ void TileXRComm::FreeUDMARegistry() udmaRegistry_ = TileXRUDMARegistry {}; } +void TileXRComm::ResetDirectCcuBasicInfo() +{ + directCcuBasicInfoValid_ = false; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuBasicInfo_ = TileXRCcuBasicInfo {}; + directCcuBasicInfoReport_ = TileXRCcuDriverAdapterReport {}; +} + +void TileXRComm::ResetDirectCcuLowerLayerPlan() +{ + directCcuLowerLayerPlanValid_ = false; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuLowerLayerSnapshot_ = TileXRCcuLowerLayerTransportSnapshot {}; + directCcuLowerLayerPlan_ = TileXRCcuLowerLayerInstallPlan {}; + directCcuLowerLayerPlanReport_ = TileXRCcuLowerLayerPlanBuilderReport {}; +} + int TileXRComm::RegisterUDMAMemory(GM_ADDR localPtr, size_t bytes, TileXRUDMAMemHandle *handle) { if (!inited_) { @@ -402,8 +610,10 @@ int TileXRComm::RegisterUDMAMemory(GM_ADDR localPtr, size_t bytes, TileXRUDMAMem udmaTransport_->UnregisterMemory(localPtr); udmaRegisteredPtr_ = nullptr; FreeUDMARegistry(); + return ret; } - return ret; + + return TILEXR_SUCCESS; } int TileXRComm::UnregisterUDMAMemory(TileXRUDMAMemHandle handle) @@ -432,6 +642,796 @@ const TileXRUDMARegistry* TileXRComm::GetUDMARegistryHost() const return UDMARegistryValid(&udmaRegistry_, rankSize_) ? &udmaRegistry_ : nullptr; } +int TileXRComm::RefreshDirectCcuBasicInfo(uint8_t dieId) +{ + ResetDirectCcuBasicInfo(); + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuBasicInfoReport_.message = "direct CCU runtime is unavailable for basic info"; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuBasicInfoStatus_; + } + + TileXRCcuBasicInfo basicInfo; + TileXRCcuDriverAdapterReport report; + const int ret = ccuDirectRuntime_->QueryBasicInfo(dieId, &basicInfo, &report); + directCcuBasicInfoReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuBasicInfoStatus_ = ret; + return directCcuBasicInfoStatus_; + } + + directCcuBasicInfo_ = basicInfo; + directCcuBasicInfoReport_.message = "direct CCU basic info cached"; + directCcuBasicInfoValid_ = true; + directCcuBasicInfoStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRComm::HasDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_; +} + +int TileXRComm::GetDirectCcuBasicInfoStatus() const +{ + return directCcuBasicInfoStatus_; +} + +const TileXRCcuBasicInfo *TileXRComm::GetDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_ ? &directCcuBasicInfo_ : nullptr; +} + +const TileXRCcuDriverAdapterReport &TileXRComm::GetDirectCcuBasicInfoReport() const +{ + return directCcuBasicInfoReport_; +} + +int TileXRComm::ConfigureDirectCcuLowerLayerTemplate( + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot) +{ + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes( + const std::vector &verifiedRoutes) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes = verifiedRoutes; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuVerifiedEndpointRoutes_ = verifiedRoutes; + if (directCcuLowerLayerTemplateConfigured_) { + return RefreshDirectCcuLowerLayerPlan(); + } + return TILEXR_SUCCESS; +} + +int TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute( + const TileXRCcuLowerLayerTransportRoute &route) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes.push_back(route); + TileXRCcuLowerLayerPlanBuilderReport report; + std::vector routes {route}; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(routes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; + directCcuLocalVerifiedEndpointRouteValid_ = false; + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuLocalVerifiedEndpointRoute_ = route; + directCcuLocalVerifiedEndpointRouteValid_ = true; + if (ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable()) { + return ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(route); + } + return TILEXR_SUCCESS; +} + +int TileXRComm::ConfigureDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers) +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuBasicInfoValid_) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuBuildLowerLayerTransportTemplate( + directCcuBasicInfo_, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation) +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuBasicInfoValid_) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuLowerLayerPlanReport_.message = + "direct CCU runtime is unavailable for resource window registration"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + int ret = ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to register direct CCU resource window"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLocalResourceWindowInfo localCcuResourceWindow; + ret = ccuDirectRuntime_->ExportLocalCcuRmaBuffer(&localCcuResourceWindow); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU local resource window token"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + if (directCcuLocalVerifiedEndpointRouteValid_) { + ret = ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(directCcuLocalVerifiedEndpointRoute_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to configure direct CCU local verified endpoint route"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + } else { + TileXRCcuDirectRuntimeReport endpointRouteReport; + ret = ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute(&endpointRouteReport); + if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU local endpoint route collection failed closed: " + << ret << ", " << endpointRouteReport.message; + } + } + + std::vector remoteCcuBuffers; + ret = ccuDirectRuntime_->ExportRemoteCcuRmaBuffers(&remoteCcuBuffers); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU peer resource window tokens"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = ExchangeDirectCcuRemoteNotifyCke(allocation, &remoteCcuBuffers, &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerTransportTemplate( + directCcuBasicInfo_, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + templateSnapshot.msidToken.dieId = directCcuBasicInfo_.dieId; + templateSnapshot.msidToken.msId = directCcuBasicInfo_.msId; + templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId; + templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue; + templateSnapshot.msidToken.valid = true; + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (plan == nullptr || report == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + int ret = PrepareDirectCcuLowerLayerTemplateFromAllocation(allocation); + if (ret != TILEXR_SUCCESS) { + *report = directCcuLowerLayerPlanReport_; + return ret; + } + if (!directCcuLowerLayerPlanValid_) { + *report = directCcuLowerLayerPlanReport_; + return TILEXR_ERROR_NOT_FOUND; + } + *plan = directCcuLowerLayerPlan_; + *report = directCcuLowerLayerPlanReport_; + return TILEXR_SUCCESS; +} + +int TileXRComm::ExchangeDirectCcuRemoteNotifyCke( + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (remoteCcuBuffers == nullptr) { + if (report != nullptr) { + report->message = "missing direct CCU remote notify CKE exchange inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize_ <= 1 || rank_ < 0 || rank_ >= rankSize_) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const size_t peerRouteCount = static_cast(rankSize_ - 1); + const size_t syncRouteCount = allocation.remoteXn.num; + if (allocation.localXn.num == 0 || + allocation.localWaitCke.num == 0 || + allocation.remoteNotifyCke.num == 0 || + allocation.remoteXn.num < static_cast(rankSize_ - 1) || + allocation.localWaitCke.num < allocation.remoteXn.num || + allocation.remoteNotifyCke.num < allocation.remoteXn.num || + allocation.channels.num == 0 || + remoteCcuBuffers->size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + struct PeerResourceExchange { + uint16_t localXnStartId; + uint16_t localXnCount; + uint16_t remoteXnStartId; + uint16_t remoteXnCount; + uint16_t localWaitCkeStartId; + uint16_t localWaitCkeCount; + uint16_t remoteNotifyCkeStartId; + uint16_t remoteNotifyCkeCount; + uint16_t channelStartId; + uint16_t channelCount; + }; + PeerResourceExchange local { + allocation.localXn.startId, + allocation.localXn.num, + allocation.remoteXn.startId, + DirectCcuRemoteXnProofSpan(allocation.remoteXn.num), + allocation.localWaitCke.startId, + allocation.localWaitCke.num, + allocation.remoteNotifyCke.startId, + allocation.remoteNotifyCke.num, + allocation.channels.startId, + allocation.channels.num, + }; + std::vector all(rankSize_); + const int ret = DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to exchange direct CCU peer XN/CKE resources"; + } + return ret; + } + + std::vector peerRanks; + peerRanks.reserve(peerRouteCount); + for (int peer = 0; peer < rankSize_; ++peer) { + if (peer != rank_) { + peerRanks.push_back(peer); + } + } + if (peerRanks.size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector peerCcuBuffers = *remoteCcuBuffers; + remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); + + size_t routeIndex = 0; + for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { + const size_t peerBufferIndex = syncIndex % peerRouteCount; + const int peer = peerRanks[peerBufferIndex]; + const PeerResourceExchange &peerResources = all[peer]; + const size_t peerLocalIndex = static_cast(rank_ < peer ? rank_ : rank_ - 1); + const uint32_t peerLocalXnOffset = + SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t selectedRemoteXnOffset = + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t peerLocalWaitCkeOffset = routeIndex; + if (peerResources.localXnCount == 0 || + peerResources.remoteXnCount == 0 || + peerResources.localWaitCkeCount == 0 || + peerResources.channelCount == 0 || + peerLocalXnOffset >= peerResources.localXnCount || + selectedRemoteXnOffset >= peerResources.remoteXnCount || + peerLocalIndex >= peerResources.channelCount || + peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { + if (report != nullptr) { + report->message = "peer direct CCU local XN/CKE resources are incomplete"; + } + return TILEXR_ERROR_NOT_FOUND; + } + uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( + peerResources.remoteXnStartId, + peerLocalIndex, + syncIndex, + peerRouteCount); + const uint16_t peerLocalXnId = + static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); + uint16_t remoteNotifyCke = + static_cast(static_cast(peerResources.localWaitCkeStartId) + + peerLocalWaitCkeOffset); + (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; + (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; + (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; + const bool peerLocalXnOwnerVerified = + static_cast(peerLocalXnId) >= peerResources.localXnStartId && + static_cast(peerLocalXnId) < + static_cast(peerResources.localXnStartId) + peerResources.localXnCount; + const bool notifyCkeOwnerVerified = + static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && + static_cast(remoteNotifyCke) < + static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; + const bool localChannelOwnerVerified = + allocation.channels.num != 0 && + peerLocalXnOwnerVerified && + static_cast(channelBoundRemoteXnId) >= peerResources.remoteXnStartId && + static_cast(channelBoundRemoteXnId) < + static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && + routeIndex < allocation.channels.num && + peerResources.channelStartId != 0 && + peerLocalIndex < peerResources.channelCount; + const bool transportResourceExchangeVerified = + notifyCkeOwnerVerified && + allocation.localWaitCke.num != 0 && + routeIndex < allocation.localWaitCke.num && + peerLocalWaitCkeOffset < peerResources.localWaitCkeCount; + (*remoteCcuBuffers)[routeIndex].channelResourceOwnerVerified = localChannelOwnerVerified; + (*remoteCcuBuffers)[routeIndex].transportResourceExchangeVerified = transportResourceExchangeVerified; + ++routeIndex; + } + return TILEXR_SUCCESS; +} + +int TileXRComm::DirectCcuAllGatherCallback( + const void *sendBuf, + size_t sendBytes, + void *recvBuf, + void *userData) +{ + auto *comm = static_cast(userData); + if (comm == nullptr || sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (comm->socketExchange_ == nullptr) { + return comm->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf); + } + return comm->socketExchange_->AllGather( + static_cast(sendBuf), + sendBytes, + static_cast(recvBuf)); +} + +int TileXRComm::DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) +{ + if (sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0 || rank_ < 0 || + rank_ >= rankSize_ || rankSize_ <= 0 || uid_.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint64_t round = directCcuThreadAllGatherRound_++; + const std::string key = uid_ + ":ccu:" + std::to_string(round); + auto start = high_resolution_clock::now(); + for (;;) { + { + lock_guard lock(g_mtx); + auto &state = g_directCcuAllGatherStates[key]; + if (state.bytes == 0) { + state.bytes = sendBytes; + } else if (state.bytes != sendBytes) { + g_directCcuAllGatherStates.erase(key); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (state.data[rank_].empty()) { + state.data[rank_].resize(sendBytes); + std::memcpy(state.data[rank_].data(), sendBuf, sendBytes); + ++state.arrivals; + } + if (state.arrivals == static_cast(rankSize_)) { + auto *output = static_cast(recvBuf); + for (int i = 0; i < rankSize_; ++i) { + std::memcpy(output + static_cast(i) * sendBytes, state.data[i].data(), sendBytes); + } + ++state.departures; + if (state.departures == static_cast(rankSize_)) { + g_directCcuAllGatherStates.erase(key); + } + return TILEXR_SUCCESS; + } + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round << ", " + << processUnavailableMessage; + return TILEXR_ERROR_NOT_FOUND; + } + this_thread::sleep_for(1ms); + auto elapsed = duration_cast(high_resolution_clock::now() - start); + if (elapsed.count() > TILEXR_INIT_TIMEOUT) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather timeout rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round; + return TILEXR_ERROR_TIMEOUT; + } + } +} + +int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData) +{ + auto *comm = static_cast(userData); + if (comm == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return comm->FillDirectCcuLowerLayerPlanFromAllocation(allocation, plan, report); +} + +int TileXRComm::PrepareDirectCcuInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!inited_) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "TileXRComm is not initialized for direct CCU install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { + const int ret = RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = directCcuBasicInfoReport_.message; + } + return ret; + } + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "direct CCU runtime is unavailable for install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = adapterReport.message; + } + return ret; + } + + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = &directCcuBasicInfo_; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = this; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu"; + } + + return TileXRCcuRunDirectInstallAttempt(next, attempt, report); +} + +int TileXRComm::PrepareDirectCcuMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!inited_) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRComm is not initialized for direct CCU memory copy install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize_) || peerRank == static_cast(rank_)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { + const int ret = RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = directCcuBasicInfoReport_.message; + } + return ret; + } + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for memory copy install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + int ret = BuildDirectCcuLocalMemoryCopyEndpoint( + static_cast(rank_), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU memory copy local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize_)); + ret = DirectCcuAllGatherCallback( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data(), + this); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU memory copy peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuDirectMemoryCopySpec memoryCopy; + memoryCopy.direction = direction; + memoryCopy.lengthBytes = bytes; + if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { + memoryCopy.localAddr = localEndpoint.destinationAddr; + memoryCopy.localToken = localEndpoint.destinationToken; + memoryCopy.remoteAddr = peerEndpoint.sourceAddr; + memoryCopy.remoteToken = peerEndpoint.sourceToken; + } else { + memoryCopy.localAddr = localEndpoint.sourceAddr; + memoryCopy.localToken = localEndpoint.sourceToken; + memoryCopy.remoteAddr = peerEndpoint.destinationAddr; + memoryCopy.remoteToken = peerEndpoint.destinationToken; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = &directCcuBasicInfo_; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = this; + next.sqeArgCount = 0; + next.syncResourceCount = 1; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-memory-copy"; + } + + return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); +} + +int TileXRComm::RefreshDirectCcuLowerLayerPlan() +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuLowerLayerTemplateConfigured_) { + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer template is not configured"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuLowerLayerPlanReport_.message = "direct CCU runtime is unavailable for lower-layer planning"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + int ret = ccuDirectRuntime_->ExportLowerLayerTransportSnapshot(directCcuLowerLayerTemplate_, &snapshot); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU lower-layer transport snapshot"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &snapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerSnapshot_ = snapshot; + directCcuLowerLayerPlan_ = plan; + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer install plan cached"; + directCcuLowerLayerPlanValid_ = true; + directCcuLowerLayerPlanStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRComm::HasDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_; +} + +int TileXRComm::GetDirectCcuLowerLayerPlanStatus() const +{ + return directCcuLowerLayerPlanStatus_; +} + +const TileXRCcuLowerLayerPlanBuilderReport &TileXRComm::GetDirectCcuLowerLayerPlanReport() const +{ + return directCcuLowerLayerPlanReport_; +} + +const TileXRCcuLowerLayerInstallPlan *TileXRComm::GetDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_ ? &directCcuLowerLayerPlan_ : nullptr; +} + +int TileXRComm::ReadDirectCcuInstructionsForDebug( + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report) +{ + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + } + if (!inited_) { + if (report != nullptr) { + report->message = "TileXRComm is not initialized for direct CCU instruction readback"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + report->message = "direct CCU runtime is unavailable for instruction readback"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); +} + int TileXRComm::InitCommon() { // enable peer device @@ -533,6 +1533,10 @@ int TileXRComm::Init() if (ret != TILEXR_SUCCESS) { return ret; } + ret = InitDirectCcuRuntime(); + if (ret != TILEXR_SUCCESS) { + return ret; + } ret = InitSDMA(); if (ret != TILEXR_SUCCESS) { return ret; @@ -550,6 +1554,47 @@ int TileXRComm::Init() return TILEXR_SUCCESS; } +int TileXRComm::InitDirectCcuOnly() +{ + if (inited_) { + return TILEXR_SUCCESS; + } + if (rank_ < 0 || rank_ >= rankSize_ || rankSize_ <= 0 || rankSize_ > TILEXR_MAX_RANK_SIZE) { + TILEXR_LOG(ERROR) << "The rank is invalid for direct CCU only init! rank:" << rank_ + << " rankSize:" << rankSize_; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (TileXRSockExchange::CheckValid(commId_)) { + socketExchange_ = new (nothrow) TileXRSockExchange(rank_, rankSize_, commId_); + } else { + socketExchange_ = new (nothrow) TileXRSockExchange(rank_, rankSize_, commDomain_); + } + if (socketExchange_ == nullptr) { + TILEXR_LOG(ERROR) << "TileXRSockExchange create failed for direct CCU only init. rank : " + << rank_ << " rankSize:" << rankSize_; + return TILEXR_ERROR_INTERNAL; + } + + int ret = GetDev(); + if (ret != TILEXR_SUCCESS) { + TILEXR_LOG(ERROR) << "direct CCU only init context failed! ret: " << ret; + return ret; + } + TILEXR_LOG(INFO) << "direct CCU only init rank " << rank_ << "/" << rankSize_ + << " running devId:" << devId_; + + if (InitCommon() != TILEXR_SUCCESS) { + TILEXR_LOG(ERROR) << "direct CCU only init common failed!"; + return TILEXR_ERROR_INTERNAL; + } + ret = InitDirectCcuRuntime(); + if (ret != TILEXR_SUCCESS) { + return ret; + } + inited_ = true; + return TILEXR_SUCCESS; +} + int TileXRComm::InitThread(const std::string &uid) { if (inited_) { @@ -605,6 +1650,10 @@ int TileXRComm::InitThread(const std::string &uid) // UDMA 主要用于跨进程/跨节点通信，线程模式使用进程内共享内存即可 TILEXR_LOG(DEBUG) << "Thread mode: UDMA initialization skipped (single-process multi-thread scenario)"; + ret = InitDirectCcuRuntime(); + if (ret != TILEXR_SUCCESS) { + return ret; + } ret = InitSDMA(); if (ret != TILEXR_SUCCESS) { return ret; @@ -887,6 +1936,9 @@ int TileXRComm::OpenIpcMem(const char names[TILEXR_MAX_RANK_SIZE][IPC_NAME_SIZE] if (ret != RT_ERROR_NONE) { CloseIpcMem(); TILEXR_LOG(ERROR) << "rank : " << rank_ << " localRank : " << localRank_ << " peerMem: " << i << + " devId : " << devId_ << " peerDevId : " << (i < static_cast(devList_.size()) ? devList_[i] : -1) << + " localRankSize : " << localRankSize_ << " ipcNameLen : " << std::strlen(names[i]) << + " ipcNamePrefix : " << std::string(names[i], std::min(std::strlen(names[i]), 16U)) << " IpcOpenMemory err " << ret; return TILEXR_ERROR_INTERNAL; } @@ -963,6 +2015,7 @@ TileXRComm::~TileXRComm() } udmaRegisteredPtr_ = nullptr; udmaInfoDev_ = nullptr; + ResetDirectCcuBasicInfo(); ResetSDMAState(); } diff --git a/src/comm/tilexr_comm.h b/src/comm/tilexr_comm.h index 0f6e37fa..279aa929 100644 --- a/src/comm/tilexr_comm.h +++ b/src/comm/tilexr_comm.h @@ -11,6 +11,7 @@ #define TILEXR_COMM_H #include +#include #include #include #include @@ -19,6 +20,9 @@ #include "../include/tilexr_types.h" #include "../include/tilexr_api.h" #include "../include/comm_args.h" +#include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" namespace TileXR { constexpr int IPC_NAME_SIZE = 65; @@ -35,6 +39,7 @@ class TileXRComm { TileXRComm(const TileXRComm &) = delete; TileXRComm &operator=(const TileXRComm &) = delete; int Init(); + int InitDirectCcuOnly(); int InitThread(const std::string &uid = "default"); int GetRank() const; int GetRankSize() const; @@ -47,6 +52,44 @@ class TileXRComm { int UnregisterUDMAMemory(TileXRUDMAMemHandle handle); GM_ADDR GetUDMARegistryPtr() const; const TileXRUDMARegistry* GetUDMARegistryHost() const; + int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); + bool HasDirectCcuBasicInfo() const; + int GetDirectCcuBasicInfoStatus() const; + const TileXRCcuBasicInfo *GetDirectCcuBasicInfo() const; + const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; + int ConfigureDirectCcuLowerLayerTemplate(const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot); + int ConfigureDirectCcuVerifiedEndpointRoutes( + const std::vector &verifiedRoutes); + int ConfigureDirectCcuLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route); + int ConfigureDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers); + int PrepareDirectCcuLowerLayerTemplateFromAllocation(const TileXRCcuResourceAllocation &allocation); + int RefreshDirectCcuLowerLayerPlan(); + bool HasDirectCcuLowerLayerPlan() const; + int GetDirectCcuLowerLayerPlanStatus() const; + const TileXRCcuLowerLayerPlanBuilderReport &GetDirectCcuLowerLayerPlanReport() const; + const TileXRCcuLowerLayerInstallPlan *GetDirectCcuLowerLayerPlan() const; + int ReadDirectCcuInstructionsForDebug( + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report); + int PrepareDirectCcuInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); bool IsSDMAAvailable() const; GM_ADDR GetSDMAWorkspacePtr() const; SDMAInitStatus GetSDMAInitStatus() const; @@ -73,9 +116,31 @@ class TileXRComm { int SyncCommArgs(); int InitDumpAddr(); int InitUDMA(); + int InitDirectCcuRuntime(); int InitSDMA(); int UpdateCommArgsDev(); void FreeUDMARegistry(); + void ResetDirectCcuBasicInfo(); + void ResetDirectCcuLowerLayerPlan(); + int FillDirectCcuLowerLayerPlanFromAllocation( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report); + int ExchangeDirectCcuRemoteNotifyCke( + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report); + static int PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData); + static int DirectCcuAllGatherCallback( + const void *sendBuf, + size_t sendBytes, + void *recvBuf, + void *userData); + int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); void ResetSDMAState(); private: @@ -106,6 +171,22 @@ class TileXRComm { GM_ADDR udmaRegisteredPtr_ = nullptr; TileXRUDMARegistry udmaRegistry_ = {}; std::unique_ptr udmaTransport_; + std::unique_ptr ccuDirectRuntime_; + bool directCcuBasicInfoValid_ = false; + int directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuBasicInfo directCcuBasicInfo_ = {}; + TileXRCcuDriverAdapterReport directCcuBasicInfoReport_ = {}; + bool directCcuLowerLayerTemplateConfigured_ = false; + bool directCcuLowerLayerPlanValid_ = false; + int directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerTemplate_ = {}; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerSnapshot_ = {}; + TileXRCcuLowerLayerInstallPlan directCcuLowerLayerPlan_ = {}; + TileXRCcuLowerLayerPlanBuilderReport directCcuLowerLayerPlanReport_ = {}; + std::vector directCcuVerifiedEndpointRoutes_ = {}; + TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; + bool directCcuLocalVerifiedEndpointRouteValid_ = false; + uint64_t directCcuThreadAllGatherRound_ = 0; GM_ADDR sdmaWorkspaceDev_ = nullptr; SDMAInitStatus sdmaInitStatus_ = SDMAInitStatus::DISABLED_BY_ENV; std::unique_ptr sdmaTransport_; diff --git a/src/comm/tilexr_internal.cpp b/src/comm/tilexr_internal.cpp index e39bd332..3f53f246 100644 --- a/src/comm/tilexr_internal.cpp +++ b/src/comm/tilexr_internal.cpp @@ -45,6 +45,25 @@ const std::unordered_map CHIP_MAP = { {"Ascend950PR_9599", ChipName::CHIP_950PR} }; +const std::unordered_map CHIP_PREFIX_MAP = { + {"Ascend950DT_", ChipName::CHIP_950}, + {"Ascend950PR_", ChipName::CHIP_950PR} +}; + +ChipName ResolveChipNameFromSocVersion(const std::string& chipName) +{ + auto it = CHIP_MAP.find(chipName); + if (it != CHIP_MAP.end()) { + return it->second; + } + for (const auto& item : CHIP_PREFIX_MAP) { + if (chipName.find(item.first) == 0) { + return item.second; + } + } + return ChipName::RESERVED; +} + /** * @brief 用于获取芯片名称 */ @@ -65,12 +84,8 @@ ChipName GetChipName() string chipName(ver); TILEXR_LOG(DEBUG) << "rtGetSocVersion -- The result after converting ver to string is:" << chipName; - auto it = CHIP_MAP.find(chipName); - if (it != CHIP_MAP.end()) { - curChipName = it->second; - } else if (chipName.find("Ascend950PR_") == 0) { - curChipName = ChipName::CHIP_950PR; - } else { + curChipName = ResolveChipNameFromSocVersion(chipName); + if (curChipName == ChipName::RESERVED) { TILEXR_LOG(WARN) << "There is no commitment to the supported chip types yet," << " and it is not certain whether the functions will work properly."; } diff --git a/src/include/tilexr_api.h b/src/include/tilexr_api.h index 7b4fd3a7..d71faf8e 100644 --- a/src/include/tilexr_api.h +++ b/src/include/tilexr_api.h @@ -10,19 +10,123 @@ #ifndef TILEXR_API_H #define TILEXR_API_H +#ifdef __cplusplus #include -#include +#include #include "comm_args.h" #include "tilexr_udma_reg.h" +#else +#include +#include +#include +typedef uint8_t *GM_ADDR; +#endif + #ifdef __cplusplus extern "C" { #endif // __cplusplus typedef void *TileXRCommPtr; +typedef void *TileXRDirectCcuPreparedTasksPtr; typedef uint32_t TileXRUDMAMemHandle; #define TILEXRUNIQUE_ID_BYTES 128 +#define TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES 2048 +#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION 0U +#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY 1U +#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES 0U +#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES 1U +#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL 0U +#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3 1U +#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM 2U +#define TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST 0U +#define TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST 1U +#define TILEXR_DIRECT_CCU_MEMORY_COPY_REMOTE_TO_LOCAL 0U +#define TILEXR_DIRECT_CCU_MEMORY_COPY_LOCAL_TO_REMOTE 1U +#define TILEXR_DIRECT_CCU_SQE_ARGS_LEN 13U typedef struct { char internal[TILEXRUNIQUE_ID_BYTES]; } TileXRUniqueId; +typedef struct TileXRDirectCcuPrepareOptions { + uint32_t syncResourceCount; + /* 0 is a valid explicit no-SQE-load mode; use TILEXR_DIRECT_CCU_SQE_ARGS_LEN for full SQE args. */ + uint32_t sqeArgCount; + uint32_t syncInstructionCount; + uint32_t bindingsPerSyncResource; + uint16_t missionStartId; + uint16_t instructionStartId; + uint16_t missionInstructionStartId; + uint16_t xnStartId; + uint16_t gsaStartId; + uint16_t remoteXnStartId; + uint16_t remoteXnCount; + uint16_t ckeStartId; + uint16_t channelStartId; + uint16_t localWaitCkeStartId; + uint16_t localWaitCkeCount; + uint16_t remoteNotifyCkeStartId; + uint16_t remoteNotifyCkeCount; + uint32_t repositoryInstallWindow; + uint32_t repositoryInstallDataLenMode; + uint32_t repositoryMemoryAllocMode; + uint32_t installOrder; + uint32_t deviceId; + uint32_t rank; + const char *provider; +} TileXRDirectCcuPrepareOptions; + +typedef struct TileXRDirectCcuMemoryCopyPrepareOptions { + TileXRDirectCcuPrepareOptions prepare; + uint64_t localSourceAddr; + uint64_t localDestinationAddr; + uint64_t bytes; + uint32_t peerRank; + uint32_t direction; +} TileXRDirectCcuMemoryCopyPrepareOptions; + +typedef struct TileXRDirectCcuPrepareReport { + bool pipelineBuilt; + bool installAttempted; + bool installSucceeded; + bool submitReady; + uint32_t requiredInstallSurfaceCount; + uint32_t publicVerifiedInstallSurfaceCount; + uint32_t missingInstallSurfaceCount; + uint32_t taskCount; + uint32_t submitTaskCount; + char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; +} TileXRDirectCcuPrepareReport; + +typedef struct TileXRDirectCcuSubmitReport { + bool submitted; + uint32_t taskCount; + uint32_t submittedTaskCount; + char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; +} TileXRDirectCcuSubmitReport; + +typedef struct TileXRDirectCcuTaskInfo { + uint8_t dieId; + uint8_t missionId; + uint16_t timeout; + uint16_t instStartId; + uint16_t instCnt; + uint32_t key; + uint32_t argSize; + uint64_t args[TILEXR_DIRECT_CCU_SQE_ARGS_LEN]; +} TileXRDirectCcuTaskInfo; + +#define TILEXR_DIRECT_CCU_INSTRUCTION_WORDS 4U +typedef struct TileXRDirectCcuInstructionWords { + uint64_t words[TILEXR_DIRECT_CCU_INSTRUCTION_WORDS]; +} TileXRDirectCcuInstructionWords; + +typedef struct TileXRDirectCcuInstructionReadbackReport { + bool readbackAttempted; + uint32_t readInstructionCount; + uint32_t opcode; + int driverRet; + int opRet; + char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; +} TileXRDirectCcuInstructionReadbackReport; + int TileXRGetUniqueId(TileXRUniqueId *uniqueId, int commDomain); int TileXRCommInitRankLocal(int rankSize, int rank, TileXRCommPtr *comm); @@ -33,9 +137,13 @@ int TileXRCommInitRankWithCustDomainSize(int commDomain, int bufferSize, int ran int TileXRCommInitRankWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm); +int TileXRCommInitRankDirectCcuWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm); + +#ifdef __cplusplus int TileXRGetCommArgsDev(TileXRCommPtr comm, GM_ADDR &commArgsPtr); int TileXRGetCommArgsHost(TileXRCommPtr comm, TileXR::CommArgs *&commArgsPtr); +#endif int TileXRCommNextMagic(TileXRCommPtr comm, int64_t *magic); @@ -43,14 +151,54 @@ int TileXRUDMARegister(TileXRCommPtr comm, GM_ADDR localPtr, size_t bytes, TileX int TileXRUDMAUnregister(TileXRCommPtr comm, TileXRUDMAMemHandle handle); +#ifdef __cplusplus int TileXRGetUDMARegistryDev(TileXRCommPtr comm, GM_ADDR ®istryPtr); int TileXRGetUDMARegistryHost(TileXRCommPtr comm, const TileXR::TileXRUDMARegistry **registry); +#endif int TileXRSDMAAvailable(TileXRCommPtr comm, bool *available); int TileXRGetSDMAWorkspaceDev(TileXRCommPtr comm, GM_ADDR *workspace); +int TileXRCommPrepareDirectCcu( + TileXRCommPtr comm, + const TileXRDirectCcuPrepareOptions *options, + TileXRDirectCcuPreparedTasksPtr *prepared, + TileXRDirectCcuPrepareReport *report); + +int TileXRCommPrepareDirectCcuMemoryCopy( + TileXRCommPtr comm, + const TileXRDirectCcuMemoryCopyPrepareOptions *options, + TileXRDirectCcuPreparedTasksPtr *prepared, + TileXRDirectCcuPrepareReport *report); + +int TileXRDirectCcuGetPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t index, + TileXRDirectCcuTaskInfo *task); + +int TileXRDirectCcuSubmitPrepared( + TileXRDirectCcuPreparedTasksPtr prepared, + void *stream, + TileXRDirectCcuSubmitReport *report); + +int TileXRDirectCcuSubmitPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t index, + void *stream, + TileXRDirectCcuSubmitReport *report); + +int TileXRCommReadDirectCcuInstructions( + TileXRCommPtr comm, + uint8_t dieId, + uint16_t instructionStartId, + uint32_t instructionCount, + TileXRDirectCcuInstructionWords *instructions, + TileXRDirectCcuInstructionReadbackReport *report); + +int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared); + void TileXRPrintDFX2Log(TileXRCommPtr comm); int TileXRCommInit(int rank, int rankSize, TileXRCommPtr *comms); diff --git a/tests/ccu/CMakeLists.txt b/tests/ccu/CMakeLists.txt new file mode 100644 index 00000000..212090fc --- /dev/null +++ b/tests/ccu/CMakeLists.txt @@ -0,0 +1,10 @@ +# +# Copyright (c) 2026 TileXR Project +# +# The direct CCU smoke runner compiles its probe against the active TileXR build +# so that it can test the exact libtile-comm.so under validation. Keep this +# directory free of private producer probe targets. +# + +cmake_minimum_required(VERSION 3.16) +project(TileXR_CCU_Tests LANGUAGES CXX) diff --git a/tests/ccu/ccu_lower_layer_payload_hcomm_oracle.cpp b/tests/ccu/ccu_lower_layer_payload_hcomm_oracle.cpp new file mode 100644 index 00000000..8659f4f7 --- /dev/null +++ b/tests/ccu/ccu_lower_layer_payload_hcomm_oracle.cpp @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2026 TileXR Project + * + * Offline byte-level oracle for TileXR CCU lower-layer payloads. + * + * Reference hcomm files mirrored here as packed test-only structs: + * - ccu_pfe/ccu_pfe_mgr.h + * - ccu_pfe/ccu_pfe_mgr.cc + * - ccu_jetty_ctx_mgr.h + * - ccu_jetty_ctx_mgr.cc + * - ccu_channel_ctx_v1/ccu_channel_ctx_mgr_v1.h + * - ccu_channel_ctx_v1/ccu_channel_ctx_mgr_v1.cc + * + * This probe intentionally does not include hcomm/HCCL headers. It is a + * second implementation of the hcomm packing formulas, used to compare the + * bytes generated by TileXR's direct-CCU payload packers. + */ + +#include "ccu/tilexr_ccu_lower_layer_payloads.h" + +#include +#include +#include +#include +#include + +using namespace TileXR; + +namespace { + +constexpr uint8_t HCOMM_DB_ADDR_TYPE = 1; +constexpr uint8_t HCOMM_TOKEN_VALUE_IS_VALID = 1; +constexpr uint16_t HCOMM_TOKEN_VALUE_VALID = 1; +constexpr uint16_t HCOMM_REMOTE_CCU_VA_RIGHT_SHIFT_NUM = 23; +constexpr uint32_t HCOMM_CCU_WQE_NUM_PER_SQE = 4; + +#pragma pack(push, 1) +struct HcommPfeCtx { + uint16_t startJettyId; + uint16_t jettyNum : 7; + uint16_t startLocalJettyCtxId : 7; + uint16_t rsvBit : 2; + uint16_t rsv[2]; +}; + +struct HcommLocalJettyCtxData { + uint16_t doorbellAddr[4]; + uint16_t pfeIdx : 4; + uint16_t ioDieId : 1; + uint16_t doorbellAddrType : 1; + uint16_t tokenValueIsValid : 1; + uint16_t cqeErrValue : 1; + uint16_t tokenIdLow : 8; + uint16_t tokenIdHigh : 12; + uint16_t tokenValueLow : 4; + uint16_t tokenValueMiddle; + uint16_t tokenValueHigh : 12; + uint16_t sqeBasicBlockLeftShifts : 4; + uint16_t pi; + uint16_t ci; + uint16_t maxCi; + uint16_t oooCqeCnt : 12; + uint16_t startWqeBasicBlockIdxLow : 4; + uint16_t startWqeBasicBlockIdxHigh : 8; + uint16_t doorbellSendState : 2; + uint16_t rsvSixBits : 6; + uint16_t rsvs[3]; +}; + +struct HcommChannelCtxDataV1 { + uint8_t eidRaw[TILEXR_CCU_EID_BYTES]; + uint16_t vtpLow; + uint16_t vtpHigh : 8; + uint16_t srcPfeId : 4; + uint16_t startJettyIdLow : 4; + uint16_t startJettyIdHigh : 12; + uint16_t jettyNumLow : 4; + uint16_t jettyNumHigh : 3; + uint16_t ioDieId : 1; + uint16_t dstTokenIdLow : 12; + uint16_t dstTokenIdHigh : 8; + uint16_t dstTokenValueLow : 8; + uint16_t dstTokenValueMiddle; + uint16_t dstTokenValueHigh : 8; + uint16_t dstVaLow : 8; + uint16_t dstVaMiddle; + uint16_t dstVaHigh; + uint16_t dstVaHigher : 1; + uint16_t dstTokenValueValid : 1; + uint16_t rsv14Bits : 14; + uint16_t rsvs[14]; +}; +#pragma pack(pop) + +static_assert(sizeof(HcommPfeCtx) == TILEXR_CCU_PFE_CTX_BYTES, "hcomm PFE ctx size mismatch"); +static_assert(sizeof(HcommLocalJettyCtxData) == TILEXR_CCU_LOCAL_JETTY_CTX_BYTES, + "hcomm local jetty ctx size mismatch"); +static_assert(sizeof(HcommChannelCtxDataV1) == TILEXR_CCU_CHANNEL_CTX_V1_BYTES, + "hcomm channel ctx v1 size mismatch"); + +uint16_t Log2PowerOfTwo(uint32_t value) +{ + uint16_t log2 = 0; + while (value > 1U) { + value >>= 1U; + ++log2; + } + return log2; +} + +HcommPfeCtx BuildHcommPfeCtx( + uint16_t startTaJettyId, + uint16_t jettyCount, + uint16_t startLocalJettyCtxId) +{ + HcommPfeCtx ctx {}; + ctx.startJettyId = startTaJettyId; + ctx.jettyNum = static_cast(jettyCount - 1U); + ctx.startLocalJettyCtxId = startLocalJettyCtxId; + return ctx; +} + +HcommLocalJettyCtxData BuildHcommLocalJettyCtx( + uint8_t dieId, + uint32_t pfeId, + uint64_t doorbellVa, + uint32_t doorbellTokenId, + uint32_t doorbellTokenValue, + uint32_t sqDepth, + uint16_t wqeBasicBlockStartId) +{ + HcommLocalJettyCtxData data {}; + uint16_t doorbell[4] {}; + std::memcpy(doorbell, &doorbellVa, sizeof(doorbell)); + data.doorbellAddr[0] = doorbell[0]; + data.doorbellAddr[1] = doorbell[1]; + data.doorbellAddr[2] = doorbell[2]; + data.doorbellAddr[3] = doorbell[3]; + data.pfeIdx = static_cast(pfeId); + data.ioDieId = dieId; + data.doorbellAddrType = HCOMM_DB_ADDR_TYPE; + data.tokenValueIsValid = HCOMM_TOKEN_VALUE_IS_VALID; + data.tokenIdLow = doorbellTokenId & 0x000000ffU; + data.tokenIdHigh = (doorbellTokenId >> 8U) & 0x00000fffU; + data.tokenValueLow = doorbellTokenValue & 0x0000000fU; + data.tokenValueMiddle = (doorbellTokenValue >> 4U) & 0x0000ffffU; + data.tokenValueHigh = (doorbellTokenValue >> 20U) & 0x00000fffU; + data.sqeBasicBlockLeftShifts = Log2PowerOfTwo(sqDepth * HCOMM_CCU_WQE_NUM_PER_SQE); + data.startWqeBasicBlockIdxLow = wqeBasicBlockStartId & 0x0000000fU; + data.startWqeBasicBlockIdxHigh = (wqeBasicBlockStartId >> 4U) & 0x000000ffU; + return data; +} + +HcommChannelCtxDataV1 BuildHcommChannelCtxV1( + const std::array& remoteEid, + uint32_t tpn, + uint32_t feId, + uint16_t startTaJettyId, + uint16_t jettyCount, + uint8_t dieId, + uint32_t memTokenId, + uint32_t memTokenValue, + uint64_t remoteCcuVa) +{ + HcommChannelCtxDataV1 data {}; + std::copy(remoteEid.begin(), remoteEid.end(), data.eidRaw); + data.vtpLow = tpn & 0x0000ffffU; + data.vtpHigh = ((tpn & 0xffff0000U) >> 16U) & 0x000000ffU; + data.srcPfeId = static_cast(feId); + data.startJettyIdLow = startTaJettyId & 0x000fU; + data.startJettyIdHigh = (startTaJettyId >> 4U) & 0x0fffU; + const uint8_t jettyNum = static_cast(jettyCount - 1U); + data.jettyNumLow = jettyNum & 0x000fU; + data.jettyNumHigh = (jettyNum >> 4U) & 0x0007U; + data.ioDieId = dieId; + data.dstTokenIdLow = memTokenId & 0x00000fffU; + data.dstTokenIdHigh = (memTokenId >> 12U) & 0x000000ffU; + data.dstTokenValueLow = memTokenValue & 0x000000ffU; + data.dstTokenValueMiddle = (memTokenValue >> 8U) & 0x0000ffffU; + data.dstTokenValueHigh = (memTokenValue >> 24U) & 0x000000ffU; + const uint64_t dstVa = remoteCcuVa >> HCOMM_REMOTE_CCU_VA_RIGHT_SHIFT_NUM; + data.dstVaLow = dstVa & 0x00000000000000ffULL; + data.dstVaMiddle = (dstVa >> 8U) & 0x000000000000ffffULL; + data.dstVaHigh = (dstVa >> 24U) & 0x000000000000ffffULL; + data.dstVaHigher = (dstVa >> 40U) & 0x0000000000000001ULL; + data.dstTokenValueValid = HCOMM_TOKEN_VALUE_VALID; + return data; +} + +template +bool RawEquals(const HcommStruct& expected, const TileXRStruct& actual) +{ + return sizeof(expected) == sizeof(actual.raw) && + std::memcmp(&expected, actual.raw, sizeof(expected)) == 0; +} + +template +int CheckRawEquals(const char* name, const HcommStruct& expected, const TileXRStruct& actual) +{ + if (RawEquals(expected, actual)) { + return 0; + } + const auto* expectedBytes = reinterpret_cast(&expected); + for (uint32_t i = 0; i < sizeof(expected); ++i) { + if (expectedBytes[i] != actual.raw[i]) { + std::cerr << name << " mismatch at byte " << i + << " expected=0x" << std::hex << static_cast(expectedBytes[i]) + << " actual=0x" << static_cast(actual.raw[i]) + << std::dec << "\n"; + return 1; + } + } + std::cerr << name << " mismatch\n"; + return 1; +} + +int CheckPfe() +{ + TileXRCcuPfeCtx actual; + TileXRCcuLowerLayerPayloadReport report; + TileXRCcuPfeCtxSpec spec; + spec.startJettyId = 0x1234; + spec.jettyCount = 5; + spec.startLocalJettyCtxId = 0x22; + if (TileXRCcuBuildPfeCtx(spec, &actual, &report) != TILEXR_SUCCESS) { + std::cerr << "TileXR PFE build failed: " << report.message << "\n"; + return 1; + } + return CheckRawEquals("PFE", BuildHcommPfeCtx(0x1234, 5, 0x22), actual); +} + +int CheckJetty() +{ + TileXRCcuLocalJettyCtxData actual; + TileXRCcuLowerLayerPayloadReport report; + TileXRCcuLocalJettyCtxSpec spec; + spec.dieId = 1; + spec.pfeId = 3; + spec.doorbellVa = 0x1122334455667788ULL; + spec.doorbellTokenId = 0x000abcdeU; + spec.doorbellTokenValue = 0x89abcdefU; + spec.sqDepth = 16; + spec.wqeBasicBlockStartId = 0x9a; + if (TileXRCcuBuildLocalJettyCtx(spec, &actual, &report) != TILEXR_SUCCESS) { + std::cerr << "TileXR jetty build failed: " << report.message << "\n"; + return 1; + } + return CheckRawEquals( + "Jetty", + BuildHcommLocalJettyCtx(1, 3, 0x1122334455667788ULL, 0x000abcdeU, 0x89abcdefU, 16, 0x9a), + actual); +} + +int CheckChannel() +{ + std::array remoteEid {}; + for (uint32_t i = 0; i < remoteEid.size(); ++i) { + remoteEid[i] = static_cast(0x10 + i); + } + + TileXRCcuChannelCtxDataV1 actual; + TileXRCcuLowerLayerPayloadReport report; + TileXRCcuChannelCtxV1Spec spec; + spec.remoteEid = remoteEid; + spec.tpn = 0x00ab5678U; + spec.sourcePfeId = 5; + spec.startJettyId = 0x0234; + spec.jettyCount = 7; + spec.dieId = 1; + spec.memoryTokenId = 0x000abcdeU; + spec.memoryTokenValue = 0x89abcdefU; + spec.remoteCcuVa = 0x000123456789ab00ULL; + if (TileXRCcuBuildChannelCtxV1(spec, &actual, &report) != TILEXR_SUCCESS) { + std::cerr << "TileXR channel build failed: " << report.message << "\n"; + return 1; + } + return CheckRawEquals( + "Channel", + BuildHcommChannelCtxV1( + remoteEid, + 0x00ab5678U, + 5, + 0x0234, + 7, + 1, + 0x000abcdeU, + 0x89abcdefU, + 0x000123456789ab00ULL), + actual); +} + +int CheckZeroTokenValues() +{ + TileXRCcuLowerLayerPayloadReport report; + TileXRCcuLocalJettyCtxData actualJetty; + TileXRCcuLocalJettyCtxSpec jettySpec; + jettySpec.dieId = 0; + jettySpec.pfeId = 2; + jettySpec.doorbellVa = 0x1020304050607080ULL; + jettySpec.doorbellTokenId = 0x12345U; + jettySpec.doorbellTokenValue = 0; + jettySpec.sqDepth = 8; + if (TileXRCcuBuildLocalJettyCtx(jettySpec, &actualJetty, &report) != TILEXR_SUCCESS) { + std::cerr << "TileXR zero-token jetty build failed: " << report.message << "\n"; + return 1; + } + if (CheckRawEquals( + "ZeroTokenJetty", + BuildHcommLocalJettyCtx(0, 2, 0x1020304050607080ULL, 0x12345U, 0, 8, 0), + actualJetty) != 0) { + return 1; + } + + std::array remoteEid {}; + for (uint32_t i = 0; i < remoteEid.size(); ++i) { + remoteEid[i] = static_cast(0xa0 + i); + } + TileXRCcuChannelCtxDataV1 actualChannel; + TileXRCcuChannelCtxV1Spec channelSpec; + channelSpec.remoteEid = remoteEid; + channelSpec.tpn = 0x13579U; + channelSpec.sourcePfeId = 2; + channelSpec.startJettyId = 0x44; + channelSpec.jettyCount = 1; + channelSpec.dieId = 0; + channelSpec.memoryTokenId = 0x12345U; + channelSpec.memoryTokenValue = 0; + channelSpec.remoteCcuVa = 0x0000001234000000ULL; + if (TileXRCcuBuildChannelCtxV1(channelSpec, &actualChannel, &report) != TILEXR_SUCCESS) { + std::cerr << "TileXR zero-token channel build failed: " << report.message << "\n"; + return 1; + } + return CheckRawEquals( + "ZeroTokenChannel", + BuildHcommChannelCtxV1(remoteEid, 0x13579U, 2, 0x44, 1, 0, 0x12345U, 0, 0x0000001234000000ULL), + actualChannel); +} + +} // namespace + +int main() +{ + if (CheckPfe() != 0 || CheckJetty() != 0 || CheckChannel() != 0 || CheckZeroTokenValues() != 0) { + return 1; + } + std::cout << "hcomm lower-layer payload oracle matched" << std::endl; + return 0; +} diff --git a/tests/ccu/ccu_npu_smi_busy_guard.py b/tests/ccu/ccu_npu_smi_busy_guard.py new file mode 100644 index 00000000..a8f103e5 --- /dev/null +++ b/tests/ccu/ccu_npu_smi_busy_guard.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import argparse +import re +import sys +from pathlib import Path + + +def parse_devices(text): + devices = set() + for item in text.split(","): + item = item.strip() + if not item: + continue + devices.add(int(item, 0)) + return devices + + +def first_int(text): + match = re.search(r"\d+", text) + return int(match.group(0), 10) if match else None + + +def parse_busy_processes(log_text, selected_devices): + busy = [] + in_process_table = False + for line in log_text.splitlines(): + if "|" not in line: + continue + lower_line = line.lower() + if "process id" in lower_line or "process name" in lower_line: + in_process_table = True + continue + if not in_process_table: + continue + fields = [field.strip() for field in line.strip().strip("|").split("|")] + if len(fields) < 3: + continue + device = first_int(fields[0]) + if device is None or device not in selected_devices: + continue + + pid = first_int(fields[1]) + process_index = 2 + if (pid is None or pid == 0) and len(fields) >= 4: + pid = first_int(fields[2]) + process_index = 3 + if pid is None or pid == 0: + continue + process = fields[process_index].split()[0] if fields[process_index].split() else "unknown" + if process in {"-", "N/A", "NA"}: + continue + busy.append((device, pid, process)) + return busy + + +def parse_unhealthy_devices(log_text, selected_devices): + unhealthy = [] + observed_selected = set() + in_status_table = False + for line in log_text.splitlines(): + if "|" not in line: + continue + lower_line = line.lower() + if "process id" in lower_line or "process name" in lower_line: + in_status_table = False + continue + if "health" in lower_line: + in_status_table = True + continue + if not in_status_table: + continue + + fields = [field.strip() for field in line.strip().strip("|").split("|")] + if len(fields) < 3: + continue + device = first_int(fields[0]) + if device is None or device not in selected_devices: + continue + health = fields[2].split()[0] if fields[2].split() else "" + if not health: + continue + observed_selected.add(device) + if health.upper() != "OK": + unhealthy.append((device, health)) + return unhealthy, observed_selected + + +def main(): + parser = argparse.ArgumentParser(description="Reject TileXR CCU smoke runs on busy or unhealthy selected NPUs.") + parser.add_argument("--log", required=True, help="npu-smi info log path") + parser.add_argument("--devices", required=True, help="comma-separated NPU device ids selected for the smoke") + parser.add_argument( + "--allow-unhealthy", + action="store_true", + help="Allow selected devices with non-OK health while still rejecting busy processes.", + ) + args = parser.parse_args() + + try: + selected_devices = parse_devices(args.devices) + except ValueError as exc: + print(f"invalid device list {args.devices!r}: {exc}", file=sys.stderr) + return 2 + if not selected_devices: + print("no selected NPU devices to check", file=sys.stderr) + return 2 + + log_path = Path(args.log) + log_text = log_path.read_text(encoding="utf-8", errors="replace") + unhealthy, observed_health = parse_unhealthy_devices(log_text, selected_devices) + if unhealthy: + if not args.allow_unhealthy: + for device, health in unhealthy: + print(f"unhealthy selected NPU device={device} health={health}") + return 1 + for device, health in unhealthy: + print(f"unhealthy selected NPU devices allowed device={device} health={health}") + if observed_health and not unhealthy: + print(f"selected NPU devices healthy devices={args.devices}") + + busy = parse_busy_processes(log_text, selected_devices) + if busy: + for device, pid, process in busy: + print(f"busy selected NPU process device={device} pid={pid} process={process}") + return 1 + + print(f"no selected NPU processes devices={args.devices}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/ccu/ccu_public_direct_api_compile_probe.c b/tests/ccu/ccu_public_direct_api_compile_probe.c new file mode 100644 index 00000000..fb866496 --- /dev/null +++ b/tests/ccu/ccu_public_direct_api_compile_probe.c @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2026 TileXR Project + * + * External-user compile probe for the public direct CCU API. + * + * This file intentionally includes only tilexr_api.h from TileXR. It is + * compiled to an object file, not linked, so it verifies public declarations + * without requiring Ascend runtime or NPU hardware. + */ + +#include "tilexr_api.h" + +#ifndef TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES +#error "TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES must be public" +#endif + +int TileXRPublicDirectCcuApiCompileProbe(void) +{ + TileXRCommPtr comm = 0; + TileXRDirectCcuPreparedTasksPtr prepared = 0; + TileXRDirectCcuPrepareOptions options = {0}; + TileXRDirectCcuPrepareReport prepareReport = {0}; + TileXRDirectCcuSubmitReport submitReport = {0}; + TileXRDirectCcuTaskInfo task = {0}; + + int (*prepareFn)( + TileXRCommPtr, + const TileXRDirectCcuPrepareOptions *, + TileXRDirectCcuPreparedTasksPtr *, + TileXRDirectCcuPrepareReport *) = &TileXRCommPrepareDirectCcu; + int (*getTaskFn)( + TileXRDirectCcuPreparedTasksPtr, + uint32_t, + TileXRDirectCcuTaskInfo *) = &TileXRDirectCcuGetPreparedTask; + int (*submitFn)( + TileXRDirectCcuPreparedTasksPtr, + void *, + TileXRDirectCcuSubmitReport *) = &TileXRDirectCcuSubmitPrepared; + int (*destroyFn)(TileXRDirectCcuPreparedTasksPtr) = &TileXRDirectCcuDestroyPrepared; + + options.syncResourceCount = 1; + options.syncInstructionCount = 2; + options.bindingsPerSyncResource = 1; + options.gsaStartId = 510; + options.provider = "tilexr-public-direct-ccu-compile-probe"; + + prepareReport.message[0] = '\0'; + submitReport.message[0] = '\0'; + task.args[0] = 0; + + (void)comm; + (void)prepared; + (void)prepareFn; + (void)getTaskFn; + (void)submitFn; + (void)destroyFn; + return (int)(options.syncResourceCount + options.gsaStartId + prepareReport.taskCount + + submitReport.taskCount + task.argSize); +} diff --git a/tests/ccu/ccu_tilexr_basic_info_probe.cpp b/tests/ccu/ccu_tilexr_basic_info_probe.cpp new file mode 100644 index 00000000..73e673fb --- /dev/null +++ b/tests/ccu/ccu_tilexr_basic_info_probe.cpp @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + * + * Probe TileXR-owned direct CCU runtime basic-info path. + * This does not install repository state and does not submit CCU tasks. + */ + +#include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_specs.h" + +#include +#include +#include +#include + +namespace { + +uint32_t ParseUintArg(const char* value, uint32_t fallback) +{ + if (value == nullptr) { + return fallback; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 0); + if (end == value || *end != '\0') { + return fallback; + } + return static_cast(parsed); +} + +void PrintHex32(const char* label, uint32_t value) +{ + std::cout << label << "=0x" << std::hex << std::setw(8) << std::setfill('0') << value << std::dec; +} + +void PrintHex64(const char* label, uint64_t value) +{ + std::cout << label << "=0x" << std::hex << std::setw(16) << std::setfill('0') << value << std::dec; +} + +void PrintBasicInfo( + uint32_t deviceLogicId, + uint8_t dieId, + const TileXR::TileXRCcuBasicInfo& basicInfo, + const TileXR::TileXRCcuDriverAdapterReport& adapterReport, + const TileXR::TileXRCcuSpecInfo& specInfo) +{ + std::cout << "tilexr_ccu_basic_info result" + << " deviceLogicId=" << deviceLogicId + << " devicePhyId=" << adapterReport.devicePhyId + << " dieId=" << static_cast(dieId) + << " driverRet=" << adapterReport.driverRet + << " opRet=" << adapterReport.opRet + << " msId=" << basicInfo.msId + << " tokenId=" << basicInfo.msidToken.tokenId + << " tokenValue=" << basicInfo.msidToken.tokenValue + << " tokenValid=" << (basicInfo.msidToken.valid ? 1 : 0) + << " "; + PrintHex32("missionKey", basicInfo.missionKey); + std::cout << " "; + PrintHex64("resourceAddr", basicInfo.resourceAddr); + std::cout << " instructionNum=" << specInfo.instructionNum + << " xnNum=" << specInfo.xnNum + << " ckeNum=" << specInfo.ckeNum + << " channelNum=" << specInfo.channelNum + << " missionNum=" << specInfo.missionNum + << std::endl; +} + +} // namespace + +int main(int argc, char** argv) +{ + const uint32_t deviceLogicId = ParseUintArg(argc > 1 ? argv[1] : nullptr, 0); + const uint8_t dieId = static_cast(ParseUintArg(argc > 2 ? argv[2] : nullptr, 0)); + + TileXR::TileXRCcuDirectRuntime runtime; + TileXR::TileXRCcuDirectRuntimeOptions options; + options.devId = static_cast(deviceLogicId); + options.rank = 0; + options.rankSize = 1; + + TileXR::TileXRCcuDirectRuntimeReport runtimeReport; + int ret = runtime.Init(options, &runtimeReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << "failed to initialize TileXR CCU direct runtime: " + << runtimeReport.message + << " deviceLogicId=" << runtimeReport.logicDevId + << " devicePhyId=" << runtimeReport.devicePhyId + << " hdcType=" << runtimeReport.hdcType + << " raInitialized=" << (runtimeReport.raInitialized ? 1 : 0) + << std::endl; + return 2; + } + std::cout << "tilexr_ccu_basic_info runtime" + << " deviceLogicId=" << runtimeReport.logicDevId + << " devicePhyId=" << runtimeReport.devicePhyId + << " hdcType=" << runtimeReport.hdcType + << " raInitialized=" << (runtimeReport.raInitialized ? 1 : 0) + << " message=\"" << runtimeReport.message << "\"" + << std::endl; + + TileXR::TileXRCcuBasicInfo basicInfo; + TileXR::TileXRCcuDriverAdapterReport adapterReport; + ret = runtime.QueryBasicInfo(dieId, &basicInfo, &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << "failed to query CCU basic info: " << adapterReport.message + << " driverRet=" << adapterReport.driverRet + << " opRet=" << adapterReport.opRet << std::endl; + return 3; + } + + TileXR::TileXRCcuSpecInfo specInfo; + TileXR::TileXRCcuSpecsReport specsReport; + ret = TileXR::TileXRCcuDecodeBasicInfo(basicInfo, &specInfo, &specsReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << "failed to decode CCU basic info: " << specsReport.message << std::endl; + return 4; + } + + PrintBasicInfo(deviceLogicId, dieId, basicInfo, adapterReport, specInfo); + return 0; +} diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp new file mode 100644 index 00000000..232d46da --- /dev/null +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -0,0 +1,1261 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + * + * Private C++ integration probe for the TileXR-owned direct CCU prepare path. + * The default run is intentionally hardware-safe. Set TILEXR_CCU_DIRECT_SMOKE_ENABLE=1 + * in a real multi-rank TileXRComm launch to prepare the no-hcomm direct CCU install attempt. + */ + +#include "acl/acl.h" +#include "acl/acl_rt.h" +#include "tilexr_api.h" +#include "tilexr_types.h" +#include "runtime/dev.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr const char* kEnableEnv = "TILEXR_CCU_DIRECT_SMOKE_ENABLE"; +constexpr const char* kThreadModeEnv = "TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE"; +constexpr const char* kDirectCcuOnlyInitEnv = "TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT"; +constexpr const char* kFastExitOnPrepareFailureEnv = "TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE"; +constexpr const char* kFastExitAfterRunEnv = "TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN"; +constexpr const char* kTraceLifecycleEnv = "TILEXR_CCU_DIRECT_SMOKE_TRACE_LIFECYCLE"; +constexpr const char* kReadbackInstructionsEnv = "TILEXR_CCU_DIRECT_SMOKE_READBACK_INSTRUCTIONS"; +constexpr const char* kSubmitEnv = "TILEXR_CCU_DIRECT_SMOKE_SUBMIT"; +constexpr const char* kReadyDirEnv = "TILEXR_CCU_DIRECT_SMOKE_READY_DIR"; +constexpr const char* kDoneDirEnv = "TILEXR_CCU_DIRECT_SMOKE_DONE_DIR"; +constexpr const char* kReadyTimeoutMsEnv = "TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS"; +constexpr const char* kSubmitTaskSelectorEnv = "TILEXR_CCU_DIRECT_SMOKE_SUBMIT_TASK_SELECTOR"; +constexpr const char* kDelayRankEnv = "TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK"; +constexpr const char* kPreSubmitDelayMsEnv = "TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS"; +constexpr const char* kP2pCcuCopyEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY"; +constexpr const char* kExpectP2pCcuCopyEnv = "TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY"; +constexpr const char* kP2pCcuCopyBytesEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES"; +constexpr const char* kLocalWaitCkeStartEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START"; +constexpr const char* kLocalWaitCkeCountEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT"; +constexpr const char* kRemoteNotifyCkeStartEnv = "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START"; +constexpr const char* kRemoteNotifyCkeCountEnv = "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT"; +constexpr const char* kBarrierModeEnv = "TILEXR_CCU_DIRECT_BARRIER_MODE"; +constexpr const char* kRepositoryInstallWindowEnv = "TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW"; +constexpr const char* kRepositoryInstallDataLenModeEnv = "TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE"; +constexpr const char* kRepositoryMemoryAllocModeEnv = "TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE"; +constexpr const char* kInstallOrderEnv = "TILEXR_CCU_DIRECT_INSTALL_ORDER"; +constexpr const char* kResourceWindowTokenIdEnv = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID"; +constexpr const char* kResourceWindowRawTokenIdEnv = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID"; +constexpr const char* kResourceWindowTokenValueEnv = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE"; +constexpr uint32_t kHcommStyleTask1PreludeInstructionCount = 5U; + +struct DeviceBuffer { + void* ptr = nullptr; + size_t bytes = 0; + + ~DeviceBuffer() + { + if (ptr != nullptr) { + aclrtFree(ptr); + } + } + + int Allocate(size_t size) + { + if (size == 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + bytes = size; + return aclrtMalloc(&ptr, bytes, ACL_MEM_MALLOC_HUGE_FIRST); + } +}; + +struct P2pCcuCopyState { + DeviceBuffer source; + DeviceBuffer destination; + std::vector expected; + std::vector observed; + size_t bytes = 0; + int initRet = ACL_SUCCESS; + int readRet = ACL_SUCCESS; + uint32_t mismatchCount = 0; + bool passed = false; +}; + +bool EnvFlag(const char* name) +{ + const char* value = std::getenv(name); + if (value == nullptr) { + return false; + } + const std::string text(value); + return text == "1" || text == "true" || text == "TRUE" || text == "yes" || text == "on"; +} + +bool ShouldFastExitAfterPrepareFailure(int ret) +{ + return ret != 0 && EnvFlag(kFastExitOnPrepareFailureEnv); +} + +const char* FastExitReasonForReturnCode(int ret) +{ + switch (ret) { + case 6: + return "prepare failed; skipping cleanup to preserve diagnostic status"; + case 8: + return "direct CCU stream synchronize failed; skipping cleanup to preserve diagnostic status"; + case 9: + return "direct CCU submit failed; skipping cleanup to preserve diagnostic status"; + case 13: + return "direct CCU collective completion timed out; skipping cleanup to preserve diagnostic status"; + case 14: + return "direct CCU P2P CCU-copy check failed; skipping cleanup to preserve diagnostic status"; + default: + return "direct CCU smoke failed; skipping cleanup to preserve diagnostic status"; + } +} + +bool ShouldFastExitAfterRun() +{ + return EnvFlag(kFastExitAfterRunEnv); +} + +void TraceLifecycle(const char* stage) +{ + if (!EnvFlag(kTraceLifecycleEnv)) { + return; + } + std::cout << "tilexr_ccu_direct_smoke lifecycle " << stage << std::endl; +} + +int ParseInt(const char* value, int fallback) +{ + if (value == nullptr || value[0] == '\0') { + return fallback; + } + char* end = nullptr; + const long parsed = std::strtol(value, &end, 0); + if (end == value || *end != '\0') { + return fallback; + } + return static_cast(parsed); +} + +int EnvInt(const char* name, int fallback) +{ + return ParseInt(std::getenv(name), fallback); +} + +uint64_t ParseU64(const char* value, uint64_t fallback) +{ + if (value == nullptr || value[0] == '\0') { + return fallback; + } + char* end = nullptr; + const unsigned long long parsed = std::strtoull(value, &end, 0); + if (end == value || *end != '\0') { + return fallback; + } + return static_cast(parsed); +} + +uint64_t EnvU64(const char* name, uint64_t fallback) +{ + return ParseU64(std::getenv(name), fallback); +} + +std::string RankEnvName(const char* prefix, int rank, const char* suffix) +{ + return std::string(prefix) + std::to_string(rank) + suffix; +} + +int RankEnvInt(const char* prefix, int rank, const char* suffix, const char* commonName, int fallback) +{ + const std::string rankName = RankEnvName(prefix, rank, suffix); + const char* rankValue = std::getenv(rankName.c_str()); + if (rankValue != nullptr && rankValue[0] != '\0') { + return ParseInt(rankValue, fallback); + } + return EnvInt(commonName, fallback); +} + +bool SyncCkeBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "sync_cke"; +} + +bool SyncCkeSetWaitBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "sync_cke_set_wait"; +} + +bool SyncCkePostOnlyBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "sync_cke_post_only"; +} + +bool LocalCkeBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "local_cke"; +} + +bool LocalCkePostOnlyBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "local_cke_post_only"; +} + +bool SyncXnPostOnlyBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "sync_xn_post_only"; +} + +bool SyncXnLoadPostOnlyBarrierMode() +{ + const char* value = std::getenv(kBarrierModeEnv); + return value != nullptr && std::string(value) == "sync_xn_load_post_only"; +} + +uint32_t RepositoryInstallWindowFromEnv() +{ + const char* value = std::getenv(kRepositoryInstallWindowEnv); + if (value == nullptr || value[0] == '\0') { + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION; + } + const std::string text(value); + if (text == "full_repository" || text == "full" || text == "1") { + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY; + } + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION; +} + +uint32_t RepositoryInstallDataLenModeFromEnv() +{ + const char* value = std::getenv(kRepositoryInstallDataLenModeEnv); + if (value == nullptr || value[0] == '\0') { + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES; + } + const std::string text(value); + if (text == "descriptor_bytes" || text == "descriptor" || text == "1") { + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES; + } + return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES; +} + +uint32_t RepositoryMemoryAllocModeFromEnv() +{ + const char* value = std::getenv(kRepositoryMemoryAllocModeEnv); + if (value == nullptr || value[0] == '\0') { + return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL; + } + const std::string text(value); + if (text == "acl_module3" || text == "acl_hccl_module" || text == "module3" || text == "1") { + return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3; + } + if (text == "rt_hbm" || text == "rt" || text == "runtime_hbm" || text == "2") { + return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM; + } + return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL; +} + +uint32_t InstallOrderFromEnv() +{ + const char* value = std::getenv(kInstallOrderEnv); + if (value == nullptr || value[0] == '\0') { + return TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST; + } + const std::string text(value); + if (text == "lower_layer_first" || text == "install_lower_layer_first" || text == "1") { + return TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST; + } + if (text == "repository_first" || text == "repo_first" || text == "0") { + return TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST; + } + return TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST; +} + +uint32_t DefaultSyncInstructionCount(uint32_t syncResourceCount) +{ + if (SyncCkeBarrierMode() || SyncCkeSetWaitBarrierMode()) { + return syncResourceCount * 2U + 1U; + } + if (SyncCkePostOnlyBarrierMode()) { + return syncResourceCount + 1U; + } + if (LocalCkePostOnlyBarrierMode()) { + return syncResourceCount; + } + if (SyncXnPostOnlyBarrierMode()) { + return kHcommStyleTask1PreludeInstructionCount + syncResourceCount; + } + if (SyncXnLoadPostOnlyBarrierMode()) { + return kHcommStyleTask1PreludeInstructionCount + syncResourceCount * 2U; + } + return kHcommStyleTask1PreludeInstructionCount + syncResourceCount * 2U; +} + +const char* FirstEnv(const char* a, const char* b, const char* c, const char* d) +{ + const char* value = std::getenv(a); + if (value != nullptr && value[0] != '\0') { + return value; + } + value = std::getenv(b); + if (value != nullptr && value[0] != '\0') { + return value; + } + value = std::getenv(c); + if (value != nullptr && value[0] != '\0') { + return value; + } + value = std::getenv(d); + return value != nullptr && value[0] != '\0' ? value : nullptr; +} + +int RankFromEnv() +{ + const char* rank = std::getenv("TILEXR_CCU_PROBE_RANK"); + if (rank != nullptr && rank[0] != '\0') { + return ParseInt(rank, 0); + } + return ParseInt(FirstEnv("PMI_RANK", "OMPI_COMM_WORLD_RANK", "MV2_COMM_WORLD_RANK", "RANK"), 0); +} + +int RankSizeFromEnv() +{ + const char* rankSize = std::getenv("TILEXR_CCU_PROBE_RANK_SIZE"); + if (rankSize != nullptr && rankSize[0] != '\0') { + return ParseInt(rankSize, 1); + } + return ParseInt(FirstEnv("PMI_SIZE", "OMPI_COMM_WORLD_SIZE", "MV2_COMM_WORLD_SIZE", "RANK_SIZE"), 1); +} + +std::vector BuildP2pCcuCopyPattern(int rank, size_t bytes) +{ + std::vector pattern(bytes); + for (size_t i = 0; i < bytes; ++i) { + pattern[i] = static_cast((static_cast(rank + 1) * 17U + i * 13U) & 0xffU); + } + return pattern; +} + +int InitP2pCcuCopyState(int rank, int peer, P2pCcuCopyState* state) +{ + if (state == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->bytes = static_cast(EnvInt(kP2pCcuCopyBytesEnv, 4096)); + if (state->bytes == 0) { + state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + return state->initRet; + } + state->expected = BuildP2pCcuCopyPattern(peer, state->bytes); + state->observed.assign(state->bytes, 0); + const std::vector source = BuildP2pCcuCopyPattern(rank, state->bytes); + std::vector destination(state->bytes, 0xa5U); + + int ret = state->source.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = state->destination.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->source.ptr, + state->bytes, + source.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + state->initRet = ret; + return ret; +} + +int CheckP2pCcuCopyState(P2pCcuCopyState* state) +{ + if (state == nullptr || state->destination.ptr == nullptr || state->bytes == 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->readRet = aclrtMemcpy( + state->observed.data(), + state->observed.size(), + state->destination.ptr, + state->bytes, + ACL_MEMCPY_DEVICE_TO_HOST); + if (state->readRet != ACL_SUCCESS) { + return state->readRet; + } + state->mismatchCount = 0; + for (size_t i = 0; i < state->bytes; ++i) { + if (state->observed[i] != state->expected[i]) { + ++state->mismatchCount; + } + } + state->passed = state->mismatchCount == 0; + return state->passed ? ACL_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + +void PrintP2pCcuCopy( + int rank, + int peer, + const P2pCcuCopyState& state, + int prepareRet, + int submitRet, + int syncRet) +{ + const uint32_t firstObserved = state.observed.empty() ? 0U : state.observed.front(); + const uint32_t firstExpected = state.expected.empty() ? 0U : state.expected.front(); + std::cout << "tilexr_ccu_direct_smoke p2pCcuCopy" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << state.bytes + << " initRet=" << state.initRet + << " prepareRet=" << prepareRet + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " readRet=" << state.readRet + << " mismatches=" << state.mismatchCount + << " firstObserved=0x" << std::hex << firstObserved + << " firstExpected=0x" << firstExpected + << std::dec + << " passed=" << (state.passed ? 1 : 0) + << std::endl; +} + +int RunP2pCcuCopy( + int rank, + int peer, + P2pCcuCopyState* state, + int prepareRet, + int submitRet, + int syncRet) +{ + if (state == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { + (void)CheckP2pCcuCopyState(state); + } + PrintP2pCcuCopy(rank, peer, *state, prepareRet, submitRet, syncRet); + return state->passed ? TileXR::TILEXR_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + +int DeviceFromList(const char* devices, int rank, int fallback) +{ + if (devices == nullptr || devices[0] == '\0') { + return fallback; + } + std::string list(devices); + size_t start = 0; + int index = 0; + while (start <= list.size()) { + const size_t comma = list.find(',', start); + const size_t end = comma == std::string::npos ? list.size() : comma; + if (index == rank && end > start) { + return ParseInt(list.substr(start, end - start).c_str(), fallback); + } + if (comma == std::string::npos) { + break; + } + start = comma + 1; + ++index; + } + return fallback; +} + +int DeviceFromEnv(int rank) +{ + const char* explicitDevice = std::getenv("TILEXR_CCU_PROBE_DEVICE"); + if (explicitDevice != nullptr && explicitDevice[0] != '\0') { + return ParseInt(explicitDevice, 0); + } + const int firstDevice = EnvInt("TILEXR_TEST_FIRST_NPU", 0); + return DeviceFromList(std::getenv("TILEXR_TEST_DEVICES"), rank, firstDevice + rank); +} + +int InitCommForDirectCcuSmoke(int commDomain, int rankSize, int rank, TileXRCommPtr* raw) +{ + if (EnvFlag(kDirectCcuOnlyInitEnv)) { + return TileXRCommInitRankDirectCcuWithDomain(commDomain, rankSize, rank, raw); + } + return TileXRCommInitRankWithDomain(commDomain, rankSize, rank, raw); +} + +TileXRDirectCcuPrepareOptions MakePrepareOptions(int rank, int rankSize, int device) +{ + TileXRDirectCcuPrepareOptions options {}; + options.syncResourceCount = static_cast(EnvInt("TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT", 1)); + options.sqeArgCount = + static_cast(EnvInt("TILEXR_CCU_PROBE_SQE_ARG_COUNT", TILEXR_DIRECT_CCU_SQE_ARGS_LEN)); + const uint32_t defaultSyncInstructionCount = DefaultSyncInstructionCount(options.syncResourceCount); + options.syncInstructionCount = + static_cast(EnvInt("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT", defaultSyncInstructionCount)); + options.bindingsPerSyncResource = static_cast(EnvInt("TILEXR_CCU_PROBE_BINDINGS_PER_RESOURCE", 1)); + options.missionStartId = static_cast(EnvInt("TILEXR_CCU_PROBE_MISSION_START", 1)); + options.instructionStartId = static_cast(EnvInt("TILEXR_CCU_PROBE_INSTRUCTION_START", 1)); + options.missionInstructionStartId = + static_cast(EnvInt("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START", 0)); + options.xnStartId = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_XN_START", "TILEXR_CCU_PROBE_XN_START", 1)); + options.gsaStartId = static_cast(EnvInt("TILEXR_CCU_PROBE_GSA_START", 0)); + options.remoteXnStartId = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_XN_START", "TILEXR_CCU_PROBE_REMOTE_XN_START", 0)); + options.remoteXnCount = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_XN_COUNT", "TILEXR_CCU_PROBE_REMOTE_XN_COUNT", 0)); + options.ckeStartId = static_cast(EnvInt("TILEXR_CCU_PROBE_CKE_START", 1)); + options.channelStartId = static_cast(EnvInt("TILEXR_CCU_PROBE_CHANNEL_START", 1)); + options.localWaitCkeStartId = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_LOCAL_WAIT_CKE_START", kLocalWaitCkeStartEnv, 0)); + options.localWaitCkeCount = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_LOCAL_WAIT_CKE_COUNT", kLocalWaitCkeCountEnv, 0)); + options.remoteNotifyCkeStartId = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_NOTIFY_CKE_START", kRemoteNotifyCkeStartEnv, 0)); + options.remoteNotifyCkeCount = static_cast( + RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_NOTIFY_CKE_COUNT", kRemoteNotifyCkeCountEnv, 0)); + options.repositoryInstallWindow = RepositoryInstallWindowFromEnv(); + options.repositoryInstallDataLenMode = RepositoryInstallDataLenModeFromEnv(); + options.repositoryMemoryAllocMode = RepositoryMemoryAllocModeFromEnv(); + options.installOrder = InstallOrderFromEnv(); + options.deviceId = static_cast(device); + options.rank = static_cast(rank); + options.provider = rankSize > 0 ? "tilexr-direct-smoke-probe" : ""; + return options; +} + +void PrintInstallReport( + const char* prefix, + int ret, + const TileXRDirectCcuPrepareReport& report) +{ + std::cout << prefix + << " ret=" << ret + << " pipelineBuilt=" << (report.pipelineBuilt ? 1 : 0) + << " installAttempted=" << (report.installAttempted ? 1 : 0) + << " installSucceeded=" << (report.installSucceeded ? 1 : 0) + << " submitReady=" << (report.submitReady ? 1 : 0) + << " requiredInstallSurfaceCount=" << report.requiredInstallSurfaceCount + << " publicVerifiedInstallSurfaceCount=" << report.publicVerifiedInstallSurfaceCount + << " missingInstallSurfaceCount=" << report.missingInstallSurfaceCount + << " taskCount=" << report.taskCount + << " submitTaskCount=" << report.submitTaskCount + << " message=\"" << report.message << "\"" + << std::endl; +} + +void PrintSubmitReport( + const char* prefix, + int ret, + const TileXRDirectCcuSubmitReport& report) +{ + std::cout << prefix + << " ret=" << ret + << " submitted=" << (report.submitted ? 1 : 0) + << " taskCount=" << report.taskCount + << " submittedTaskCount=" << report.submittedTaskCount + << " message=\"" << report.message << "\"" + << std::endl; +} + +int SubmitPreparedWithSelector( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t taskCount, + void* stream, + TileXRDirectCcuSubmitReport* report) +{ + const char* value = std::getenv(kSubmitTaskSelectorEnv); + const std::string selector = value == nullptr ? "all" : std::string(value); + if (selector.empty() || selector == "all") { + return TileXRDirectCcuSubmitPrepared(prepared, stream, report); + } + const bool selectFirst = selector == "first"; + const bool selectSecond = selector == "second"; + if (!selectFirst && !selectSecond) { + if (report != nullptr) { + *report = TileXRDirectCcuSubmitReport {}; + std::strncpy( + report->message, + "invalid TILEXR_CCU_DIRECT_SMOKE_SUBMIT_TASK_SELECTOR", + sizeof(report->message) - 1U); + } + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint32_t selectedIndex = selectFirst ? 0U : 1U; + std::cout << "tilexr_ccu_direct_smoke submitTaskSelector=" + << selector + << " selectedIndex=" << selectedIndex + << " preparedTaskCount=" << taskCount + << std::endl; + if (selectedIndex >= taskCount) { + if (report != nullptr) { + *report = TileXRDirectCcuSubmitReport {}; + report->taskCount = taskCount; + std::strncpy(report->message, "selected direct CCU submit task is missing", sizeof(report->message) - 1U); + } + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + + return TileXRDirectCcuSubmitPreparedTask(prepared, selectedIndex, stream, report); +} + +void PrintSubmitTiming( + int rank, + int delayMs, + int submitRet, + int syncRet, + long long submitMs, + long long syncMs) +{ + std::cout << "tilexr_ccu_direct_smoke submitTiming" + << " rank=" << rank + << " preSubmitDelayMs=" << delayMs + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " submitMs=" << submitMs + << " syncMs=" << syncMs + << std::endl; +} + +void PrintConfig( + const TileXRDirectCcuPrepareOptions& options, + int rankSize) +{ + std::cout << "tilexr_ccu_direct_smoke config" + << " rank=" << options.rank + << " rankSize=" << rankSize + << " device=" << options.deviceId + << " syncResourceCount=" << options.syncResourceCount + << " sqeArgCount=" << options.sqeArgCount + << " syncInstructionCount=" << options.syncInstructionCount + << " bindingsPerSyncResource=" << options.bindingsPerSyncResource + << " missionStartId=" << options.missionStartId + << " instructionStartId=" << options.instructionStartId + << " missionInstructionStartId=" << options.missionInstructionStartId + << " xnStartId=" << options.xnStartId + << " gsaStartId=" << options.gsaStartId + << " remoteXnStartId=" << options.remoteXnStartId + << " remoteXnCount=" << options.remoteXnCount + << " ckeStartId=" << options.ckeStartId + << " channelStartId=" << options.channelStartId + << " localWaitCkeStartId=" << options.localWaitCkeStartId + << " localWaitCkeCount=" << options.localWaitCkeCount + << " remoteNotifyCkeStartId=" << options.remoteNotifyCkeStartId + << " remoteNotifyCkeCount=" << options.remoteNotifyCkeCount + << " repositoryInstallWindow=" << options.repositoryInstallWindow + << " repositoryInstallDataLenMode=" << options.repositoryInstallDataLenMode + << " repositoryMemoryAllocMode=" << options.repositoryMemoryAllocMode + << " installOrder=" << options.installOrder + << " barrierMode=\"" << (std::getenv(kBarrierModeEnv) == nullptr ? "" : std::getenv(kBarrierModeEnv)) + << "\"" + << " resourceWindowTokenId=\"" + << (std::getenv(kResourceWindowTokenIdEnv) == nullptr ? "" : std::getenv(kResourceWindowTokenIdEnv)) + << "\"" + << " resourceWindowRawTokenId=\"" + << (std::getenv(kResourceWindowRawTokenIdEnv) == nullptr ? "" : std::getenv(kResourceWindowRawTokenIdEnv)) + << "\"" + << " resourceWindowTokenValue=\"" + << (std::getenv(kResourceWindowTokenValueEnv) == nullptr ? "" : std::getenv(kResourceWindowTokenValueEnv)) + << "\"" + << " provider=\"" << (options.provider == nullptr ? "" : options.provider) << "\"" + << std::endl; +} + +void PrintPreparedTasks(TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) +{ + std::cout << "tilexr_ccu_direct_smoke preparedTasks" + << " count=" << taskCount; + const uint32_t previewCount = taskCount < 2U ? taskCount : 2U; + for (size_t i = 0; i < previewCount; ++i) { + TileXRDirectCcuTaskInfo task; + const int ret = TileXRDirectCcuGetPreparedTask(prepared, static_cast(i), &task); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cout << " task" << i << ".ret=" << ret; + continue; + } + std::cout << " task" << i + << ".dieId=" << static_cast(task.dieId) + << " task" << i << ".missionId=" << static_cast(task.missionId) + << " task" << i << ".timeout=" << task.timeout + << " task" << i << ".instStartId=" << task.instStartId + << " task" << i << ".instCnt=" << task.instCnt + << " task" << i << ".key=0x" << std::hex << task.key << std::dec + << " task" << i << ".argSize=" << task.argSize; + for (uint32_t arg = 0; arg < TILEXR_DIRECT_CCU_SQE_ARGS_LEN; ++arg) { + std::cout << " task" << i + << ".arg" << arg << "=0x" + << std::hex << task.args[arg] << std::dec; + } + } + std::cout << std::endl; +} + +void PrintInstructionReadback(TileXRCommPtr raw, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) +{ + if (!EnvFlag(kReadbackInstructionsEnv)) { + return; + } + if (raw == nullptr) { + std::cout << "tilexr_ccu_direct_smoke instructionReadback ret=" + << TileXR::TILEXR_ERROR_PARA_CHECK_FAIL + << " message=\"missing TileXRComm\"" + << std::endl; + return; + } + for (uint32_t taskIndex = 0; taskIndex < taskCount; ++taskIndex) { + TileXRDirectCcuTaskInfo task; + const int taskRet = TileXRDirectCcuGetPreparedTask(prepared, taskIndex, &task); + if (taskRet != TileXR::TILEXR_SUCCESS) { + std::cout << "tilexr_ccu_direct_smoke instructionReadback" + << " task=" << taskIndex + << " ret=" << taskRet + << " message=\"failed to read prepared task\"" + << std::endl; + continue; + } + + constexpr uint32_t kMaxReadbackInstructionCount = 8U; + const uint32_t readCount = task.instCnt < kMaxReadbackInstructionCount ? + task.instCnt : kMaxReadbackInstructionCount; + std::vector readback(readCount); + TileXRDirectCcuInstructionReadbackReport report; + const int readRet = TileXRCommReadDirectCcuInstructions( + raw, + static_cast(task.dieId), + static_cast(task.instStartId), + readCount, + readback.data(), + &report); + std::cout << "tilexr_ccu_direct_smoke instructionReadback" + << " task=" << taskIndex + << " ret=" << readRet + << " dieId=" << static_cast(task.dieId) + << " instStartId=" << task.instStartId + << " requestedCount=" << task.instCnt + << " readCount=" << readCount + << " opcode=" << report.opcode + << " driverRet=" << report.driverRet + << " opRet=" << report.opRet + << " message=\"" << report.message << "\""; + if (readRet == TileXR::TILEXR_SUCCESS) { + for (uint32_t i = 0; i < readCount; ++i) { + std::cout << " instr" << i << "="; + for (uint32_t word = 0; word < 4U; ++word) { + if (word != 0) { + std::cout << ","; + } + std::cout << "0x" << std::hex << std::nouppercase << readback[i].words[word] << std::dec; + } + } + } + std::cout << std::endl; + } +} + +bool CollectiveSubmitReadyGateConfigured() +{ + const char* readyDir = std::getenv(kReadyDirEnv); + return readyDir != nullptr && readyDir[0] != '\0'; +} + +std::string SubmitReadinessPath(int rank) +{ + const char* readyDir = std::getenv(kReadyDirEnv); + if (readyDir == nullptr || readyDir[0] == '\0') { + return {}; + } + return std::string(readyDir) + "/rank" + std::to_string(rank) + ".ready"; +} + +bool WriteSubmitReadiness(int rank, bool ready) +{ + const std::string path = SubmitReadinessPath(rank); + if (path.empty()) { + return true; + } + std::ofstream out(path.c_str(), std::ios::out | std::ios::trunc); + if (!out) { + return false; + } + out << (ready ? 1 : 0) << "\n"; + return static_cast(out); +} + +bool ReadSubmitReadiness(int rank, bool* ready) +{ + if (ready == nullptr) { + return false; + } + const std::string path = SubmitReadinessPath(rank); + if (path.empty()) { + return false; + } + std::ifstream in(path.c_str()); + if (!in) { + return false; + } + int value = 0; + in >> value; + if (!in) { + return false; + } + *ready = value != 0; + return true; +} + +bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) +{ + if (!CollectiveSubmitReadyGateConfigured()) { + return localReady; + } + const bool wrote = WriteSubmitReadiness(rank, localReady); + const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); + const auto start = std::chrono::steady_clock::now(); + for (;;) { + bool allSeen = wrote; + bool allReady = wrote && localReady; + for (int peer = 0; peer < rankSize; ++peer) { + bool peerReady = false; + if (!ReadSubmitReadiness(peer, &peerReady)) { + allSeen = false; + allReady = false; + break; + } + allReady = allReady && peerReady; + } + if (allSeen) { + std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" + << " rank=" << rank + << " localReady=" << (localReady ? 1 : 0) + << " allRanksReady=" << (allReady ? 1 : 0) + << std::endl; + return allReady; + } + const auto elapsedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + if (elapsedMs >= timeoutMs) { + std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" + << " rank=" << rank + << " localReady=" << (localReady ? 1 : 0) + << " allRanksReady=0" + << " timeoutMs=" << timeoutMs + << std::endl; + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } +} + +bool CollectiveSubmitDoneGateConfigured() +{ + const char* doneDir = std::getenv(kDoneDirEnv); + return doneDir != nullptr && doneDir[0] != '\0'; +} + +std::string SubmitDonePath(int rank) +{ + const char* doneDir = std::getenv(kDoneDirEnv); + if (doneDir == nullptr || doneDir[0] == '\0') { + return {}; + } + return std::string(doneDir) + "/rank" + std::to_string(rank) + ".done"; +} + +bool WriteSubmitDone(int rank, int result) +{ + const std::string path = SubmitDonePath(rank); + if (path.empty()) { + return true; + } + std::ofstream out(path.c_str(), std::ios::out | std::ios::trunc); + if (!out) { + return false; + } + out << result << "\n"; + return static_cast(out); +} + +bool ReadSubmitDone(int rank, int* result) +{ + if (result == nullptr) { + return false; + } + const std::string path = SubmitDonePath(rank); + if (path.empty()) { + return false; + } + std::ifstream in(path.c_str()); + if (!in) { + return false; + } + in >> *result; + return static_cast(in); +} + +bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) +{ + if (!CollectiveSubmitDoneGateConfigured()) { + return true; + } + const bool wrote = WriteSubmitDone(rank, localResult); + const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); + const auto start = std::chrono::steady_clock::now(); + for (;;) { + bool allSeen = wrote; + bool allSucceeded = wrote && localResult == 0; + for (int peer = 0; peer < rankSize; ++peer) { + int peerResult = 0; + if (!ReadSubmitDone(peer, &peerResult)) { + allSeen = false; + allSucceeded = false; + break; + } + allSucceeded = allSucceeded && peerResult == 0; + } + if (allSeen) { + std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" + << " rank=" << rank + << " localResult=" << localResult + << " allRanksDone=1" + << " allRanksSucceeded=" << (allSucceeded ? 1 : 0) + << std::endl; + return true; + } + const auto elapsedMs = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start).count(); + if (elapsedMs >= timeoutMs) { + std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" + << " rank=" << rank + << " localResult=" << localResult + << " allRanksDone=0" + << " timeoutMs=" << timeoutMs + << std::endl; + return false; + } + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } +} + +int RunPreparedSmokeForRank(TileXRCommPtr raw, int rank, int rankSize, int device) +{ + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + const int peer = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; + const bool p2pCcuCopyEnabled = EnvFlag(kP2pCcuCopyEnv); + P2pCcuCopyState p2pCcuCopy; + if (p2pCcuCopyEnabled) { + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 7; + p2pCcuCopy.initRet = InitP2pCcuCopyState(rank, peer, &p2pCcuCopy); + } + PrintConfig(options, rankSize); + + TileXRDirectCcuPreparedTasksPtr prepared = nullptr; + TileXRDirectCcuPrepareReport installReport; + int ret = p2pCcuCopyEnabled && p2pCcuCopy.initRet != ACL_SUCCESS ? + p2pCcuCopy.initRet : + p2pCcuCopyEnabled ? + [&]() { + TileXRDirectCcuMemoryCopyPrepareOptions copyOptions {}; + copyOptions.prepare = options; + copyOptions.localSourceAddr = reinterpret_cast(p2pCcuCopy.source.ptr); + copyOptions.localDestinationAddr = reinterpret_cast(p2pCcuCopy.destination.ptr); + copyOptions.bytes = p2pCcuCopy.bytes; + copyOptions.peerRank = static_cast(peer); + copyOptions.direction = TILEXR_DIRECT_CCU_MEMORY_COPY_REMOTE_TO_LOCAL; + return TileXRCommPrepareDirectCcuMemoryCopy(raw, ©Options, &prepared, &installReport); + }() : + TileXRCommPrepareDirectCcu(raw, &options, &prepared, &installReport); + PrintInstallReport("tilexr_ccu_direct_smoke prepare", ret, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(raw, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + ret == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + false; + if (ret != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_direct_smoke submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_direct_smoke submit skipped reason=\"prepare did not reach submitReady\"" + << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + const int delayRank = EnvInt(kDelayRankEnv, -1); + const int preSubmitDelayMs = EnvInt(kPreSubmitDelayMsEnv, 0); + const int effectiveDelayMs = rank == delayRank && preSubmitDelayMs > 0 ? preSubmitDelayMs : 0; + if (effectiveDelayMs > 0) { + std::cout << "tilexr_ccu_direct_smoke preSubmitDelay" + << " rank=" << rank + << " delayMs=" << effectiveDelayMs + << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(effectiveDelayMs)); + } + + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const char* submitTaskSelector = std::getenv(kSubmitTaskSelectorEnv); + const bool submitAllTasks = submitTaskSelector == nullptr || submitTaskSelector[0] == '\0' || + std::string(submitTaskSelector) == "all"; + const int submitRet = submitAllTasks ? + TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport) : + SubmitPreparedWithSelector(prepared, installReport.submitTaskCount, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_direct_smoke submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + TraceLifecycle("before aclrtSynchronizeStream"); + const int syncRet = aclrtSynchronizeStream(stream); + TraceLifecycle("after aclrtSynchronizeStream"); + const auto syncEnd = std::chrono::steady_clock::now(); + PrintSubmitTiming( + rank, + effectiveDelayMs, + submitRet, + syncRet, + std::chrono::duration_cast(submitEnd - submitBegin).count(), + std::chrono::duration_cast(syncEnd - syncBegin).count()); + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke aclrtSynchronizeStream ret=" << syncRet << std::endl; + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (p2pCcuCopyEnabled) { + const int p2pCcuCopyRet = RunP2pCcuCopy(rank, peer, &p2pCcuCopy, ret, submitRet, syncRet); + if (p2pCcuCopyRet != TileXR::TILEXR_SUCCESS && finalRet == 0) { + finalRet = 14; + } + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + TraceLifecycle("before aclrtDestroyStream"); + aclrtDestroyStream(stream); + TraceLifecycle("after aclrtDestroyStream"); + } + } + + if (prepared != nullptr && ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_direct_smoke fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + if (prepared != nullptr) { + TraceLifecycle("before TileXRDirectCcuDestroyPrepared"); + const int preparedDestroyRet = TileXRDirectCcuDestroyPrepared(prepared); + TraceLifecycle("after TileXRDirectCcuDestroyPrepared"); + if (preparedDestroyRet != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke prepared destroy ret=" << preparedDestroyRet << std::endl; + finalRet = finalRet == 0 ? 11 : finalRet; + } + } + return finalRet; +} + +int RunThreadModeSmoke(int rankSize) +{ + std::cout << "tilexr_ccu_direct_smoke threadMode begin" + << " rankSize=" << rankSize + << std::endl; + std::vector devices(static_cast(rankSize)); + for (int rank = 0; rank < rankSize; ++rank) { + devices[rank] = DeviceFromEnv(rank); + } + std::vector comms(static_cast(rankSize), nullptr); + int ret = TileXRCommInitAll(static_cast(rankSize), devices.data(), comms.data()); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke threadMode comm init ret=" << ret << std::endl; + return 5; + } + + std::vector rankResults(static_cast(rankSize), 0); + std::vector rankThreads; + rankThreads.reserve(static_cast(rankSize)); + for (int rank = 0; rank < rankSize; ++rank) { + rankThreads.emplace_back([&, rank]() { + const int setDeviceRet = aclrtSetDevice(devices[rank]); + if (setDeviceRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke threadMode aclrtSetDevice ret=" + << setDeviceRet + << " rank=" << rank + << " device=" << devices[rank] + << std::endl; + rankResults[rank] = 14; + return; + } + rankResults[rank] = RunPreparedSmokeForRank(comms[rank], rank, rankSize, devices[rank]); + }); + } + for (auto& rankThread : rankThreads) { + rankThread.join(); + } + + int finalRet = 0; + for (int rank = 0; rank < rankSize; ++rank) { + std::cout << "tilexr_ccu_direct_smoke threadMode rank=" << rank + << " ret=" << rankResults[rank] + << std::endl; + if (rankResults[rank] != 0 && finalRet == 0) { + finalRet = rankResults[rank]; + } + } + for (auto comm : comms) { + if (comm == nullptr) { + continue; + } + const int destroyRet = TileXRCommDestroy(comm); + if (destroyRet != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke threadMode destroy ret=" << destroyRet << std::endl; + finalRet = finalRet == 0 ? 10 : finalRet; + } + } + return finalRet; +} + +} // namespace + +int main() +{ + if (!EnvFlag(kEnableEnv)) { + std::cout << "tilexr_ccu_direct_smoke skipped set " + << kEnableEnv << "=1 to run private C++ integration probe" + << std::endl; + return 0; + } + + const int rank = RankFromEnv(); + const int rankSize = RankSizeFromEnv(); + const int device = DeviceFromEnv(rank); + const int commDomain = EnvInt("TILEXR_CCU_PROBE_COMM_DOMAIN", 0); + + if (rankSize <= 1) { + std::cout << "tilexr_ccu_direct_smoke skipped rankSize=" << rankSize + << " reason=\"direct CCU prepare requires a multi-rank communicator\"" + << std::endl; + return 0; + } + if (rank < 0 || rank >= rankSize) { + std::cerr << "tilexr_ccu_direct_smoke invalid rank=" << rank + << " rankSize=" << rankSize << std::endl; + return 2; + } + if (EnvFlag(kThreadModeEnv)) { + const int aclRet = aclInit(nullptr); + if (aclRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke aclInit ret=" << aclRet << std::endl; + return 3; + } + const int threadRet = RunThreadModeSmoke(rankSize); + aclFinalize(); + return threadRet; + } + + std::cout << "tilexr_ccu_direct_smoke begin" + << " rank=" << rank + << " rankSize=" << rankSize + << " device=" << device + << " commDomain=" << commDomain + << std::endl; + + int ret = aclInit(nullptr); + if (ret != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke aclInit ret=" << ret << std::endl; + return 3; + } + + ret = aclrtSetDevice(device); + if (ret != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke aclrtSetDevice ret=" << ret + << " device=" << device << std::endl; + aclFinalize(); + return 4; + } + + TileXRCommPtr raw = nullptr; + ret = InitCommForDirectCcuSmoke(commDomain, rankSize, rank, &raw); + if (ret != TileXR::TILEXR_SUCCESS || raw == nullptr) { + std::cerr << "tilexr_ccu_direct_smoke comm init ret=" << ret + << " raw=" << raw << std::endl; + aclrtResetDevice(device); + aclFinalize(); + return 5; + } + + int finalRet = RunPreparedSmokeForRank(raw, rank, rankSize, device); + if (ShouldFastExitAfterPrepareFailure(finalRet)) { + std::cout << "tilexr_ccu_direct_smoke fastExitOnPrepareFailure=1" + << " ret=" << finalRet + << " reason=\"" << FastExitReasonForReturnCode(finalRet) << "\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_direct_smoke fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping communicator cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + TraceLifecycle("before TileXRCommDestroy"); + const int destroyRet = TileXRCommDestroy(raw); + TraceLifecycle("after TileXRCommDestroy"); + if (destroyRet != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke destroy ret=" << destroyRet << std::endl; + finalRet = finalRet == 0 ? 10 : finalRet; + } + TraceLifecycle("before aclrtResetDevice"); + aclrtResetDevice(device); + TraceLifecycle("after aclrtResetDevice"); + TraceLifecycle("before aclFinalize"); + aclFinalize(); + TraceLifecycle("after aclFinalize"); + return finalRet; +} diff --git a/tests/ccu/check_tile_comm_no_hcomm_deps.sh b/tests/ccu/check_tile_comm_no_hcomm_deps.sh new file mode 100644 index 00000000..a0e8ba9b --- /dev/null +++ b/tests/ccu/check_tile_comm_no_hcomm_deps.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +lib="${1:-install/lib/libtile-comm.so}" + +if [ ! -f "${lib}" ]; then + echo "ERROR: ${lib} not found" >&2 + exit 1 +fi + +if command -v readelf >/dev/null 2>&1; then + needed=$(readelf -d "${lib}" 2>/dev/null | grep -E 'NEEDED' || true) +else + needed="" +fi + +deps=$(ldd "${lib}" 2>/dev/null || true) +forbidden='libhcomm\.so|libhccl_v2\.so|libhccl_fwk\.so|libmc2_client\.so|HcclCcuKernel|HcclGetCcuTaskInfo|HcomGetCcuTaskInfo|HcclChannelAcquire|HcclGetChannelForCcu|HcclAllocAlgResourceCcu|HcommChannelNotify|HcommChannelFence|rtGetNotifyAddress|HrtCcuLaunch|HrtGetDevResAddress|HrtReleaseDevResAddress|HrtNotifyGetAddr|HrtRaCustomChannel|HrtCntNotify|CcuResBatchAllocator|CcuResRepository|CcuDeviceManager|CcuDevMgrImp|CcuRepContext|CcuKernelMgr|CtxMgrImp|CcuInstrInfo|CcuTaskParam|CcuTaskArg|GeneTaskParam|GetMissionKey|SetMissionId|SetMissionKey|SetInstrId|SetCcuInstrInfo|LoadInstruction|AllocIns|AllocCke|AllocXn|COMM_ENGINE_CCU|COMM_PROTOCOL_UBC_CTP|HCCL_SERVER_TYPE_CCU|RT_RES_TYPE_CCU_CKE|RT_RES_TYPE_CCU_XN' + +printf '%s\n' "${needed}" +printf '%s\n' "${deps}" + +if printf '%s\n%s\n' "${needed}" "${deps}" | grep -E "${forbidden}" >/dev/null; then + echo "ERROR: libtile-comm.so links an hcomm/HCCL CCU reference library" >&2 + exit 1 +fi + +if command -v nm >/dev/null 2>&1; then + symbol_hits=$(nm -D "${lib}" 2>/dev/null | c++filt | grep -E "${forbidden}" || true) +else + symbol_hits="" +fi +if [ -n "${symbol_hits}" ]; then + printf '%s\n' "${symbol_hits}" >&2 + echo "ERROR: libtile-comm.so exports or imports private hcomm/HCCL CCU symbols" >&2 + exit 1 +fi + +if command -v strings >/dev/null 2>&1; then + string_hits=$(strings -a "${lib}" 2>/dev/null | grep -E "${forbidden}" || true) +else + string_hits="" +fi +if [ -n "${string_hits}" ]; then + printf '%s\n' "${string_hits}" >&2 + echo "ERROR: libtile-comm.so contains private hcomm/HCCL CCU references" >&2 + exit 1 +fi + +echo "TileXR CCU dependency guard passed: no hcomm/HCCL private CCU dependency or symbol reference" diff --git a/tests/ccu/run_tilexr_ccu_direct_evening_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_evening_smoke.sh new file mode 100644 index 00000000..142552df --- /dev/null +++ b/tests/ccu/run_tilexr_ccu_direct_evening_smoke.sh @@ -0,0 +1,538 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026 TileXR Project +# +# One-shot direct CCU bring-up wrapper for the reserved 20:00+ hardware window. +# It still fails closed: submit/barrier/P2P stages run only after prepare logs +# show submitReady=1 for both ranks. + +set -euo pipefail + +for arg in "$@"; do + case "${arg}" in + --dry-run) + export TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN=1 + ;; + *) + echo "ERROR: unknown argument: ${arg}" >&2 + exit 2 + ;; + esac +done + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${repo_root}" + +set_evening_defaults() +{ + export TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 + export TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI=1 + if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-}" = "" ]; then + export TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=0 + else + export TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE + fi + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_SMOKE_DEVICES="${TILEXR_CCU_SMOKE_DEVICES:-0,1}" + export TILEXR_CCU_DIRECT_TRACE="${TILEXR_CCU_DIRECT_TRACE:-1}" + export TILEXR_LOG_LEVEL="${TILEXR_LOG_LEVEL:-INFO}" + export TILEXR_CCU_SMOKE_TIMEOUT="${TILEXR_CCU_SMOKE_TIMEOUT:-120}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-143}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_INSTALL_ORDER="${TILEXR_CCU_DIRECT_INSTALL_ORDER:-lower_layer_first}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_GSA_START="${TILEXR_CCU_PROBE_GSA_START:-510}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + if [ "${TILEXR_CCU_DIRECT_BARRIER_MODE:-}" = "" ]; then + export TILEXR_CCU_DIRECT_BARRIER_MODE=sync_cke + else + export TILEXR_CCU_DIRECT_BARRIER_MODE + fi + if [ "${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-}" = "" ]; then + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE=hcomm_cap + else + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE + fi + export TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES="${TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES:-prepare,submit,barrier,p2p}" + export TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES="${TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES:-acl,acl_module3,rt_hbm}" + export TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES="${TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES:-}" +} + +prepare_profiles=() +prepare_profile_name="" +prepare_profile_alloc="" +prepare_profile_window="" +prepare_profile_data_len_mode="" +prepare_profile_install_order="" +prepare_profile_pfe_offset_source="" +prepare_profile_pfe_partition="" + +build_prepare_profiles() +{ + prepare_profiles=() + if [ "${TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES:-}" != "" ]; then + IFS=',' read -r -a prepare_profiles <<< "${TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES}" + return + fi + + local prepare_alloc_modes=() + local prepare_mode + IFS=',' read -r -a prepare_alloc_modes <<< "${TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES}" + for prepare_mode in "${prepare_alloc_modes[@]}"; do + prepare_mode="${prepare_mode//[[:space:]]/}" + if [ "${prepare_mode}" = "" ]; then + continue + fi + prepare_profiles+=( + "${prepare_mode}:${prepare_mode}:${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW}:${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE}:${TILEXR_CCU_DIRECT_INSTALL_ORDER}" + ) + done +} + +parse_prepare_profile() +{ + local profile="$1" + local extra1="" + local extra2="" + local extra3="" + IFS=':' read -r \ + prepare_profile_name \ + prepare_profile_alloc \ + prepare_profile_window \ + prepare_profile_data_len_mode \ + prepare_profile_install_order \ + extra1 \ + extra2 \ + extra3 <<< "${profile}" + prepare_profile_name="${prepare_profile_name//[[:space:]]/}" + prepare_profile_alloc="${prepare_profile_alloc//[[:space:]]/}" + prepare_profile_window="${prepare_profile_window//[[:space:]]/}" + prepare_profile_data_len_mode="${prepare_profile_data_len_mode//[[:space:]]/}" + prepare_profile_install_order="${prepare_profile_install_order//[[:space:]]/}" + prepare_profile_pfe_offset_source="${extra1//[[:space:]]/}" + prepare_profile_pfe_partition="${extra2//[[:space:]]/}" + if [ "${prepare_profile_name}" = "" ] || + [ "${prepare_profile_alloc}" = "" ] || + [ "${prepare_profile_window}" = "" ] || + [ "${prepare_profile_data_len_mode}" = "" ] || + [ "${prepare_profile_install_order}" = "" ] || + [ "${extra3}" != "" ]; then + echo "ERROR: invalid prepare profile '${profile}', expected name:alloc:window:dataLenMode:installOrder[:pfeOffsetSource:pfePartition]" >&2 + exit 21 + fi + if { [ "${prepare_profile_pfe_offset_source}" != "" ] && [ "${prepare_profile_pfe_partition}" = "" ]; } || + { [ "${prepare_profile_pfe_offset_source}" = "" ] && [ "${prepare_profile_pfe_partition}" != "" ]; }; then + echo "ERROR: invalid prepare profile '${profile}', pfeOffsetSource and pfePartition must be provided together" >&2 + exit 21 + fi +} + +print_prepare_profile_dry_run() +{ + build_prepare_profiles + local index=0 + local profile + for profile in "${prepare_profiles[@]}"; do + parse_prepare_profile "${profile}" + echo "dryRun prepareProfile[${index}] name=${prepare_profile_name} alloc=${prepare_profile_alloc} window=${prepare_profile_window} dataLenMode=${prepare_profile_data_len_mode} installOrder=${prepare_profile_install_order} pfeOffsetSource=${prepare_profile_pfe_offset_source:-default} pfePartition=${prepare_profile_pfe_partition:-default}" + index=$((index + 1)) + done +} + +print_resource_window_token_dry_run() +{ + local token_field + for token_field in TOKEN_ID RAW_TOKEN_ID TOKEN_VALUE; do + local token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}" + local token_value="${!token_var:-}" + local rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" + local rank0_token_value="${!rank0_token_var:-}" + local rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" + local rank1_token_value="${!rank1_token_var:-}" + if [ "${token_value}" != "" ]; then + echo "${token_var}=${token_value}" + fi + if [ "${rank0_token_value}" != "" ]; then + echo "${rank0_token_var}=${rank0_token_value}" + fi + if [ "${rank1_token_value}" != "" ]; then + echo "${rank1_token_var}=${rank1_token_value}" + fi + done +} + +run_dry_run() +{ + export TILEXR_CCU_DIRECT_SMOKE_DRY_RUN=1 + echo "tilexr_ccu_direct_evening_smoke dryRun=1" + echo "TILEXR_CCU_SMOKE_DEVICES=${TILEXR_CCU_SMOKE_DEVICES}" + echo "TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE}" + echo "TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT}" + echo "TILEXR_CCU_DIRECT_SMOKE_SUBMIT=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" + echo "TILEXR_CCU_DIRECT_SMOKE_DRY_RUN=${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN}" + echo "TILEXR_CCU_SMOKE_DRY_RUN=${TILEXR_CCU_SMOKE_DRY_RUN:-0}" + echo "TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES=${TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES}" + echo "TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES=${TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES}" + echo "TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES=${TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES}" + echo "TILEXR_CCU_DIRECT_BARRIER_MODE=${TILEXR_CCU_DIRECT_BARRIER_MODE}" + echo "TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE=${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE}" + echo "TILEXR_CCU_PROBE_MISSION_START=${TILEXR_CCU_PROBE_MISSION_START}" + echo "TILEXR_CCU_PROBE_INSTRUCTION_START=${TILEXR_CCU_PROBE_INSTRUCTION_START}" + echo "TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START=${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START}" + echo "TILEXR_CCU_PROBE_SQE_ARG_COUNT=${TILEXR_CCU_PROBE_SQE_ARG_COUNT}" + echo "TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT=${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT}" + echo "TILEXR_CCU_PROBE_GSA_START=${TILEXR_CCU_PROBE_GSA_START}" + echo "TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE=${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE}" + echo "TILEXR_CCU_DIRECT_INSTALL_ORDER=${TILEXR_CCU_DIRECT_INSTALL_ORDER}" + print_resource_window_token_dry_run + print_prepare_profile_dry_run + bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +} + +if [ "${TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN:-0}" = "1" ] || + [ "${TILEXR_CCU_SMOKE_DRY_RUN:-0}" = "1" ]; then + set_evening_defaults + run_dry_run + exit 0 +fi + +if [ -f scripts/common_env.sh ]; then + # shellcheck source=/dev/null + source scripts/common_env.sh >/tmp/tilexr_env_evening_smoke.log 2>&1 +fi + +set_evening_defaults +evening_work_root="${TILEXR_CCU_EVENING_WORK_ROOT:-${repo_root}/build/ccu_direct_evening_smoke/$(date +%Y%m%d_%H%M%S)}" + +stage_enabled() +{ + case ",${TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES}," in + *",$1,"*) return 0 ;; + *) return 1 ;; + esac +} + +append_profile_pfe_env() +{ + if [ "${1:-}" = "" ] && [ "${2:-}" = "" ]; then + return + fi + printf '%s\n' \ + "TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE=$1" \ + "TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION=$2" +} + +prepare_has_submit_ready() +{ + local prepare_dir="$1" + local thread_log="${prepare_dir}/ccu_thread.log" + local rank0_log="${prepare_dir}/ccu_rank0.log" + local rank1_log="${prepare_dir}/ccu_rank1.log" + if [ -f "${thread_log}" ]; then + [ "$(grep -c "submitReady=1" "${thread_log}")" -ge 2 ] + return + fi + [ -f "${rank0_log}" ] && [ -f "${rank1_log}" ] && + grep -q "submitReady=1" "${rank0_log}" && + grep -q "submitReady=1" "${rank1_log}" +} + +summarize_stage_logs() +{ + local stage="$1" + local stage_dir="$2" + local status="$3" + local mode="${4:-}" + local found=0 + for log in \ + "${stage_dir}/ccu_thread.log" \ + "${stage_dir}/ccu_rank0.log" \ + "${stage_dir}/ccu_rank1.log"; do + if [ ! -f "${log}" ]; then + continue + fi + found=1 + if [ "${stage}" = "prepare" ]; then + echo "tilexr_ccu_direct_evening_smoke prepareLogSummary mode=${mode} status=${status} log=${log}" + else + echo "tilexr_ccu_direct_evening_smoke stageLogSummary stage=${stage} status=${status} log=${log}" + fi + grep -E 'direct CCU submit failed|rtRet=|args\[' "${log}" | + head -n "${TILEXR_CCU_EVENING_SUBMIT_FAILURE_SUMMARY_LINES:-8}" || true + grep -E \ + 'tilexr_ccu_direct_smoke config|tilexr_ccu_direct_smoke prepare|tilexr_ccu_direct_smoke preparedTasks|tilexr_ccu_direct_smoke submit|tilexr_ccu_direct_smoke submitTiming|tilexr_ccu_direct_smoke p2pCcuCopy|tilexr_ccu_direct_smoke aclrtSynchronizeStream|direct CCU submit failed|rtRet=|args\[|CCU custom channel call failed|op=[0-9]+|driverRet=|opRet=|SET_INSTRUCTION|SET_MSID_TOKEN|submitReady=|TileXRDirectCcuTrace .*decoded=|TileXRDirectCcuTrace remoteXnBinding|TileXRDirectCcuTrace task\[|TileXRDirectCcuTrace finalRuntimeTask|TileXRDirectCcuTrace customChannel.return|TileXRDirectCcuTrace program.sync' \ + "${log}" | tail -n "${TILEXR_CCU_EVENING_LOG_SUMMARY_LINES:-24}" || true + done + if [ "${found}" -eq 0 ]; then + if [ "${stage}" = "prepare" ]; then + echo "tilexr_ccu_direct_evening_smoke prepareLogSummary mode=${mode} status=${status} log=missing workDir=${stage_dir}" + else + echo "tilexr_ccu_direct_evening_smoke stageLogSummary stage=${stage} status=${status} log=missing workDir=${stage_dir}" + fi + fi +} + +extract_last_log_field() +{ + local key="$1" + shift + awk -v key="${key}" ' + { + for (i = 1; i <= NF; ++i) { + if ($i ~ ("^" key "=")) { + split($i, parts, "="); + value = parts[2]; + gsub(/[^0-9A-Za-z_.:-].*$/, "", value); + last = value; + } + } + } + END { + if (last != "") { + print last; + } + } + ' "$@" 2>/dev/null || true +} + +print_prepare_matrix_summary() +{ + local prepare_dir="$1" + local status="$2" + local profile="$3" + local alloc="$4" + local window="$5" + local data_len_mode="$6" + local install_order="$7" + local logs=() + local log + for log in \ + "${prepare_dir}/ccu_thread.log" \ + "${prepare_dir}/ccu_rank0.log" \ + "${prepare_dir}/ccu_rank1.log"; do + if [ -f "${log}" ]; then + logs+=("${log}") + fi + done + + local submit_ready="NA" + local op="NA" + local driver_ret="NA" + local op_ret="NA" + local lower_layer_preconditions=0 + local summary_log="missing" + if [ "${#logs[@]}" -gt 0 ]; then + summary_log="${logs[0]}" + submit_ready="$(extract_last_log_field submitReady "${logs[@]}")" + op="$(extract_last_log_field op "${logs[@]}")" + driver_ret="$(extract_last_log_field driverRet "${logs[@]}")" + op_ret="$(extract_last_log_field opRet "${logs[@]}")" + if grep -q 'lowerLayerPreconditions{' "${logs[@]}"; then + lower_layer_preconditions=1 + fi + fi + submit_ready="${submit_ready:-NA}" + op="${op:-NA}" + driver_ret="${driver_ret:-NA}" + op_ret="${op_ret:-NA}" + + echo "tilexr_ccu_direct_evening_smoke prepareMatrix profile=${profile} status=${status} submitReady=${submit_ready} op=${op} driverRet=${driver_ret} opRet=${op_ret} lowerLayerPreconditions=${lower_layer_preconditions} alloc=${alloc} window=${window} dataLenMode=${data_len_mode} installOrder=${install_order} log=${summary_log}" +} + +run_smoke_stage() +{ + local stage="$1" + shift + local stage_dir="${evening_work_root}/${stage}" + mkdir -p "${stage_dir}" + echo "tilexr_ccu_direct_evening_smoke stage=${stage} workDir=${stage_dir}" + timeout "${TILEXR_CCU_EVENING_TOTAL_TIMEOUT:-160}s" \ + env TILEXR_CCU_SMOKE_WORK_DIR="${stage_dir}" "$@" bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +} + +prepare_stage_exit_is_environmental() +{ + local status="$1" + [ "${status}" -eq 3 ] || [ "${status}" -eq 124 ] +} + +cmake --build build --target tile-comm -j"${TILEXR_CCU_EVENING_BUILD_JOBS:-2}" + +if ! stage_enabled prepare; then + echo "ERROR: TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES must include prepare" >&2 + exit 20 +fi + +build_prepare_profiles +selected_prepare_alloc_mode="" +selected_prepare_window="" +selected_prepare_data_len_mode="" +selected_prepare_install_order="" +selected_prepare_pfe_offset_source="" +selected_prepare_pfe_partition="" +selected_prepare_dir="" +selected_prepare_profile_name="" +prepare_status_summary="" + +print_prepare_failure_final_status() +{ + local final_status_line="tilexr_ccu_direct_evening_smoke finalStatus prepare=fail submit=skipped barrier=skipped p2p=skipped completionCandidate=0 failedStage=prepare selectedProfile=none selectedAlloc=none selectedWindow=none selectedDataLenMode=none selectedInstallOrder=none pfeOffsetSource=default pfePartition=default prepareStatusSummary=${prepare_status_summary}" + echo "${final_status_line}" + mkdir -p "${evening_work_root}" + printf '%s\n' "${final_status_line}" > "${evening_work_root}/final_status.log" +} + +for prepare_profile in "${prepare_profiles[@]}"; do + parse_prepare_profile "${prepare_profile}" + if [ "${prepare_profile_name}" = "" ]; then + continue + fi + safe_prepare_mode="$(printf '%s' "${prepare_profile_name}" | sed 's/[^A-Za-z0-9_]/_/g')" + prepare_status=0 + run_smoke_stage "prepare_${safe_prepare_mode}" \ + TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${prepare_profile_alloc}" \ + TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${prepare_profile_window}" \ + TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${prepare_profile_data_len_mode}" \ + TILEXR_CCU_DIRECT_INSTALL_ORDER="${prepare_profile_install_order}" \ + $(append_profile_pfe_env "${prepare_profile_pfe_offset_source}" "${prepare_profile_pfe_partition}") || + prepare_status=$? + prepare_dir="${evening_work_root}/prepare_${safe_prepare_mode}" + echo "tilexr_ccu_direct_evening_smoke prepareStatus=${prepare_status} profile=${prepare_profile_name} alloc=${prepare_profile_alloc} window=${prepare_profile_window} dataLenMode=${prepare_profile_data_len_mode} installOrder=${prepare_profile_install_order} pfeOffsetSource=${prepare_profile_pfe_offset_source:-default} pfePartition=${prepare_profile_pfe_partition:-default} workDir=${prepare_dir}" + print_prepare_matrix_summary \ + "${prepare_dir}" \ + "${prepare_status}" \ + "${prepare_profile_name}" \ + "${prepare_profile_alloc}" \ + "${prepare_profile_window}" \ + "${prepare_profile_data_len_mode}" \ + "${prepare_profile_install_order}" + summarize_stage_logs prepare "${prepare_dir}" "${prepare_status}" "${prepare_profile_name}" + prepare_status_summary="${prepare_status_summary}${prepare_status_summary:+,}${prepare_profile_name}:${prepare_status}:${prepare_profile_alloc}:${prepare_profile_window}:${prepare_profile_data_len_mode}:${prepare_profile_install_order}:${prepare_profile_pfe_offset_source:-default}:${prepare_profile_pfe_partition:-default}:${prepare_dir}" + if prepare_stage_exit_is_environmental "${prepare_status}"; then + echo "ERROR: direct CCU prepare stopped on environmental gate status=${prepare_status} profile=${prepare_profile_name} alloc=${prepare_profile_alloc} window=${prepare_profile_window} dataLenMode=${prepare_profile_data_len_mode} installOrder=${prepare_profile_install_order} pfeOffsetSource=${prepare_profile_pfe_offset_source:-default} pfePartition=${prepare_profile_pfe_partition:-default} workDir=${prepare_dir}" >&2 + exit "${prepare_status}" + fi + if [ "${prepare_status}" -eq 0 ] && prepare_has_submit_ready "${prepare_dir}"; then + selected_prepare_alloc_mode="${prepare_profile_alloc}" + selected_prepare_window="${prepare_profile_window}" + selected_prepare_data_len_mode="${prepare_profile_data_len_mode}" + selected_prepare_install_order="${prepare_profile_install_order}" + selected_prepare_pfe_offset_source="${prepare_profile_pfe_offset_source}" + selected_prepare_pfe_partition="${prepare_profile_pfe_partition}" + selected_prepare_dir="${prepare_dir}" + selected_prepare_profile_name="${prepare_profile_name}" + break + fi +done + +if [ "${selected_prepare_alloc_mode}" = "" ]; then + print_prepare_failure_final_status + echo 'tilexr_ccu_direct_evening_smoke stopAfter=prepare reason="submitReady=1 missing for every prepare profile" workRoot='"${evening_work_root} prepareStatusSummary=${prepare_status_summary}" + exit 0 +fi +echo "tilexr_ccu_direct_evening_smoke selectedPrepare alloc=${selected_prepare_alloc_mode} window=${selected_prepare_window} dataLenMode=${selected_prepare_data_len_mode} installOrder=${selected_prepare_install_order} pfeOffsetSource=${selected_prepare_pfe_offset_source:-default} pfePartition=${selected_prepare_pfe_partition:-default} workDir=${selected_prepare_dir}" + +submit_final_status="skipped" +barrier_final_status="skipped" +p2p_final_status="skipped" + +print_final_status() +{ + local failed_stage="${1:-none}" + local completion_candidate=0 + if [ "${failed_stage}" = "none" ] && + [ "${submit_final_status}" = "pass" ] && + [ "${barrier_final_status}" = "pass" ] && + [ "${p2p_final_status}" = "pass" ]; then + completion_candidate=1 + fi + + local final_status_line="tilexr_ccu_direct_evening_smoke finalStatus prepare=pass submit=${submit_final_status} barrier=${barrier_final_status} p2p=${p2p_final_status} completionCandidate=${completion_candidate} failedStage=${failed_stage} selectedProfile=${selected_prepare_profile_name} selectedAlloc=${selected_prepare_alloc_mode} selectedWindow=${selected_prepare_window} selectedDataLenMode=${selected_prepare_data_len_mode} selectedInstallOrder=${selected_prepare_install_order} pfeOffsetSource=${selected_prepare_pfe_offset_source:-default} pfePartition=${selected_prepare_pfe_partition:-default}" + echo "${final_status_line}" + mkdir -p "${evening_work_root}" + printf '%s\n' "${final_status_line}" > "${evening_work_root}/final_status.log" +} + +if stage_enabled submit; then + submit_status=0 + run_smoke_stage submit \ + TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}" \ + TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${selected_prepare_window}" \ + TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${selected_prepare_data_len_mode}" \ + TILEXR_CCU_DIRECT_INSTALL_ORDER="${selected_prepare_install_order}" \ + $(append_profile_pfe_env "${selected_prepare_pfe_offset_source}" "${selected_prepare_pfe_partition}") \ + TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 || + submit_status=$? + summarize_stage_logs submit "${evening_work_root}/submit" "${submit_status}" + if [ "${submit_status}" -ne 0 ]; then + submit_final_status="fail" + print_final_status submit + exit "${submit_status}" + fi + submit_final_status="pass" +fi + +if stage_enabled barrier; then + barrier_status=0 + run_smoke_stage barrier \ + TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}" \ + TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${selected_prepare_window}" \ + TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${selected_prepare_data_len_mode}" \ + TILEXR_CCU_DIRECT_INSTALL_ORDER="${selected_prepare_install_order}" \ + $(append_profile_pfe_env "${selected_prepare_pfe_offset_source}" "${selected_prepare_pfe_partition}") \ + TILEXR_CCU_DIRECT_BARRIER_MODE="${TILEXR_CCU_DIRECT_BARRIER_MODE}" \ + TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ + TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK="${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK:-0}" \ + TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS="${TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS:-300}" \ + TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT=1 \ + TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS="${TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS:-100}" || + barrier_status=$? + summarize_stage_logs barrier "${evening_work_root}/barrier" "${barrier_status}" + if [ "${barrier_status}" -ne 0 ]; then + barrier_final_status="fail" + print_final_status barrier + exit "${barrier_status}" + fi + barrier_final_status="pass" +fi + +if stage_enabled p2p; then + p2p_status=0 + run_smoke_stage p2p \ + TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}" \ + TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${selected_prepare_window}" \ + TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${selected_prepare_data_len_mode}" \ + TILEXR_CCU_DIRECT_INSTALL_ORDER="${selected_prepare_install_order}" \ + $(append_profile_pfe_env "${selected_prepare_pfe_offset_source}" "${selected_prepare_pfe_partition}") \ + TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=1 \ + TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ + TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1 \ + TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY=1 || + p2p_status=$? + summarize_stage_logs p2p "${evening_work_root}/p2p" "${p2p_status}" + if [ "${p2p_status}" -ne 0 ]; then + p2p_final_status="fail" + print_final_status p2p + exit "${p2p_status}" + fi + p2p_final_status="pass" +fi + +print_final_status none +echo "tilexr_ccu_direct_evening_smoke success workRoot=${evening_work_root}" diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh new file mode 100644 index 00000000..d48e1dbd --- /dev/null +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -0,0 +1,609 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026 TileXR Project +# +# Two-rank runner for the private TileXR direct CCU smoke probe. +# Default execution is safe and does not touch ACL/NPU runtime. + +set -euo pipefail + +if [ "${TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE:-0}" != "1" ]; then + echo "tilexr_ccu_direct_smoke_runner skipped: set TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 to run hardware smoke" + exit 0 +fi + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "${repo_root}" + +ASCEND_HOME_PATH="${ASCEND_HOME_PATH:-/usr/local/Ascend/ascend-toolkit/latest}" +ARCH="${ARCH:-$(uname -m)}" +if [ "${ARCH}" = "arm64" ]; then + ARCH="aarch64" +fi +ASCEND_DRIVER_PATH="${ASCEND_DRIVER_PATH:-/usr/local/Ascend/driver}" +export ASCEND_HOME_PATH ARCH ASCEND_DRIVER_PATH +export TILEXR_CCU_DIRECT_INSTALL_ORDER="${TILEXR_CCU_DIRECT_INSTALL_ORDER:-lower_layer_first}" + +work_dir="${TILEXR_CCU_SMOKE_WORK_DIR:-${repo_root}/build/ccu_direct_smoke}" +mkdir -p "${work_dir}" + +endpoint_fields=( + EID + TPN + DOORBELL_VA + DOORBELL_TOKEN_ID + DOORBELL_TOKEN_VALUE + SQ_DEPTH +) + +resource_window_token_fields=( + TOKEN_ID + RAW_TOKEN_ID + TOKEN_VALUE +) + +parse_int() +{ + local value="$1" + local fallback="$2" + if [ -z "${value}" ]; then + echo "${fallback}" + return + fi + printf "%d" "${value}" 2>/dev/null || printf "%d" "${fallback}" +} + +default_sync_instruction_count() +{ + local sync_resource_count="$1" + local barrier_mode="${TILEXR_CCU_DIRECT_BARRIER_MODE:-}" + local hcomm_style_task1_prelude_count=5 + case "${barrier_mode}" in + sync_cke|sync_cke_set_wait) + echo $((sync_resource_count * 2 + 1)) + ;; + sync_cke_post_only) + echo $((sync_resource_count + 1)) + ;; + local_cke_post_only) + echo "${sync_resource_count}" + ;; + sync_xn_post_only) + echo $((hcomm_style_task1_prelude_count + sync_resource_count)) + ;; + sync_xn_load_post_only) + echo $((hcomm_style_task1_prelude_count + sync_resource_count * 2)) + ;; + *) + echo $((hcomm_style_task1_prelude_count + sync_resource_count * 2)) + ;; + esac +} + +if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then + echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" + for diagnostic_var in \ + TILEXR_CCU_DIRECT_BARRIER_MODE \ + TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE \ + TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW \ + TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE \ + TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE \ + TILEXR_CCU_DIRECT_INSTALL_ORDER \ + TILEXR_CCU_PROBE_SQE_ARG_COUNT \ + TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do + diagnostic_value="${!diagnostic_var:-}" + if [ "${diagnostic_value}" != "" ]; then + echo "dryRun ${diagnostic_var}=${diagnostic_value}" + fi + done + sqe_arg_count="$(parse_int "${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" 13)" + sync_resource_count="$(parse_int "${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" 1)" + default_sync_instruction_count_value="$(default_sync_instruction_count "${sync_resource_count}")" + sync_instruction_count="$(parse_int \ + "${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-${default_sync_instruction_count_value}}" \ + "${default_sync_instruction_count_value}")" + repository_start="$(parse_int "${TILEXR_CCU_PROBE_INSTRUCTION_START:-1}" 1)" + mission_instruction_start="$(parse_int "${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-0}" 0)" + if [ "${mission_instruction_start}" -eq 0 ]; then + mission_instruction_start="${repository_start}" + fi + repository_prefix_count=$((mission_instruction_start - repository_start)) + if [ "${repository_prefix_count}" -lt 0 ]; then + repository_prefix_count=0 + fi + mission_instruction_count=$((sqe_arg_count + sync_instruction_count)) + repository_count=$((repository_prefix_count + mission_instruction_count)) + task0_start="${mission_instruction_start}" + task0_count="${sqe_arg_count}" + task1_start=$((mission_instruction_start + sqe_arg_count)) + task1_count="${sync_instruction_count}" + if [ "${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-}" = "full_repository" ] || + [ "${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-}" = "full" ] || + [ "${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-}" = "1" ]; then + install_start="${repository_start}" + install_count="${repository_count}" + else + install_start="${mission_instruction_start}" + install_count="${mission_instruction_count}" + fi + instruction_data_len=$((install_count * 32)) + if [ "${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-}" = "descriptor_bytes" ] || + [ "${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-}" = "descriptor" ] || + [ "${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-}" = "1" ]; then + set_instruction_data_len=24 + else + set_instruction_data_len="${instruction_data_len}" + fi + echo "dryRun derived repositoryStartId=${repository_start} repositoryCount=${repository_count} missionInstructionStartId=${mission_instruction_start} missionInstructionCount=${mission_instruction_count}" + echo "dryRun derived task0.instStartId=${task0_start} task0.instCnt=${task0_count}" + echo "dryRun derived task1.instStartId=${task1_start} task1.instCnt=${task1_count}" + echo "dryRun derived SET_INSTRUCTION offsetStartIdx=${install_start} dataLen=${set_instruction_data_len} instructionBytes=${instruction_data_len}" + for endpoint_field in "${endpoint_fields[@]}"; do + endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" + common_endpoint_value="${!endpoint_var:-}" + rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" + rank0_endpoint_value="${!rank0_endpoint_var:-${common_endpoint_value}}" + rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" + rank1_endpoint_value="${!rank1_endpoint_var:-${common_endpoint_value}}" + if [ "${rank0_endpoint_value}" != "" ]; then + echo "dryRun rank0 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}" + fi + if [ "${rank1_endpoint_value}" != "" ]; then + echo "dryRun rank1 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}" + fi + done + for token_field in "${resource_window_token_fields[@]}"; do + token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}" + common_token_value="${!token_var:-}" + rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" + rank0_token_value="${!rank0_token_var:-${common_token_value}}" + rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" + rank1_token_value="${!rank1_token_var:-${common_token_value}}" + if [ "${rank0_token_value}" != "" ]; then + echo "dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}" + fi + if [ "${rank1_token_value}" != "" ]; then + echo "dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}" + fi + done + exit 0 +fi + +tile_comm_lib="${TILEXR_TILE_COMM_LIB:-}" +if [ -z "${tile_comm_lib}" ]; then + for candidate in \ + "${repo_root}/build/src/comm/libtile-comm.so" \ + "${repo_root}/install/lib64/libtile-comm.so" \ + "${repo_root}/install/lib/libtile-comm.so" \ + "${repo_root}/install_direct_ccu_guard/lib64/libtile-comm.so"; do + if [ -f "${candidate}" ]; then + tile_comm_lib="${candidate}" + break + fi + done +fi +if [ -z "${tile_comm_lib}" ] || [ ! -f "${tile_comm_lib}" ]; then + echo "ERROR: libtile-comm.so not found; build tile-comm first or set TILEXR_TILE_COMM_LIB" >&2 + exit 2 +fi +tile_comm_dir="$(cd "$(dirname "${tile_comm_lib}")" && pwd)" + +cann_root="${ASCEND_HOME_PATH}/${ARCH}-linux" +cann_lib_dir="${cann_root}/lib64" +driver_lib_dir="${ASCEND_DRIVER_PATH}/lib64/driver" +probe_bin="${work_dir}/ccu_tilexr_direct_smoke_probe" + +c++ -std=c++14 \ + -I "${repo_root}/src/include" \ + -I "${repo_root}/src/comm" \ + -I "${cann_root}/pkg_inc" \ + -I "${cann_root}/pkg_inc/runtime" \ + -I "${cann_root}/include" \ + "${repo_root}/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp" \ + -L "${tile_comm_dir}" \ + -L "${cann_lib_dir}" \ + -L "${driver_lib_dir}" \ + -Wl,-rpath-link,"${tile_comm_dir}" \ + -Wl,-rpath-link,"${cann_lib_dir}" \ + -Wl,-rpath-link,"${driver_lib_dir}" \ + -ltile-comm -lascendcl -lruntime -ldl -pthread \ + -o "${probe_bin}" + +devices="${TILEXR_CCU_SMOKE_DEVICES:-${TILEXR_TEST_DEVICES:-0,1}}" + +if command -v npu-smi >/dev/null 2>&1; then + npu_smi_rc=0 + timeout "${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}s" npu-smi info > "${work_dir}/npu-smi.log" 2>&1 || npu_smi_rc=$? + if [ "${npu_smi_rc}" -ne 0 ]; then + echo "ERROR: npu-smi info did not complete; refusing to run ACL/CCU smoke" >&2 + echo "npu-smi rc=${npu_smi_rc}" >&2 + echo "npu-smi log: ${work_dir}/npu-smi.log" >&2 + exit 3 + fi + if [ "${TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU:-0}" != "1" ]; then + busy_rc=0 + npu_guard_args=( + --log "${work_dir}/npu-smi.log" + --devices "${devices}" + ) + if [ "${TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU:-0}" = "1" ]; then + npu_guard_args+=(--allow-unhealthy) + fi + python3 "${repo_root}/tests/ccu/ccu_npu_smi_busy_guard.py" \ + "${npu_guard_args[@]}" > "${work_dir}/npu-smi-busy.log" 2>&1 || busy_rc=$? + if [ "${busy_rc}" -ne 0 ]; then + echo "ERROR: selected NPU device is busy or unhealthy; refusing to run ACL/CCU smoke" >&2 + cat "${work_dir}/npu-smi-busy.log" >&2 + echo "npu-smi log: ${work_dir}/npu-smi.log" >&2 + echo "set TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU=1 to allow Alarm health while still rejecting busy devices" >&2 + echo "set TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU=1 only for an explicitly approved short test that may use busy devices" >&2 + exit 3 + fi + fi +elif [ "${TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI:-0}" = "1" ]; then + echo "ERROR: npu-smi not found; refusing to run ACL/CCU smoke" >&2 + exit 3 +fi + +comm_port="${TILEXR_CCU_SMOKE_PORT:-$((30000 + (RANDOM % 20000)))}" +comm_id="${TILEXR_COMM_ID:-127.0.0.1:${comm_port}}" +comm_domain="${TILEXR_CCU_PROBE_COMM_DOMAIN:-0}" +timeout_s="${TILEXR_CCU_SMOKE_TIMEOUT:-180}" +ready_dir="${work_dir}/submit_ready_${comm_port}" +done_dir="${work_dir}/submit_done_${comm_port}" +rank0_log="${work_dir}/ccu_rank0.log" +rank1_log="${work_dir}/ccu_rank1.log" +rm -rf "${ready_dir}" "${done_dir}" +mkdir -p "${ready_dir}" "${done_dir}" +rm -f "${rank0_log}" "${rank1_log}" + +common_env=( + "LD_LIBRARY_PATH=${tile_comm_dir}:${cann_lib_dir}:${driver_lib_dir}:${LD_LIBRARY_PATH:-}" + "TILEXR_COMM_ID=${comm_id}" + "TILEXR_TEST_DEVICES=${devices}" + "TILEXR_CCU_DIRECT_SMOKE_ENABLE=1" + "TILEXR_CCU_DIRECT_SMOKE_READY_DIR=${ready_dir}" + "TILEXR_CCU_DIRECT_SMOKE_DONE_DIR=${done_dir}" + "TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE:-1}" + "TILEXR_CCU_PROBE_RANK_SIZE=2" + "TILEXR_CCU_PROBE_COMM_DOMAIN=${comm_domain}" +) +if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-0}" = "1" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=1") +fi +if [ "${TILEXR_CCU_DIRECT_BARRIER_MODE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_BARRIER_MODE=${TILEXR_CCU_DIRECT_BARRIER_MODE}") +fi +if [ "${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW=${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW}") +fi +if [ "${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE=${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE}") +fi +if [ "${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE=${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE}") +fi +if [ "${TILEXR_CCU_DIRECT_INSTALL_ORDER:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_INSTALL_ORDER=${TILEXR_CCU_DIRECT_INSTALL_ORDER}") +fi +if [ "${TILEXR_CCU_DIRECT_INSTALL_DIE_ID:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_INSTALL_DIE_ID=${TILEXR_CCU_DIRECT_INSTALL_DIE_ID}") +fi +if [ "${TILEXR_CCU_PROBE_MISSION_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_MISSION_START=${TILEXR_CCU_PROBE_MISSION_START}") +fi +if [ "${TILEXR_CCU_PROBE_INSTRUCTION_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_INSTRUCTION_START=${TILEXR_CCU_PROBE_INSTRUCTION_START}") +fi +if [ "${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START=${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START}") +fi +if [ "${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_SQE_ARG_COUNT=${TILEXR_CCU_PROBE_SQE_ARG_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT=${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT=${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_BINDINGS_PER_RESOURCE:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_BINDINGS_PER_RESOURCE=${TILEXR_CCU_PROBE_BINDINGS_PER_RESOURCE}") +fi +if [ "${TILEXR_CCU_PROBE_CKE_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_CKE_START=${TILEXR_CCU_PROBE_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_GSA_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_GSA_START=${TILEXR_CCU_PROBE_GSA_START}") +fi +if [ "${TILEXR_CCU_PROBE_CHANNEL_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_CHANNEL_START=${TILEXR_CCU_PROBE_CHANNEL_START}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK=${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS=${TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES}") +fi +if [ "${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT}") +fi +for endpoint_field in "${endpoint_fields[@]}"; do + endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" + endpoint_value="${!endpoint_var:-}" + if [ "${endpoint_value}" != "" ]; then + common_env+=("${endpoint_var}=${endpoint_value}") + fi +done +for token_field in "${resource_window_token_fields[@]}"; do + token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}" + token_value="${!token_var:-}" + if [ "${token_value}" != "" ]; then + common_env+=("${token_var}=${token_value}") + fi + rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" + rank0_token_value="${!rank0_token_var:-}" + if [ "${rank0_token_value}" != "" ]; then + common_env+=("${rank0_token_var}=${rank0_token_value}") + fi + rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" + rank1_token_value="${!rank1_token_var:-}" + if [ "${rank1_token_value}" != "" ]; then + common_env+=("${rank1_token_var}=${rank1_token_value}") + fi +done + +rank0_env=() +rank1_env=() +if [ "${TILEXR_CCU_PROBE_RANK0_XN_START:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK0_XN_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_XN_START:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK1_XN_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT}") +elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT}") +elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START}") +fi +if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then + rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT}") +fi +if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then + rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT}") +fi +for endpoint_field in "${endpoint_fields[@]}"; do + rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" + rank0_endpoint_value="${!rank0_endpoint_var:-}" + if [ "${rank0_endpoint_value}" != "" ]; then + rank0_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}") + fi + rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" + rank1_endpoint_value="${!rank1_endpoint_var:-}" + if [ "${rank1_endpoint_value}" != "" ]; then + rank1_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}") + fi +done +for token_field in "${resource_window_token_fields[@]}"; do + rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" + rank0_token_value="${!rank0_token_var:-}" + if [ "${rank0_token_value}" != "" ]; then + rank0_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}") + fi + rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" + rank1_token_value="${!rank1_token_var:-}" + if [ "${rank1_token_value}" != "" ]; then + rank1_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}") + fi +done + +echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" + +if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then + thread_log="${work_dir}/ccu_thread.log" + rm -f "${thread_log}" + thread_status=0 + timeout "${timeout_s}s" env "${common_env[@]}" TILEXR_CCU_PROBE_RANK=0 "${probe_bin}" > "${thread_log}" 2>&1 || + thread_status=$? + cat "${thread_log}" + echo "tilexr_ccu_direct_smoke_runner threadMode summary status=${thread_status} log=${thread_log}" + if [ "${thread_status}" -ne 0 ]; then + echo "ERROR: direct CCU thread-mode smoke failed status=${thread_status}" >&2 + echo "thread log: ${thread_log}" >&2 + exit 4 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke prepare ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode prepare did not return success for both ranks" >&2 + exit 5 + fi + if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode prepare did not complete install attempt for both ranks" >&2 + exit 6 + fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit requested but prepare did not reach submitReady=1" >&2 + exit 6 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke submit ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit did not return success for both ranks" >&2 + exit 7 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke submitTiming" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit timing was not reported for both ranks" >&2 + exit 8 + fi + fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then + echo "ERROR: direct CCU thread-mode P2P CCU-copy check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1" >&2 + exit 11 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke p2pCcuCopy" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode P2P CCU-copy result missing" >&2 + exit 12 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke p2pCcuCopy .*passed=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode P2P CCU-copy check failed" >&2 + exit 13 + fi + fi + echo "tilexr_ccu_direct_smoke_runner success workDir=${work_dir}" + exit 0 +fi + +timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0 "${probe_bin}" > "${rank0_log}" 2>&1 & +rank0_pid=$! +sleep "${TILEXR_CCU_SMOKE_RANK1_DELAY:-1}" +timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1 "${probe_bin}" > "${rank1_log}" 2>&1 & +rank1_pid=$! + +rank0_status=0 +rank1_status=0 +wait "${rank0_pid}" || rank0_status=$? +wait "${rank1_pid}" || rank1_status=$? + +cat "${rank0_log}" +cat "${rank1_log}" + +echo "tilexr_ccu_direct_smoke_runner summary rank0Status=${rank0_status} rank1Status=${rank1_status} rank0Log=${rank0_log} rank1Log=${rank1_log}" + +if [ "${rank0_status}" -ne 0 ] || [ "${rank1_status}" -ne 0 ]; then + echo "ERROR: direct CCU smoke rank process failed rank0=${rank0_status} rank1=${rank1_status}" >&2 + echo "rank0 log: ${rank0_log}" >&2 + echo "rank1 log: ${rank1_log}" >&2 + exit 4 +fi + +for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_direct_smoke prepare ret=0" "${log}"; then + echo "ERROR: direct CCU prepare did not return success in ${log}" >&2 + exit 5 + fi + if ! grep -q "installSucceeded=1" "${log}"; then + echo "ERROR: direct CCU prepare did not complete install attempt in ${log}" >&2 + exit 6 + fi +done + +if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "submitReady=1" "${log}"; then + echo "ERROR: direct CCU submit requested but prepare did not reach submitReady=1 in ${log}" >&2 + exit 6 + fi + done + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_direct_smoke submit ret=0" "${log}"; then + echo "ERROR: direct CCU submit did not return success in ${log}" >&2 + exit 7 + fi + if ! grep -q "tilexr_ccu_direct_smoke submitTiming" "${log}"; then + echo "ERROR: direct CCU submit timing was not reported in ${log}" >&2 + exit 8 + fi + done +fi + +if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]; then + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then + echo "ERROR: direct CCU barrier wait check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1" >&2 + exit 9 + fi + delay_rank="${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK:-0}" + min_sync_ms="${TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS:-100}" + if [ "${delay_rank}" = "0" ]; then + wait_log="${rank1_log}" + else + wait_log="${rank0_log}" + fi + wait_sync_ms="$( + awk ' + /tilexr_ccu_direct_smoke submitTiming/ { + for (i = 1; i <= NF; ++i) { + if ($i ~ /^syncMs=/) { + split($i, parts, "="); + print parts[2]; + } + } + } + ' "${wait_log}" | tail -n 1 + )" + if [ -z "${wait_sync_ms}" ]; then + echo "ERROR: barrier wait timing missing from ${wait_log}" >&2 + exit 9 + fi + if [ "${wait_sync_ms}" -lt "${min_sync_ms}" ]; then + echo "ERROR: direct CCU barrier wait was too short syncMs=${wait_sync_ms} minSyncMs=${min_sync_ms} log=${wait_log}" >&2 + exit 10 + fi +fi + +if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then + echo "ERROR: direct CCU P2P CCU-copy check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1" >&2 + exit 11 + fi + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy" "${log}"; then + echo "ERROR: direct CCU P2P CCU-copy result missing in ${log}" >&2 + exit 12 + fi + if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy .*passed=1" "${log}"; then + echo "ERROR: direct CCU P2P CCU-copy check failed in ${log}" >&2 + exit 13 + fi + done +fi + +echo "tilexr_ccu_direct_smoke_runner success workDir=${work_dir}" diff --git a/tests/ccu/test_tilexr_ccu_barrier_program.py b/tests/ccu/test_tilexr_ccu_barrier_program.py new file mode 100644 index 00000000..254a1b82 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_barrier_program.py @@ -0,0 +1,649 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +BARRIER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.h" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +PRODUCER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + #include + + using namespace TileXR; + + int main() + { + std::vector specs; + specs.push_back({2361, 1961, 2, 364, 1, 332, 1}); + specs.push_back({2362, 1962, 3, 365, 1, 333, 1}); + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram(specs, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "barrier build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 4 || report.postInstructionCount != 2 || + report.waitInstructionCount != 2 || report.totalInstructionCount != 4 || + report.message != "ok") { + std::cerr << "unexpected barrier report\n"; + return 2; + } + if (program[0].words[0] != 0x000007a90939100dULL || + program[0].words[1] != 0x00000001016c0002ULL || + program[0].words[2] != 0x0001000000000000ULL || + program[0].words[3] != 0) { + std::cerr << "unexpected first post instruction\n"; + return 3; + } + if (program[1].words[0] != 0x000007aa093a100dULL || + program[1].words[1] != 0x00000001016d0003ULL || + program[1].words[2] != 0x0001000000000000ULL || + program[1].words[3] != 0) { + std::cerr << "unexpected second post instruction\n"; + return 4; + } + if (program[2].words[0] != 0x0000000000010802ULL || + program[2].words[1] != 0x000000000001014cULL || + program[2].words[2] != 0 || + program[2].words[3] != 0) { + std::cerr << "unexpected first wait/clear instruction\n"; + return 5; + } + if (program[3].words[0] != 0x0000000000010802ULL || + program[3].words[1] != 0x000000000001014dULL || + program[3].words[2] != 0 || + program[3].words[3] != 0) { + std::cerr << "unexpected second wait/clear instruction\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_rejects_incomplete_resource_bindings(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + std::vector program(1); + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram({}, &program, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty barrier specs accepted\n"; + return 1; + } + if (!program.empty() || report.message.find("missing") == std::string::npos) { + std::cerr << "empty barrier diagnostic/report mismatch: " << report.message << "\n"; + return 2; + } + + std::vector specs; + specs.push_back({2361, 1961, 2, 364, 1, 0, 1}); + if (TileXRCcuBuildBarrierProgram(specs, &program, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing local wait CKE accepted\n"; + return 3; + } + if (!program.empty() || report.message.find("local wait CKE") == std::string::npos) { + std::cerr << "weak missing local wait diagnostic: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_emit_hcomm_like_synccke_post_and_clear_wait(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.channelId = 2; + spec.remoteNotifyCke = 0x330; + spec.remoteNotifyMask = 1; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + spec.sourceCke = 0x221; + spec.sourceCkeMask = 0xffff; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::SyncCke) != TILEXR_SUCCESS) { + std::cerr << "sync_cke barrier build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 3 || report.postInstructionCount != 1 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 3) { + std::cerr << "unexpected sync_cke barrier report\n"; + return 2; + } + if (program[0].words[0] != 0xffff022100010802ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "source CKE init mismatch\n"; + return 3; + } + if (program[1].words[0] != 0x000102210330100bULL || + program[1].words[1] != 0x0000000000000002ULL || + program[1].words[2] != 0x0001000000000000ULL || + program[1].words[3] != 0) { + std::cerr << "SyncCKE post mismatch\n"; + return 4; + } + if (program[2].words[0] != 0x0000000000010804ULL || + program[2].words[1] != 0x0000000000010220ULL || + program[2].words[2] != 0 || + program[2].words[3] != 0) { + std::cerr << "ClearCKE wait mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_emit_synccke_post_and_set_wait(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.channelId = 2; + spec.remoteNotifyCke = 0x330; + spec.remoteNotifyMask = 1; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + spec.sourceCke = 0x221; + spec.sourceCkeMask = 0xffff; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::SyncCkeSetWait) != TILEXR_SUCCESS) { + std::cerr << "sync_cke_set_wait barrier build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 3 || report.postInstructionCount != 1 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 3) { + std::cerr << "unexpected sync_cke_set_wait barrier report\n"; + return 2; + } + if (program[0].words[0] != 0xffff022100010802ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "source CKE init mismatch\n"; + return 3; + } + if (program[1].words[0] != 0x000102210330100bULL || + program[1].words[1] != 0x0000000000000002ULL || + program[1].words[2] != 0x0001000000000000ULL || + program[1].words[3] != 0) { + std::cerr << "SyncCKE post mismatch\n"; + return 4; + } + if (program[2].words[0] != 0x0000000000010802ULL || + program[2].words[1] != 0x0000000000010220ULL || + program[2].words[2] != 0 || + program[2].words[3] != 0) { + std::cerr << "SetCKE wait mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_emit_synccke_post_only_diagnostic(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.channelId = 2; + spec.remoteNotifyCke = 0x330; + spec.remoteNotifyMask = 1; + spec.sourceCke = 0x221; + spec.sourceCkeMask = 0xffff; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::SyncCkePostOnly) != TILEXR_SUCCESS) { + std::cerr << "sync_cke_post_only barrier build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 2 || report.postInstructionCount != 1 || + report.waitInstructionCount != 0 || report.totalInstructionCount != 2) { + std::cerr << "unexpected sync_cke_post_only barrier report\n"; + return 2; + } + if (program[0].words[0] != 0xffff022100010802ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "source CKE init mismatch\n"; + return 3; + } + if (program[1].words[0] != 0x000102210330100bULL || + program[1].words[1] != 0x0000000000000002ULL || + program[1].words[2] != 0x0001000000000000ULL || + program[1].words[3] != 0) { + std::cerr << "SyncCKE post-only mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_emit_local_cke_completion_diagnostic(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::LocalCke) != TILEXR_SUCCESS) { + std::cerr << "local CKE diagnostic build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 2 || report.postInstructionCount != 1 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 2) { + std::cerr << "unexpected local CKE diagnostic report\n"; + return 2; + } + if (program[0].words[0] != 0x0001022000000802ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "local CKE set mismatch\n"; + return 3; + } + if (program[1].words[0] != 0x0000000000010804ULL || + program[1].words[1] != 0x0000000000010220ULL || + program[1].words[2] != 0 || + program[1].words[3] != 0) { + std::cerr << "local CKE clear/wait mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_emit_local_cke_post_only_diagnostic(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::LocalCkePostOnly) != TILEXR_SUCCESS) { + std::cerr << "local CKE post-only diagnostic build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 1 || report.postInstructionCount != 1 || + report.waitInstructionCount != 0 || report.totalInstructionCount != 1) { + std::cerr << "unexpected local CKE post-only diagnostic report\n"; + return 2; + } + if (program[0].words[0] != 0x0001022000000802ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "local CKE post-only set mismatch\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_can_load_local_xn_before_sync_xn_post_only_diagnostic(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_barrier_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBarrierSyncSpec spec; + spec.remoteXn = 0x240; + spec.localXn = 0x120; + spec.channelId = 2; + spec.remoteNotifyCke = 0x330; + spec.remoteNotifyMask = 1; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildBarrierProgram( + std::vector{spec}, + &program, + &report, + TileXRCcuBarrierMode::SyncXnLoadPostOnly) != TILEXR_SUCCESS) { + std::cerr << "sync_xn_load_post_only diagnostic build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 2 || report.postInstructionCount != 1 || + report.waitInstructionCount != 0 || report.totalInstructionCount != 2) { + std::cerr << "unexpected sync_xn_load_post_only report\n"; + return 2; + } + if (program[0].words[0] != 0x0000000101200003ULL || + program[0].words[1] != 0 || + program[0].words[2] != 0 || + program[0].words[3] != 0) { + std::cerr << "local XN load immediate mismatch\n"; + return 3; + } + if (program[1].words[0] != 0x000001200240100dULL || + program[1].words[1] != 0x0000000103300002ULL || + program[1].words[2] != 0x0001000000000000ULL || + program[1].words[3] != 0) { + std::cerr << "SyncXn post mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_producer_microcode_uses_barrier_post_and_wait_program(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_producer_plan.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 3}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 332, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 333, 3, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuProgram program; + TileXRCcuProducerPlanReport report; + if (TileXRCcuBuildMicrocode(plan, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << report.message << "\n"; + return 1; + } + if (program.sqeLoad.size() != 13 || program.sync.size() != 9) { + std::cerr << "unexpected producer barrier microcode size\n"; + return 2; + } + if (program.sync[5].words[0] != 0x000007a90939100dULL || + program.sync[5].words[1] != 0x00000001014c0002ULL || + program.sync[7].words[0] != 0x0000000000010802ULL || + program.sync[7].words[1] != 0x000000000001014cULL) { + std::cerr << "unexpected first producer barrier pair\n"; + return 3; + } + if (program.sync[6].words[0] != 0x000007aa093a100dULL || + program.sync[6].words[1] != 0x00000001014d0003ULL || + program.sync[8].words[0] != 0x0000000000010802ULL || + program.sync[8].words[1] != 0x000000000001014dULL) { + std::cerr << "unexpected second producer barrier pair\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_barrier_program_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = BARRIER_HEADER.read_text(encoding="utf-8") + source = BARRIER_SOURCE.read_text(encoding="utf-8") + producer = PRODUCER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_barrier_program.h", cmake) + self.assertIn("ccu/tilexr_ccu_barrier_program.cpp", cmake) + self.assertIn("struct TileXRCcuBarrierSyncSpec", header) + self.assertIn("enum class TileXRCcuBarrierMode", header) + self.assertIn("LocalCke", header) + self.assertIn("LocalCkePostOnly", header) + self.assertIn("SyncXnPostOnly", header) + self.assertIn("SyncXnLoadPostOnly", header) + self.assertIn("SyncCkePostOnly", header) + self.assertIn("SyncCkeSetWait", header) + self.assertIn("struct TileXRCcuBarrierProgramReport", header) + self.assertIn("TileXRCcuBuildBarrierProgram", header) + self.assertIn("TileXRCcuBarrierMode::LocalCke", source) + self.assertIn("TileXRCcuBarrierMode::LocalCkePostOnly", source) + self.assertIn("TileXRCcuBarrierMode::SyncXnPostOnly", source) + self.assertIn("TileXRCcuBarrierMode::SyncXnLoadPostOnly", source) + self.assertIn("TileXRCcuBarrierMode::SyncCkePostOnly", source) + self.assertIn("TileXRCcuBarrierMode::SyncCkeSetWait", source) + self.assertIn("TileXRCcuEncodeLoadImdToXn", source) + self.assertIn("TileXRCcuEncodeSyncXn", source) + self.assertIn("TileXRCcuEncodeSyncCke", source) + self.assertIn("TileXRCcuEncodeSetCke", source) + self.assertIn("TileXRCcuEncodeClearCke", source) + self.assertIn("TileXRCcuBuildBarrierProgram", producer) + + combined = header + "\n" + source + "\n" + producer + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_basic_info_probe.py b/tests/ccu/test_tilexr_ccu_basic_info_probe.py new file mode 100644 index 00000000..654efc78 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_basic_info_probe.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import os +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROBE_SOURCE = REPO_ROOT / "tests" / "ccu" / "ccu_tilexr_basic_info_probe.cpp" +COMM_DIR = REPO_ROOT / "src" / "comm" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +CCU_DRIVER_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_driver_adapter.cpp" +CCU_DIRECT_RUNTIME_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_direct_runtime.cpp" +CCU_HCCP_LOADER_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_hccp_loader.cpp" +CCU_RA_PROVIDER_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp" +CCU_SPECS_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_specs.cpp" + + +class TileXRCcuBasicInfoProbeTest(unittest.TestCase): + def compile_probe(self): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + temp_dir = tempfile.TemporaryDirectory() + temp_path = Path(temp_dir.name) + probe_bin = temp_path / "ccu_tilexr_basic_info_probe" + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(PROBE_SOURCE), + str(CCU_DIRECT_RUNTIME_SOURCE), + str(CCU_DRIVER_SOURCE), + str(CCU_HCCP_LOADER_SOURCE), + str(CCU_RA_PROVIDER_SOURCE), + str(CCU_SPECS_SOURCE), + "-ldl", + "-pthread", + "-o", + str(probe_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return temp_dir, probe_bin + + def test_probe_compiles_against_tilexr_owned_ccu_ra_chain(self): + temp_dir, _ = self.compile_probe() + temp_dir.cleanup() + + def test_probe_uses_tilexr_ra_provider_not_hcomm_runtime(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "TileXRCcuDirectRuntime", + "TileXRCcuDirectRuntimeOptions", + "TileXRCcuDriverAdapter", + "TileXRCcuDecodeBasicInfo", + "deviceLogicId", + "runtime.QueryBasicInfo", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + for needle in [ + "#include + #include + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + uint32_t die = 0; + uint32_t offset = 0; + uint32_t dataLen = 0; + uint32_t arraySize = 0; + }; + + struct FakeState { + std::vector deviceBytes; + std::vector calls; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + ObservedCall call; + call.op = in.op; + call.die = in.data.dataInfo.udieIdx; + call.offset = in.offsetStartIdx; + call.dataLen = in.data.dataInfo.dataLen; + call.arraySize = in.data.dataInfo.dataArraySize; + state->calls.push_back(call); + out->opRet = 0; + return 0; + } + + TileXRCcuLowerLayerInstallPlan MakeLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 7; + pfe.ctx.raw[0] = 0xa1; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb2; + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 11; + channel.ctx.raw[0] = 0xc3; + lowerLayer.channels.push_back(channel); + lowerLayer.xnClears.push_back({1, 32, 1}); + lowerLayer.ckeClears.push_back({1, 16, 1}); + return lowerLayer; + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + options.syncResourceCount = 3; + options.syncInstructionCount = 143; + options.bindingsPerSyncResource = 3; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-direct-install-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.ckeStartId = 332; + options.channelStartId = 2; + + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeLowerLayer(); + options.lowerLayerPlan = &lowerLayer; + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 1; + } + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = memoryOps; + options.repositoryMemoryUserData = &state; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + if (ret != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "direct install attempt should stop at known missing surfaces: " + << report.message << "\n"; + return 2; + } + if (!report.pipelineBuilt || !report.installAttempted || report.submitReady || + report.submitTaskCount != 0 || report.missingInstallSurfaceCount != 2 || + report.publicVerifiedInstallSurfaceCount != 4) { + std::cerr << "direct install report mismatch\n"; + return 3; + } + if (report.message.find("remote XN install provider is missing") == + std::string::npos) { + std::cerr << "direct install diagnostic is weak: " << report.message << "\n"; + return 4; + } + if (attempt.package.tasks.size() != 2 || attempt.manifest.requirements.size() != 6 || + attempt.allocation.receiptId == 0 || + attempt.repositoryReceipt.instructionStartId != attempt.package.repository.missionStartId || + attempt.repositoryReceipt.instructionCount != attempt.package.repository.missionCount || + attempt.package.tasks[0].instStartId != attempt.package.repository.missionStartId || + attempt.package.tasks[0].instCnt != 13 || + attempt.package.tasks[1].instStartId != + attempt.package.repository.missionStartId + attempt.package.repository.sqeLoadCount || + attempt.package.tasks[1].instCnt != + attempt.package.repository.missionCount - attempt.package.repository.sqeLoadCount) { + std::cerr << "attempt artifacts mismatch" + << " taskSize=" << attempt.package.tasks.size() + << " reqSize=" << attempt.manifest.requirements.size() + << " receipt=" << attempt.allocation.receiptId + << " receiptStart=" << attempt.repositoryReceipt.instructionStartId + << " missionStart=" << attempt.package.repository.missionStartId + << " receiptCount=" << attempt.repositoryReceipt.instructionCount + << " missionCount=" << attempt.package.repository.missionCount + << " task0Start=" << attempt.package.tasks[0].instStartId + << " task0Cnt=" << attempt.package.tasks[0].instCnt + << " task1Start=" << attempt.package.tasks[1].instStartId + << " expectedTask1Start=" + << attempt.package.repository.missionStartId + attempt.package.repository.sqeLoadCount + << " task1Cnt=" << attempt.package.tasks[1].instCnt + << " expectedTask1Cnt=" + << attempt.package.repository.missionCount - attempt.package.repository.sqeLoadCount + << "\n"; + return 5; + } + if (!attempt.evidence.repositoryInstalled || !attempt.evidence.missionInstalled || + !attempt.evidence.localXnInstalled || + !attempt.evidence.notifyCkeInstalled || + attempt.evidence.channelBindingsInstalled || + attempt.evidence.remoteXnBound) { + std::cerr << "attempt evidence mismatch\n"; + return 6; + } + if (state.calls.size() != 7 || + state.calls[0].op != TILEXR_CCU_U_OP_SET_MSID_TOKEN || + state.calls[1].op != TILEXR_CCU_U_OP_SET_PFE || + state.calls[2].op != TILEXR_CCU_U_OP_SET_JETTY_CTX || + state.calls[3].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[4].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[5].op != TILEXR_CCU_U_OP_SET_CKE || + state.calls[6].op != TILEXR_CCU_U_OP_SET_INSTRUCTION) { + std::cerr << "unexpected direct install call sequence\n"; + return 7; + } + if (state.calls[6].offset != attempt.package.repository.missionStartId || + state.calls[6].dataLen != + attempt.package.repository.missionCount * sizeof(TileXRCcuInstr)) { + std::cerr << "repository install range mismatch\n"; + return 8; + } + if (state.calls[0].offset != 0 || state.calls[0].dataLen != 0 || + state.calls[0].arraySize != 0) { + std::cerr << "MSID token install envelope mismatch\n"; + return 10; + } + if (TileXRCcuReleaseRepositoryInstallReceipt( + attempt.repositoryReceipt, memoryOps, &state, &attempt.repositoryReleaseReport) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "repository release failed: " << attempt.repositoryReleaseReport.message << "\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_install_attempt_becomes_submit_ready_with_remote_xn_peer_exchange_proof(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + uint32_t offset = 0; + uint32_t arraySize = 0; + }; + + struct FakeState { + std::vector deviceBytes; + std::vector calls; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls.push_back({in.op, in.offsetStartIdx, in.data.dataInfo.dataArraySize}); + out->opRet = 0; + return 0; + } + + TileXRCcuLowerLayerInstallPlan MakeCompleteLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa1; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb2; + lowerLayer.jettys.push_back(jetty); + + for (uint32_t i = 0; i < 3; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2 + i; + channel.ctx.raw[0] = 0xc3 + i; + lowerLayer.channels.push_back(channel); + } + + lowerLayer.xnClears.push_back({1, 1961, 14}); + lowerLayer.ckeClears.push_back({1, 332, 3}); + lowerLayer.remoteXnBindings.push_back({1, 2, 1961, 1975, 332, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 3, 1962, 1976, 333, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 4, 1963, 1977, 334, 0, true, 0, true, true, true}); + return lowerLayer; + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + options.syncResourceCount = 3; + options.syncInstructionCount = 143; + options.bindingsPerSyncResource = 3; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-direct-install-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.ckeStartId = 332; + options.channelStartId = 2; + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeCompleteLowerLayer(); + options.lowerLayerPlan = &lowerLayer; + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 1; + } + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = memoryOps; + options.repositoryMemoryUserData = &state; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + if (ret != TILEXR_SUCCESS) { + std::cerr << "direct install attempt should be submit-ready: " + << report.message << "\n"; + return 2; + } + if (!report.pipelineBuilt || !report.installAttempted || !report.installSucceeded || + !report.submitReady || report.missingInstallSurfaceCount != 0 || + report.publicVerifiedInstallSurfaceCount != 6 || + report.submitTaskCount != attempt.package.tasks.size()) { + std::cerr << "submit-ready direct install report mismatch\n"; + return 3; + } + if (!attempt.evidence.repositoryInstalled || !attempt.evidence.missionInstalled || + !attempt.evidence.localXnInstalled || !attempt.evidence.remoteXnBound || + !attempt.evidence.notifyCkeInstalled || !attempt.evidence.channelBindingsInstalled) { + std::cerr << "submit-ready evidence mismatch\n"; + return 4; + } + if (attempt.evidence.remoteXnSource.source.find("ValidateRemoteXnExchangeBindingProof") == + std::string::npos || + attempt.evidence.remoteXnSource.detail.find("syncXn") == std::string::npos) { + std::cerr << "remote XN evidence detail is weak\n"; + return 5; + } + if (state.calls.size() != 10 || + state.calls[0].op != TILEXR_CCU_U_OP_SET_MSID_TOKEN || + state.calls[1].op != TILEXR_CCU_U_OP_SET_PFE || + state.calls[2].op != TILEXR_CCU_U_OP_SET_JETTY_CTX || + state.calls[3].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[4].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[5].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[6].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[6].arraySize != 8 || + state.calls[7].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[7].arraySize != 6 || + state.calls[8].op != TILEXR_CCU_U_OP_SET_CKE || + state.calls[8].arraySize != 3 || + state.calls[9].op != TILEXR_CCU_U_OP_SET_INSTRUCTION) { + std::cerr << "submit-ready call sequence mismatch\n"; + return 6; + } + if (TileXRCcuReleaseRepositoryInstallReceipt( + attempt.repositoryReceipt, memoryOps, &state, &attempt.repositoryReleaseReport) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "repository release failed: " << attempt.repositoryReleaseReport.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_submit_tasks_runs_submitter_in_order_and_stops_on_failure(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + + using namespace TileXR; + + struct SubmitState { + std::vector starts; + int failOnCall = 0; + }; + + int FakeSubmit(const TileXRCcuTask& task, void* stream, void* userData) + { + if (stream != reinterpret_cast(0x1234)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* state = static_cast(userData); + state->starts.push_back(task.instStartId); + if (state->failOnCall != 0 && + state->starts.size() == static_cast(state->failOnCall)) { + return TILEXR_ERROR_MKIRT; + } + return TILEXR_SUCCESS; + } + + TileXRCcuTask Task(uint16_t start) + { + TileXRCcuTask task; + task.dieId = 1; + task.missionId = 6; + task.instStartId = start; + task.instCnt = 13; + task.key = 0x059b0f03U; + task.argSize = TILEXR_CCU_SQE_ARGS_LEN; + return task; + } + + int main() + { + std::vector tasks; + tasks.push_back(Task(489)); + tasks.push_back(Task(502)); + + SubmitState okState; + TileXRCcuDirectSubmitReport report; + if (TileXRCcuSubmitPreparedTasks( + tasks, reinterpret_cast(0x1234), FakeSubmit, &okState, &report) != + TILEXR_SUCCESS) { + std::cerr << "submit batch failed: " << report.message << "\n"; + return 1; + } + if (!report.submitted || report.taskCount != 2 || report.submittedTaskCount != 2 || + okState.starts.size() != 2 || okState.starts[0] != 489 || okState.starts[1] != 502 || + report.message.find("submitted") == std::string::npos) { + std::cerr << "submit success report mismatch\n"; + return 2; + } + + SubmitState failState; + failState.failOnCall = 2; + if (TileXRCcuSubmitPreparedTasks( + tasks, reinterpret_cast(0x1234), FakeSubmit, &failState, &report) != + TILEXR_ERROR_MKIRT) { + std::cerr << "submit failure was not propagated\n"; + return 3; + } + if (report.submitted || report.taskCount != 2 || report.submittedTaskCount != 1 || + failState.starts.size() != 2 || + report.message.find("task=1") == std::string::npos || + report.message.find("ret=-2") == std::string::npos || + report.message.find("missionId=6") == std::string::npos || + report.message.find("instStartId=502") == std::string::npos || + report.message.find("key=0x59b0f03") == std::string::npos || + report.message.find("argSize=13") == std::string::npos) { + std::cerr << "submit failure report mismatch: " << report.message << "\n"; + return 4; + } + + if (TileXRCcuSubmitPreparedTasks( + tasks, nullptr, FakeSubmit, &okState, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null stream accepted\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_submit_default_runtime_trace_indexes_final_tasks(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + #include + #include + + #include + + using namespace TileXR; + + namespace { + int g_launchCount = 0; + } + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t*, rtStream_t) + { + ++g_launchCount; + return RT_ERROR_NONE; + } + + TileXRCcuTask Task(uint16_t start) + { + TileXRCcuTask task; + task.dieId = 1; + task.missionId = 6; + task.timeout = 68; + task.instStartId = start; + task.instCnt = 13; + task.key = 0x059b0f03U; + task.argSize = TILEXR_CCU_SQE_ARGS_LEN; + task.args[0] = 0x1000ULL + start; + return task; + } + + int main() + { + setenv("TILEXR_CCU_DIRECT_TRACE", "1", 1); + + std::vector tasks; + tasks.push_back(Task(489)); + tasks.push_back(Task(502)); + + TileXRCcuDirectSubmitReport report; + const int ret = TileXRCcuSubmitPreparedTasks( + tasks, reinterpret_cast(0x1234), nullptr, nullptr, &report); + if (ret != TILEXR_SUCCESS || !report.submitted || g_launchCount != 2) { + std::cerr << "submit mismatch ret=" << ret + << " submitted=" << report.submitted + << " launchCount=" << g_launchCount + << " message=" << report.message << "\n"; + return 1; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("TileXRDirectCcuTrace finalRuntimeTask[0] dieId=1 missionId=6 timeout=68 instStartId=489", result.stderr) + self.assertIn("TileXRDirectCcuTrace finalRuntimeTask[1] dieId=1 missionId=6 timeout=68 instStartId=502", result.stderr) + self.assertNotIn("TileXRDirectCcuTrace finalRuntimeTask task=0", result.stderr) + + def test_direct_install_attempt_can_prepare_lower_layer_plan_after_allocation(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + std::vector ops; + }; + + struct CallbackState { + uint32_t callCount = 0; + uint32_t syncResourceCount = 0; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->ops.push_back(in.op); + out->opRet = 0; + return 0; + } + + int PrepareLowerLayerPlan( + const TileXRCcuResourceAllocation& allocation, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + void* userData) + { + auto* state = static_cast(userData); + ++state->callCount; + state->syncResourceCount = allocation.remoteXn.num; + if (plan == nullptr || report == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *plan = TileXRCcuLowerLayerInstallPlan{}; + plan->msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = allocation.channels.startId; + pfe.ctx.raw[0] = 0xa1; + plan->pfes.push_back(pfe); + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb2; + plan->jettys.push_back(jetty); + TileXRCcuLowerLayerPayloadReport payloadReport; + for (uint32_t i = 0; i < allocation.channels.num; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = allocation.channels.startId + i; + TileXRCcuChannelCtxV1Spec channelSpec; + channelSpec.remoteEid[0] = static_cast(0x10 + i); + channelSpec.tpn = 1; + channelSpec.sourcePfeId = allocation.channels.startId; + channelSpec.startJettyId = 1024; + channelSpec.jettyCount = 1; + channelSpec.dieId = 1; + channelSpec.memoryTokenId = 0x2345; + channelSpec.memoryTokenValue = 0x67890000U; + channelSpec.remoteCcuVa = 0x0000100054800000ULL; + if (TileXRCcuBuildChannelCtxV1(channelSpec, &channel.ctx, &payloadReport) != TILEXR_SUCCESS) { + report->message = payloadReport.message; + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + plan->channels.push_back(channel); + } + plan->xnClears.push_back({1, allocation.localXn.startId, allocation.localXn.num}); + plan->ckeClears.push_back({1, allocation.notifyCke.startId, allocation.notifyCke.num}); + for (uint32_t i = 0; i < allocation.remoteXn.num; ++i) { + plan->remoteXnBindings.push_back({ + 1, + static_cast(allocation.channels.startId + i), + static_cast(allocation.localXn.startId + i), + static_cast(allocation.remoteXn.startId + i), + static_cast(allocation.notifyCke.startId + i), + i, + true, + static_cast(allocation.localWaitCke.startId + i), + true, + true, + true}); + } + report->msidTokenCount = 1; + report->pfeCount = 1; + report->jettyCount = 1; + report->localJettyCtxCount = 1; + report->channelCount = allocation.channels.num; + report->ckeClearCount = 1; + report->message = "callback lower-layer plan prepared"; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + FakeState fake; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &fake, &driverReport) != TILEXR_SUCCESS) { + return 1; + } + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + options.syncResourceCount = 3; + options.syncInstructionCount = 143; + options.bindingsPerSyncResource = 3; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-callback-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.gsaStartId = 510; + options.remoteXnStartId = 2361; + options.remoteXnCount = 8; + options.ckeStartId = 332; + options.remoteNotifyCkeStartId = 364; + options.remoteNotifyCkeCount = 8; + options.channelStartId = 2; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = {FakeAlloc, FakeCopy, FakeFree}; + options.repositoryMemoryUserData = &fake; + + CallbackState callback; + options.prepareLowerLayerPlan = PrepareLowerLayerPlan; + options.lowerLayerPlanUserData = &callback; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + if (ret != TILEXR_SUCCESS || !report.submitReady || report.submitTaskCount != 2) { + std::cerr << "callback install should be submit-ready: " << report.message << "\n"; + return 2; + } + if (callback.callCount != 1 || callback.syncResourceCount != 3) { + std::cerr << "lower-layer callback was not called with allocation\n"; + return 3; + } + if (attempt.plan.kernelLocalGsa.startId != 510 || attempt.plan.kernelLocalGsa.num != 1 || + attempt.allocation.localGsa.startId != 510 || attempt.allocation.localGsa.num != 1) { + std::cerr << "GSA resource did not flow into direct CCU producer plan\n"; + return 13; + } + if (attempt.preparedLowerLayerPlan.remoteXnBindings.size() != 3 || + attempt.lowerLayerPlanReport.message.find("callback") == std::string::npos) { + std::cerr << "prepared lower-layer artifacts were not retained\n"; + return 4; + } + if (!attempt.evidence.remoteXnBound || !attempt.evidence.channelBindingsInstalled || + attempt.evidence.remoteXnSource.source.find("ValidateRemoteXnExchangeBindingProof") == + std::string::npos) { + std::cerr << "prepared lower-layer evidence mismatch\n"; + return 5; + } + const uint64_t expectedPackedToken = (1ULL << 52U) | (0x1234ULL << 32U) | 0x5678ULL; + if (attempt.package.plan.taskWindows[0].args.size() != TILEXR_CCU_SQE_ARGS_LEN || + attempt.package.plan.taskWindows[0].args[0] != basic.resourceAddr || + attempt.package.plan.taskWindows[0].args[1] != basic.resourceAddr || + attempt.package.plan.taskWindows[0].args[2] != expectedPackedToken || + attempt.package.plan.taskWindows[0].args[3] != 0x0000100054800000ULL || + attempt.submitTasks[0].args[0] != basic.resourceAddr || + attempt.submitTasks[0].args[1] != basic.resourceAddr || + attempt.submitTasks[0].args[2] != expectedPackedToken || + attempt.submitTasks[0].args[3] != 0x0000100054800000ULL) { + std::cerr << "hcomm-style SQE task args were not generated from lower-layer token state\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_install_attempt_passes_split_cke_ranges_to_lower_layer_callback(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + struct CallbackState { + bool sawSplitCke = false; + uint16_t localWaitStart = 0; + uint16_t remoteNotifyStart = 0; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + int PrepareSplitLowerLayerPlan( + const TileXRCcuResourceAllocation& allocation, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + void* userData) + { + auto* state = static_cast(userData); + state->localWaitStart = allocation.localWaitCke.startId; + state->remoteNotifyStart = allocation.remoteNotifyCke.startId; + state->sawSplitCke = + allocation.localWaitCke.startId == 332 && + allocation.localWaitCke.num == 3 && + allocation.remoteNotifyCke.startId == 364 && + allocation.remoteNotifyCke.num == 3 && + allocation.notifyCke.startId == 364 && + allocation.notifyCke.num == 3; + if (plan == nullptr || report == nullptr || !state->sawSplitCke) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + *plan = TileXRCcuLowerLayerInstallPlan{}; + plan->msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = allocation.channels.startId; + pfe.ctx.raw[0] = 0xa1; + plan->pfes.push_back(pfe); + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb2; + plan->jettys.push_back(jetty); + for (uint32_t i = 0; i < allocation.channels.num; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = allocation.channels.startId + i; + channel.ctx.raw[0] = 0xc3 + i; + plan->channels.push_back(channel); + } + plan->xnClears.push_back({1, allocation.localXn.startId, allocation.localXn.num}); + plan->ckeClears.push_back({1, allocation.localWaitCke.startId, allocation.localWaitCke.num}); + for (uint32_t i = 0; i < allocation.remoteXn.num; ++i) { + plan->remoteXnBindings.push_back({ + 1, + allocation.channels.startId + i, + static_cast(allocation.localXn.startId + i), + static_cast(allocation.remoteXn.startId + i), + static_cast(allocation.remoteNotifyCke.startId + i), + i, + true, + static_cast(allocation.localWaitCke.startId + i), + true, + true, + true}); + } + report->message = "split CKE lower-layer plan prepared"; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + FakeState fake; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &fake, &driverReport) != TILEXR_SUCCESS) { + return 1; + } + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.syncResourceCount = 3; + options.syncInstructionCount = 143; + options.bindingsPerSyncResource = 3; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-split-cke-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.ckeStartId = 332; + options.localWaitCkeStartId = 332; + options.remoteNotifyCkeStartId = 364; + options.channelStartId = 2; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = {FakeAlloc, FakeCopy, FakeFree}; + options.repositoryMemoryUserData = &fake; + + CallbackState callback; + options.prepareLowerLayerPlan = PrepareSplitLowerLayerPlan; + options.lowerLayerPlanUserData = &callback; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + if (ret != TILEXR_SUCCESS || !report.submitReady) { + std::cerr << "split CKE direct install should be submit-ready: " + << report.message << "\n"; + return 2; + } + if (!callback.sawSplitCke || callback.localWaitStart != 332 || + callback.remoteNotifyStart != 364) { + std::cerr << "split CKE allocation was not passed to callback\n"; + return 3; + } + if (attempt.plan.kernelLocalCke.startId != 332 || + attempt.plan.syncResources[0].localWaitCke != 332 || + attempt.plan.syncResources[0].notifyCke != 364 || + attempt.evidence.remoteXnSource.detail.find("syncXn") == std::string::npos) { + std::cerr << "split CKE plan or evidence mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_install_attempt_rewrites_sync_resources_from_peer_lower_layer_proof(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + int PreparePeerLowerLayerPlan( + const TileXRCcuResourceAllocation& allocation, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + void*) + { + if (plan == nullptr || report == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *plan = TileXRCcuLowerLayerInstallPlan{}; + plan->msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = allocation.channels.startId; + pfe.ctx.raw[0] = 0xa1; + plan->pfes.push_back(pfe); + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb2; + plan->jettys.push_back(jetty); + for (uint32_t i = 0; i < allocation.channels.num; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = allocation.channels.startId + i; + channel.ctx.raw[0] = 0xc3 + i; + plan->channels.push_back(channel); + } + plan->xnClears.push_back({1, allocation.localXn.startId, allocation.localXn.num}); + plan->ckeClears.push_back({1, allocation.localWaitCke.startId, allocation.localWaitCke.num}); + + for (uint32_t i = 0; i < allocation.remoteXn.num; ++i) { + const uint16_t peerLocalXn = static_cast(0x1a0 + i); + const uint16_t peerNotifyCke = static_cast(0x360 + i); + plan->remoteXnBindings.push_back({ + 1, + allocation.channels.startId + i, + static_cast(allocation.localXn.startId + i), + peerLocalXn, + peerNotifyCke, + i, + true, + static_cast(allocation.localWaitCke.startId + i), + true, + true, + true}); + } + report->message = "peer lower-layer plan prepared"; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + FakeState fake; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &fake, &driverReport) != TILEXR_SUCCESS) { + return 1; + } + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + options.syncResourceCount = 2; + options.syncInstructionCount = 9; + options.bindingsPerSyncResource = 1; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-peer-xn-rewrite-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 0x2a0; + options.ckeStartId = 0x220; + options.localWaitCkeStartId = 0x220; + options.remoteNotifyCkeStartId = 0x360; + options.channelStartId = 2; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = {FakeAlloc, FakeCopy, FakeFree}; + options.repositoryMemoryUserData = &fake; + options.prepareLowerLayerPlan = PreparePeerLowerLayerPlan; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + if (ret != TILEXR_SUCCESS || !report.submitReady) { + std::cerr << "peer XN rewrite should make direct install submit-ready: " + << report.message << "\n"; + return 2; + } + if (attempt.plan.syncResources.size() != 2 || + attempt.plan.syncResources[0].remoteXn != 0x1a0 || + attempt.plan.syncResources[1].remoteXn != 0x1a1 || + attempt.plan.syncResources[0].notifyCke != 0x360 || + attempt.plan.syncResources[1].notifyCke != 0x361) { + std::cerr << "producer plan was not rewritten from peer lower-layer proof\n"; + return 3; + } + if (attempt.package.plan.syncResources[0].remoteXn != 0x1a0 || + attempt.package.plan.syncResources[1].remoteXn != 0x1a1 || + attempt.package.plan.syncResources[0].notifyCke != 0x360 || + attempt.package.plan.syncResources[1].notifyCke != 0x361 || + attempt.submitTasks.size() != 2 || + attempt.submitTasks[0].instStartId != attempt.package.repository.missionStartId || + attempt.submitTasks[0].instCnt != 13 || + attempt.submitTasks[1].instStartId != + attempt.package.repository.missionStartId + attempt.package.repository.sqeLoadCount || + attempt.submitTasks[1].instCnt != + attempt.package.repository.missionCount - attempt.package.repository.sqeLoadCount) { + std::cerr << "launch package did not use peer lower-layer proof resources\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_install_attempt_release_helper_frees_repository_receipt_once(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + + using namespace TileXR; + + struct FakeState { + int freeCalls = 0; + int payload = 0; + }; + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != &state->payload) { + return -1; + } + ++state->freeCalls; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDirectInstallAttempt attempt; + attempt.repositoryMemoryOps.free = FakeFree; + attempt.repositoryMemoryUserData = &state; + attempt.repositoryReceipt.deviceInstructionPtr = &state.payload; + attempt.repositoryReceipt.deviceInstructionAddr = + reinterpret_cast(&state.payload); + attempt.repositoryReceipt.uploaded = true; + attempt.repositoryReceipt.installed = true; + + if (TileXRCcuReleaseDirectInstallAttemptResources(attempt) != TILEXR_SUCCESS) { + std::cerr << "first release failed: " << attempt.repositoryReleaseReport.message << "\n"; + return 1; + } + if (state.freeCalls != 1 || attempt.repositoryReceipt.deviceInstructionPtr != nullptr || + attempt.repositoryReleaseReport.message != "ok") { + std::cerr << "release did not clear repository receipt\n"; + return 2; + } + if (TileXRCcuReleaseDirectInstallAttemptResources(attempt) != TILEXR_SUCCESS || + state.freeCalls != 1) { + std::cerr << "release helper is not idempotent\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_install_attempt_trace_dumps_final_peer_resources_and_tasks_when_enabled(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + #include + + using namespace TileXR; + + int main() + { + setenv("TILEXR_CCU_DIRECT_TRACE", "1", 1); + + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.syncResourceCount = 1; + options.syncInstructionCount = 2; + options.bindingsPerSyncResource = 1; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-direct-trace-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.ckeStartId = 332; + options.channelStartId = 2; + options.offlineOnly = true; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + (void)ret; + if (!report.pipelineBuilt || attempt.package.tasks.empty()) { + std::cerr << "trace setup failed before package build: " << report.message << "\n"; + return 1; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("TileXRDirectCcuTrace", result.stderr) + self.assertIn("syncResource[0]", result.stderr) + self.assertIn("remoteXn=", result.stderr) + self.assertIn("taskWindow[0]", result.stderr) + self.assertIn("task[0]", result.stderr) + self.assertNotIn("program.sqeLoad[0]", result.stderr) + self.assertIn("program.sync[0]", result.stderr) + + def test_direct_install_attempt_trace_decodes_lower_layer_contexts_when_enabled(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int PrepareLowerLayer( + const TileXRCcuResourceAllocation& allocation, + TileXRCcuLowerLayerInstallPlan* plan, + TileXRCcuLowerLayerPlanBuilderReport* report, + void* userData) + { + auto* basic = static_cast(userData); + + std::array eid; + for (uint32_t i = 0; i < eid.size(); ++i) { + eid[i] = static_cast(0x20 + i); + } + + TileXRCcuLowerLayerPlanSpec spec; + spec.msidToken.dieId = basic->dieId; + spec.msidToken.msId = basic->msId; + spec.msidToken.tokenId = basic->msidToken.tokenId; + spec.msidToken.tokenValue = basic->msidToken.tokenValue; + spec.msidToken.valid = basic->msidToken.valid; + spec.pfe.dieId = basic->dieId; + spec.pfe.pfeOffset = allocation.channels.startId; + spec.pfe.startJettyId = 0x400; + spec.pfe.startLocalJettyCtxId = 0x02; + + TileXRCcuLowerLayerJettySpec jetty; + jetty.dieId = basic->dieId; + jetty.pfeId = allocation.channels.startId; + jetty.startJettyCtxId = spec.pfe.startLocalJettyCtxId; + jetty.doorbellVa = 0x1122334455667788ULL; + jetty.doorbellTokenId = 0xabcdeU; + jetty.doorbellTokenValue = 0x12345678U; + jetty.sqDepth = 8; + jetty.wqeBasicBlockStartId = 0x34; + spec.jettys.push_back(jetty); + + TileXRCcuLowerLayerChannelSpec channel; + channel.dieId = basic->dieId; + channel.channelId = allocation.channels.startId; + channel.remoteEid = eid; + channel.tpn = 0x00876543U; + channel.sourcePfeId = allocation.channels.startId; + channel.startJettyId = 0x400; + channel.jettyCount = 1; + channel.memoryTokenId = 0x23456U; + channel.memoryTokenValue = 0x3456789aU; + channel.remoteCcuVa = + basic->resourceAddr + TILEXR_CCU_V1_XN_RESOURCE_OFFSET + allocation.remoteXn.startId * 8ULL; + spec.channels.push_back(channel); + + spec.xnClear.dieId = basic->dieId; + spec.xnClear.startXnId = allocation.localXn.startId; + spec.xnClear.count = allocation.localXn.num; + spec.xnClear.valid = true; + spec.ckeClear.dieId = basic->dieId; + spec.ckeClear.startCkeId = allocation.notifyCke.startId; + spec.ckeClear.count = allocation.notifyCke.num; + spec.ckeClear.valid = true; + + TileXRCcuRemoteXnBindingProof proof; + proof.dieId = basic->dieId; + proof.channelId = allocation.channels.startId; + proof.localXn = allocation.localXn.startId; + proof.remoteXn = allocation.remoteXn.startId; + proof.notifyCke = allocation.notifyCke.startId; + proof.localWaitCke = allocation.notifyCke.startId; + proof.peerRank = 0; + proof.peerExchangeObserved = true; + proof.endpointRouteVerified = true; + spec.remoteXnBindings.push_back(proof); + + return TileXRCcuBuildLowerLayerInstallPlan(spec, plan, report); + } + + int main() + { + setenv("TILEXR_CCU_DIRECT_TRACE", "1", 1); + + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.syncResourceCount = 1; + options.syncInstructionCount = 2; + options.bindingsPerSyncResource = 1; + options.deviceId = 3; + options.rank = 1; + options.provider = "unit-test-direct-trace-lower-layer-provider"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.xnStartId = 1961; + options.ckeStartId = 332; + options.channelStartId = 2; + options.offlineOnly = true; + options.prepareLowerLayerPlan = PrepareLowerLayer; + options.lowerLayerPlanUserData = &basic; + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectInstallAttempt(options, &attempt, &report); + (void)ret; + if (!report.pipelineBuilt || attempt.preparedLowerLayerPlan.channels.empty()) { + std::cerr << "lower-layer trace setup failed: " << report.message << "\n"; + return 1; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("decoded=PfeCtx", result.stderr) + self.assertIn("decoded=LocalJettyCtx", result.stderr) + self.assertIn("decoded=ChannelCtxV1", result.stderr) + self.assertIn("startTaJettyId=1024", result.stderr) + self.assertIn("startLocalJettyCtxId=2", result.stderr) + self.assertIn("doorbellVa=0x1122334455667788", result.stderr) + self.assertIn("wqeBasicBlockStartId=52", result.stderr) + self.assertIn("sourcePfeId=2", result.stderr) + self.assertIn("remoteCcuVa=", result.stderr) + + def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_direct_orchestrator.h", cmake) + self.assertIn("ccu/tilexr_ccu_direct_orchestrator.cpp", cmake) + self.assertIn("struct TileXRCcuDirectInstallOptions", header) + self.assertIn("struct TileXRCcuDirectMemoryCopySpec", header) + self.assertIn("struct TileXRCcuDirectInstallAttempt", header) + self.assertIn("struct TileXRCcuDirectInstallReport", header) + self.assertIn("struct TileXRCcuDirectSubmitReport", header) + self.assertIn("TileXRCcuLowerLayerPlanPrepareFn", header) + self.assertIn("localWaitCkeStartId", header) + self.assertIn("gsaStartId", header) + self.assertIn("remoteNotifyCkeStartId", header) + self.assertIn("prepareLowerLayerPlan", header) + self.assertIn("preparedLowerLayerPlan", header) + self.assertIn("lowerLayerPlanReport", header) + self.assertIn("TileXRCcuSubmitPreparedTasks", header) + self.assertIn("TileXRCcuRunDirectInstallAttempt", header) + self.assertIn("TileXRCcuRunDirectMemoryCopyInstallAttempt", header) + self.assertIn("TileXRCcuDecodeBasicInfo", source) + self.assertIn("TileXRCcuBuildResourceSpec", source) + self.assertIn("TileXRCcuResourceAllocator", source) + self.assertIn("TileXRCcuBuildLaunchPackage", source) + self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) + self.assertIn("BuildDirectMemoryCopyLaunchPackage", source) + self.assertIn("TileXRCcuBindLaunchPackageInstallScope", source) + self.assertIn("TileXRCcuBuildInstallManifest", source) + self.assertIn("options.prepareLowerLayerPlan", source) + self.assertIn("TileXRCcuInstallHardware", source) + self.assertIn("TileXRCcuPrepareSubmitTasks", source) + self.assertIn("TileXRCcuSubmitTask", source) + self.assertIn("remote XN install provider is missing", source) + self.assertIn("TraceDecodedInstr", source) + self.assertIn("TraceDecodedPfeCtx", source) + self.assertIn("TraceDecodedLocalJettyCtx", source) + self.assertIn("TraceDecodedChannelCtxV1", source) + self.assertIn("decoded=SyncXn", source) + self.assertIn("decoded=SyncCke", source) + self.assertIn("decoded=LoadImdToXn", source) + self.assertIn("decoded=LoadImdToGSA", source) + self.assertIn("decoded=TransRmtMemToLocMem", source) + self.assertIn("decoded=TransLocMemToRmtMem", source) + self.assertIn("TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER", source) + self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_direct_install_options_default_to_lower_layer_first(self): + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + + self.assertIn( + "TileXRCcuInstallOrder installOrder = " + "TileXRCcuInstallOrder::InstallLowerLayerFirst", + header, + ) + self.assertIn("installRequest.installOrder = options.installOrder", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py new file mode 100644 index 00000000..29f9f6fd --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import os +import platform +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROBE_SOURCE = REPO_ROOT / "tests" / "ccu" / "ccu_tilexr_direct_smoke_probe.cpp" +COMM_DIR = REPO_ROOT / "src" / "comm" +INCLUDE_DIR = REPO_ROOT / "src" / "include" + + +PRIVATE_CCU_NEEDLES = [ + "#include ", + "#include \"hccl.h\"", + "pkg_inc/hcomm", + "pkg_inc/hccl", + "include/hccl", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "HcclChannelAcquire", + "HcclGetChannelForCcu", + "CcuResBatchAllocator", + "CcuResRepository", + "CcuDeviceManager", + "CcuKernelMgr", + "CtxMgrImp", + "GeneTaskParam", + "GetMissionKey", + "SetMissionId", + "SetMissionKey", + "SetInstrId", + "SetCcuInstrInfo", + "LoadInstruction", + "AllocIns", + "AllocCke", + "AllocXn", +] + + +def cann_paths(): + ascend_home = os.environ.get("ASCEND_HOME_PATH") or os.environ.get("ASCEND_HOME") + if not ascend_home: + return None + + arch = os.environ.get("ARCH") + if not arch: + arch = "aarch64" if platform.machine() in ("aarch64", "arm64") else "x86_64" + + cann_root = Path(ascend_home) / f"{arch}-linux" + include_dirs = [ + cann_root / "pkg_inc", + cann_root / "pkg_inc" / "runtime", + cann_root / "include", + ] + if not any((include_dir / "acl" / "acl.h").exists() for include_dir in include_dirs): + return None + + lib_dir = cann_root / "lib64" + if not (lib_dir / "libascendcl.so").exists(): + return None + driver_root = Path(os.environ.get("ASCEND_DRIVER_PATH", "/usr/local/Ascend/driver")) + driver_lib_dir = driver_root / "lib64" / "driver" + return include_dirs, lib_dir, driver_lib_dir + + +def find_tile_comm(): + env_path = os.environ.get("TILEXR_TILE_COMM_LIB") + candidates = [] + if env_path: + candidates.append(Path(env_path)) + candidates.extend( + [ + REPO_ROOT / "build" / "src" / "comm" / "libtile-comm.so", + REPO_ROOT / "install" / "lib64" / "libtile-comm.so", + REPO_ROOT / "install" / "lib" / "libtile-comm.so", + REPO_ROOT / "install_direct_ccu_guard" / "lib64" / "libtile-comm.so", + ] + ) + for candidate in candidates: + if candidate.exists(): + return candidate + return None + + +class TileXRCcuDirectSmokeProbeTest(unittest.TestCase): + def compile_probe(self): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + cann_config = cann_paths() + if cann_config is None: + self.skipTest("CANN ACL headers/libs are not configured for direct CCU smoke probe") + tile_comm = find_tile_comm() + if tile_comm is None: + self.skipTest("libtile-comm.so is not built; build tile-comm before compiling direct smoke probe") + + include_dirs, cann_lib_dir, driver_lib_dir = cann_config + temp_dir = tempfile.TemporaryDirectory() + temp_path = Path(temp_dir.name) + probe_bin = temp_path / "ccu_tilexr_direct_smoke_probe" + compile_cmd = [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + ] + for include_dir in include_dirs: + if include_dir.exists(): + compile_cmd.extend(["-I", str(include_dir)]) + compile_cmd.extend( + [ + str(PROBE_SOURCE), + "-L", + str(tile_comm.parent), + "-L", + str(cann_lib_dir), + "-L", + str(driver_lib_dir), + f"-Wl,-rpath-link,{tile_comm.parent}", + f"-Wl,-rpath-link,{cann_lib_dir}", + f"-Wl,-rpath-link,{driver_lib_dir}", + "-ltile-comm", + "-lascendcl", + "-lruntime", + "-ldl", + "-pthread", + "-o", + str(probe_bin), + ] + ) + subprocess.run( + compile_cmd, + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return temp_dir, probe_bin, tile_comm.parent, cann_lib_dir, driver_lib_dir + + def test_source_exists_and_defines_default_safe_env_guards(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ENABLE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_TRACE_LIFECYCLE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SUBMIT", source) + self.assertIn("TILEXR_CCU_PROBE_RANK", source) + self.assertIn("TILEXR_CCU_PROBE_RANK_SIZE", source) + self.assertIn("TILEXR_CCU_PROBE_DEVICE", source) + self.assertIn("TILEXR_CCU_PROBE_COMM_DOMAIN", source) + self.assertIn("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START", source) + self.assertIn("TILEXR_CCU_PROBE_SQE_ARG_COUNT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES", source) + self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE", source) + self.assertIn("TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM", source) + self.assertIn('text == "rt_hbm"', source) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_ORDER", source) + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID", source) + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID", source) + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE", source) + self.assertIn("InitCommForDirectCcuSmoke", source) + self.assertIn("RunThreadModeSmoke", source) + self.assertIn("ShouldFastExitAfterPrepareFailure", source) + self.assertIn("ShouldFastExitAfterRun", source) + self.assertIn("TraceLifecycle", source) + self.assertIn("RepositoryInstallWindowFromEnv", source) + self.assertIn("RepositoryInstallDataLenModeFromEnv", source) + self.assertIn("RepositoryMemoryAllocModeFromEnv", source) + self.assertIn("InstallOrderFromEnv", source) + self.assertIn("TileXRCommInitAll", source) + self.assertIn("DefaultSyncInstructionCount", source) + self.assertIn('std::string(value) == "sync_cke"', source) + self.assertIn('std::string(value) == "sync_cke_set_wait"', source) + self.assertIn('std::string(value) == "sync_cke_post_only"', source) + self.assertIn('std::string(value) == "local_cke"', source) + self.assertIn('std::string(value) == "local_cke_post_only"', source) + self.assertIn('std::string(value) == "sync_xn_post_only"', source) + self.assertIn('std::string(value) == "sync_xn_load_post_only"', source) + self.assertIn("TileXRCommInitRankWithDomain", source) + self.assertIn("TileXRCommInitRankDirectCcuWithDomain", source) + self.assertIn("TileXRCommPrepareDirectCcu", source) + self.assertIn("TileXRCommPrepareDirectCcuMemoryCopy", source) + self.assertIn("TileXRDirectCcuSubmitPrepared", source) + self.assertIn("TileXRDirectCcuDestroyPrepared", source) + self.assertIn("tilexr_ccu_direct_smoke config", source) + self.assertIn("barrierMode=", source) + self.assertIn("repositoryInstallWindow=", source) + self.assertIn("repositoryInstallDataLenMode=", source) + self.assertIn("repositoryMemoryAllocMode=", source) + self.assertIn("installOrder=", source) + self.assertIn("resourceWindowTokenId=", source) + self.assertIn("resourceWindowRawTokenId=", source) + self.assertIn("resourceWindowTokenValue=", source) + self.assertIn("tilexr_ccu_direct_smoke preparedTasks", source) + self.assertIn("tilexr_ccu_direct_smoke submitTiming", source) + self.assertIn("tilexr_ccu_direct_smoke p2pCcuCopy", source) + self.assertIn("aclrtMemcpy", source) + self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) + self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) + self.assertIn("passed=", source) + self.assertIn("syncMs=", source) + self.assertIn("std::this_thread::sleep_for", source) + self.assertIn("std::vector", source) + self.assertIn("std::vector", source) + + def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("kP2pCcuCopyEnv", source) + self.assertIn("RunP2pCcuCopy", source) + self.assertIn("TileXRDirectCcuMemoryCopyPrepareOptions", source) + self.assertIn("TileXRCommPrepareDirectCcuMemoryCopy", source) + self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) + self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) + self.assertIn("p2pCcuCopy", source) + + def test_thread_mode_path_uses_single_process_init_and_never_rank_ipc_init(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + thread_mode_pos = source.index("RunThreadModeSmoke") + thread_mode_body = source[thread_mode_pos: source.index("int main()")] + prepared_body = source[source.index("RunPreparedSmokeForRank"): thread_mode_pos] + self.assertIn("TileXRCommInitAll", thread_mode_body) + self.assertIn("RunPreparedSmokeForRank", thread_mode_body) + self.assertIn("TileXRCommPrepareDirectCcu", prepared_body) + self.assertIn("TileXRDirectCcuSubmitPrepared", prepared_body) + self.assertNotIn("TileXRCommInitRankWithDomain", thread_mode_body) + self.assertNotIn("TILEXR_COMM_ID", thread_mode_body) + + def test_thread_mode_worker_sets_device_before_direct_ccu_submit(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + thread_mode_body = source[source.index("int RunThreadModeSmoke"): source.index("int main()")] + + worker_body = thread_mode_body[ + thread_mode_body.index("rankThreads.emplace_back"): + thread_mode_body.index("for (auto& rankThread", thread_mode_body.index("rankThreads.emplace_back")) + ] + self.assertIn("aclrtSetDevice(devices[rank])", worker_body) + self.assertLess( + worker_body.index("aclrtSetDevice(devices[rank])"), + worker_body.index("RunPreparedSmokeForRank"), + ) + + def test_process_mode_can_opt_into_direct_ccu_only_init_to_bypass_peer_ipc(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + init_helper_body = source[ + source.index("int InitCommForDirectCcuSmoke"): + source.index("TileXRDirectCcuPrepareOptions MakePrepareOptions") + ] + main_source = source[source.index("int main()"):] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT", source) + self.assertIn("TileXRCommInitRankDirectCcuWithDomain", init_helper_body) + self.assertIn("TileXRCommInitRankWithDomain", init_helper_body) + self.assertIn("EnvFlag(kDirectCcuOnlyInitEnv)", init_helper_body) + self.assertIn("InitCommForDirectCcuSmoke", main_source) + + def test_process_mode_fast_exit_skips_comm_destroy_after_prepare_failure(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + main_source = source[source.index("int main()"):] + + final_ret_pos = main_source.index("int finalRet = RunPreparedSmokeForRank") + fast_exit_pos = main_source.index("ShouldFastExitAfterPrepareFailure(finalRet)") + destroy_pos = main_source.index("TileXRCommDestroy(raw)", final_ret_pos) + reset_pos = main_source.index("aclrtResetDevice(device)", final_ret_pos) + finalize_pos = main_source.index("aclFinalize()", final_ret_pos) + + self.assertLess(final_ret_pos, fast_exit_pos) + self.assertLess(fast_exit_pos, destroy_pos) + self.assertLess(fast_exit_pos, reset_pos) + self.assertLess(fast_exit_pos, finalize_pos) + self.assertIn("std::fflush(stdout)", main_source) + self.assertIn("std::fflush(stderr)", main_source) + self.assertIn("std::_Exit(finalRet)", main_source) + self.assertIn("tilexr_ccu_direct_smoke fastExitOnPrepareFailure=1", source) + + def test_fast_exit_reason_distinguishes_runtime_failures_from_prepare_failure(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + main_source = source[source.index("int main()"):] + + self.assertIn("FastExitReasonForReturnCode", source) + self.assertIn("direct CCU collective completion timed out; skipping cleanup", source) + self.assertIn("direct CCU P2P CCU-copy check failed; skipping cleanup", source) + self.assertIn("FastExitReasonForReturnCode(finalRet)", main_source) + self.assertNotIn( + 'reason="prepare failed; skipping cleanup to preserve diagnostic status"', + main_source, + ) + + def test_process_mode_can_fast_exit_after_run_to_isolate_cleanup_hangs(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + main_source = source[source.index("int main()"):] + + final_ret_pos = main_source.index("int finalRet = RunPreparedSmokeForRank") + fast_exit_pos = main_source.index("ShouldFastExitAfterRun()") + destroy_pos = main_source.index("TileXRCommDestroy(raw)", final_ret_pos) + + self.assertLess(final_ret_pos, fast_exit_pos) + self.assertLess(fast_exit_pos, destroy_pos) + self.assertIn("tilexr_ccu_direct_smoke fastExitAfterRun=1", source) + self.assertIn("TraceLifecycle(\"before TileXRDirectCcuDestroyPrepared\")", source) + self.assertIn("TraceLifecycle(\"after TileXRDirectCcuDestroyPrepared\")", source) + self.assertIn("TraceLifecycle(\"before aclrtSynchronizeStream\")", source) + self.assertIn("TraceLifecycle(\"after aclrtSynchronizeStream\")", source) + + def test_submit_task_selector_can_isolate_prepared_task_hangs(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SUBMIT_TASK_SELECTOR", source) + self.assertIn("SubmitPreparedWithSelector", source) + self.assertIn("TileXRDirectCcuGetPreparedTask", source) + self.assertIn("TileXRDirectCcuSubmitPreparedTask", source) + self.assertIn('selector == "first"', source) + self.assertIn('selector == "second"', source) + self.assertIn("submitTaskSelector=", source) + self.assertIn("TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport)", prepared_body) + self.assertIn("SubmitPreparedWithSelector(prepared, installReport.submitTaskCount, stream", prepared_body) + + def test_process_mode_submit_uses_collective_ready_gate(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + runner = (REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh").read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_READY_DIR", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS", source) + self.assertIn("WaitForCollectiveSubmitReadiness", source) + self.assertIn("WriteSubmitReadiness", source) + self.assertIn("collectiveSubmitReady", prepared_body) + self.assertIn("collective submitReady gate did not pass", source) + self.assertLess( + prepared_body.index("WaitForCollectiveSubmitReadiness"), + prepared_body.index("aclrtCreateStream"), + ) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_READY_DIR=${ready_dir}", runner) + self.assertIn("rm -rf \"${ready_dir}\"", runner) + + def test_process_mode_submit_waits_for_all_ranks_before_cleanup(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + runner = (REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh").read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DONE_DIR", source) + self.assertIn("WaitForCollectiveSubmitDone", source) + self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet)", prepared_body) + self.assertIn("collectiveSubmitDone", source) + self.assertLess( + prepared_body.index("aclrtSynchronizeStream"), + prepared_body.index("WaitForCollectiveSubmitDone"), + ) + self.assertLess( + prepared_body.index("WaitForCollectiveSubmitDone"), + prepared_body.index("TileXRDirectCcuDestroyPrepared"), + ) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DONE_DIR=${done_dir}", runner) + self.assertIn("rm -rf \"${ready_dir}\" \"${done_dir}\"", runner) + + def test_probe_wires_gsa_and_split_cke_env_into_prepare_options_and_config_trace(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "TILEXR_CCU_PROBE_GSA_START", + "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START", + "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT", + "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START", + "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT", + "options.localWaitCkeStartId", + "options.localWaitCkeCount", + "options.remoteNotifyCkeStartId", + "options.remoteNotifyCkeCount", + "options.repositoryInstallWindow", + "options.repositoryInstallDataLenMode", + "options.repositoryMemoryAllocMode", + "options.installOrder", + "options.sqeArgCount", + "options.missionInstructionStartId", + "options.gsaStartId", + "sqeArgCount=", + "gsaStartId=", + "localWaitCkeStartId=", + "localWaitCkeCount=", + "remoteNotifyCkeStartId=", + "remoteNotifyCkeCount=", + "repositoryInstallWindow=", + "repositoryInstallDataLenMode=", + "repositoryMemoryAllocMode=", + "installOrder=", + "missionInstructionStartId=", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + def test_probe_prepared_task_trace_prints_full_runtime_args(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepared_body = source[ + source.index("void PrintPreparedTasks"): + source.index("int RunPreparedSmokeForRank") + ] + + self.assertIn("TILEXR_DIRECT_CCU_SQE_ARGS_LEN", prepared_body) + self.assertIn("arg < TILEXR_DIRECT_CCU_SQE_ARGS_LEN", prepared_body) + self.assertIn('<< ".arg" << arg << "=0x"', prepared_body) + self.assertIn("task.args[arg]", prepared_body) + self.assertNotIn(".arg0=0x", prepared_body) + + def test_probe_defaults_to_lower_layer_first_install_order(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + install_order_body = source[ + source.index("uint32_t InstallOrderFromEnv()"): + source.index("uint32_t DefaultSyncInstructionCount") + ] + + self.assertIn("TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST", install_order_body) + self.assertIn("repository_first", install_order_body) + self.assertIn("TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST", install_order_body) + self.assertLess( + install_order_body.index("TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST"), + install_order_body.index("TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST"), + ) + + def test_probe_default_sync_instruction_count_includes_hcomm_style_task1_prelude(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + default_body = source[ + source.index("uint32_t DefaultSyncInstructionCount"): + source.index("const char* FirstEnv") + ] + + self.assertIn("kHcommStyleTask1PreludeInstructionCount", source) + self.assertIn( + "return kHcommStyleTask1PreludeInstructionCount + syncResourceCount;", + default_body, + ) + self.assertIn( + "return kHcommStyleTask1PreludeInstructionCount + syncResourceCount * 2U;", + default_body, + ) + + def test_thread_mode_rank_specific_resource_env_overrides_common_prepare_options(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepare_options_body = source[ + source.index("TileXRDirectCcuPrepareOptions MakePrepareOptions"): + source.index("void PrintInstallReport") + ] + + self.assertIn("RankEnvInt", prepare_options_body) + for suffix in [ + "XN_START", + "LOCAL_WAIT_CKE_START", + "LOCAL_WAIT_CKE_COUNT", + "REMOTE_NOTIFY_CKE_START", + "REMOTE_NOTIFY_CKE_COUNT", + ]: + with self.subTest(suffix=suffix): + self.assertIn(f"TILEXR_CCU_PROBE_RANK\", rank, \"_{suffix}", source) + self.assertIn( + 'RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_XN_START", "TILEXR_CCU_PROBE_XN_START", 1)', + prepare_options_body, + ) + self.assertIn( + 'RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_LOCAL_WAIT_CKE_START", kLocalWaitCkeStartEnv, 0)', + prepare_options_body, + ) + + def test_default_skip_happens_before_acl_comm_prepare_or_submit(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + main_source = source[source.index("int main()"):] + + guard_pos = main_source.index("if (!EnvFlag(kEnableEnv))") + for needle in [ + "aclInit(", + "InitCommForDirectCcuSmoke", + "RunPreparedSmokeForRank", + ]: + with self.subTest(needle=needle): + self.assertLess(guard_pos, main_source.index(needle)) + + def test_probe_keeps_hcomm_hccl_and_runtime_launch_out_of_source(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + for needle in PRIVATE_CCU_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, source) + self.assertNotIn("runtime/kernel.h", source) + self.assertNotIn("rtCCULaunch", source) + + def test_probe_compiles_and_default_run_skips_without_touching_hardware(self): + temp_dir, probe_bin, tile_comm_dir, cann_lib_dir, driver_lib_dir = self.compile_probe() + try: + env = os.environ.copy() + env.pop("TILEXR_CCU_DIRECT_SMOKE_ENABLE", None) + env["LD_LIBRARY_PATH"] = ( + str(tile_comm_dir) + + os.pathsep + + str(cann_lib_dir) + + os.pathsep + + str(driver_lib_dir) + + os.pathsep + + env.get("LD_LIBRARY_PATH", "") + ) + result = subprocess.run( + [str(probe_bin)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + finally: + temp_dir.cleanup() + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_smoke skipped", result.stdout) + self.assertNotIn("prepare ret=", result.stdout) + self.assertNotIn("submit ret=", result.stdout) + + def test_optional_probe_runtime_when_enabled(self): + if os.environ.get("TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE") != "1": + self.skipTest("set TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 to run direct CCU smoke on hardware") + rank_size = ( + os.environ.get("TILEXR_CCU_PROBE_RANK_SIZE") + or os.environ.get("PMI_SIZE") + or os.environ.get("OMPI_COMM_WORLD_SIZE") + or os.environ.get("MV2_COMM_WORLD_SIZE") + or os.environ.get("RANK_SIZE") + ) + if rank_size is None or int(rank_size) <= 1: + self.skipTest("direct CCU prepare smoke requires a real multi-rank TileXRComm launch") + temp_dir, probe_bin, tile_comm_dir, cann_lib_dir, driver_lib_dir = self.compile_probe() + try: + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_SMOKE_ENABLE"] = "1" + env["LD_LIBRARY_PATH"] = ( + str(tile_comm_dir) + + os.pathsep + + str(cann_lib_dir) + + os.pathsep + + str(driver_lib_dir) + + os.pathsep + + env.get("LD_LIBRARY_PATH", "") + ) + result = subprocess.run( + [str(probe_bin)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + finally: + temp_dir.cleanup() + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_smoke prepare ret=0", result.stdout) + self.assertIn("submitReady=", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py new file mode 100644 index 00000000..94c31cad --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -0,0 +1,1399 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +RUNNER = REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh" +EVENING_RUNNER = REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_evening_smoke.sh" +BUSY_GUARD = REPO_ROOT / "tests" / "ccu" / "ccu_npu_smi_busy_guard.py" + + +class TileXRCcuDirectSmokeRunnerTest(unittest.TestCase): + def test_runner_is_default_safe_and_documents_hardware_gate(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ENABLE=1", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SUBMIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES", source) + self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_ORDER", source) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", source) + self.assertIn("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START", source) + self.assertIn("TILEXR_CCU_PROBE_SQE_ARG_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_MISSION_START", source) + self.assertIn("TILEXR_CCU_PROBE_INSTRUCTION_START", source) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_BINDINGS_PER_RESOURCE", source) + self.assertIn("TILEXR_CCU_PROBE_CHANNEL_START", source) + self.assertIn("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START", source) + self.assertIn("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START", source) + self.assertIn("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_RANK0_XN_START", source) + self.assertIn("TILEXR_CCU_PROBE_RANK1_XN_START", source) + self.assertIn("TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START", source) + self.assertIn("TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START", source) + for suffix in [ + "EID", + "TPN", + "DOORBELL_VA", + "DOORBELL_TOKEN_ID", + "DOORBELL_TOKEN_VALUE", + "SQ_DEPTH", + ]: + with self.subTest(endpoint_suffix=suffix): + self.assertIn(f" {suffix}", source) + self.assertIn('endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}"', source) + self.assertIn('rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0"', source) + self.assertIn('rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1"', source) + self.assertIn('rank0_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}")', source) + self.assertIn('rank1_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}")', source) + self.assertIn("rank0_env", source) + self.assertIn("rank1_env", source) + self.assertIn("TILEXR_COMM_ID", source) + self.assertIn("TILEXR_CCU_PROBE_RANK_SIZE=2", source) + self.assertIn("TILEXR_CCU_PROBE_RANK=0", source) + self.assertIn("TILEXR_CCU_PROBE_RANK=1", source) + self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]', source) + self.assertIn('"${probe_bin}" > "${thread_log}" 2>&1', source) + self.assertIn("ccu_thread.log", source) + self.assertIn("ccu_rank0.log", source) + self.assertIn("ccu_rank1.log", source) + self.assertIn("installSucceeded=1", source) + self.assertIn("submitReady=1", source) + self.assertLess(source.index("installSucceeded=1"), source.index("submitReady=1")) + self.assertIn("${repo_root}/install/lib64/libtile-comm.so", source) + self.assertIn( + 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0', + source, + ) + self.assertIn( + 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1', + source, + ) + self.assertNotIn('bash -c "wait', source) + self.assertIn("npu-smi rc=", source) + self.assertIn("TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU", source) + self.assertIn("TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU", source) + self.assertIn("--allow-unhealthy", source) + self.assertIn("TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI", source) + self.assertIn("ccu_npu_smi_busy_guard.py", source) + self.assertIn("tilexr_ccu_direct_smoke_runner summary", source) + self.assertIn("rank0Status=", source) + self.assertIn("rank1Status=", source) + self.assertIn("rank0Log=", source) + self.assertIn("rank1Log=", source) + self.assertIn("submitTiming", source) + self.assertIn("syncMs=", source) + self.assertIn("p2pCcuCopy", source) + self.assertIn("passed=1", source) + self.assertIn('"TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE:-1}"', source) + + gate = source.index('if [ "${TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE:-0}" != "1" ]') + for needle in [ + "npu-smi info", + '"${probe_bin}"', + "TILEXR_CCU_DIRECT_SMOKE_ENABLE=1", + "TILEXR_CCU_PROBE_RANK=0", + "TILEXR_CCU_PROBE_RANK=1", + ]: + with self.subTest(needle=needle): + self.assertLess(gate, source.index(needle)) + + def test_runner_default_run_skips_without_hardware(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env.pop("TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE", None) + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("skipped", result.stdout) + self.assertNotIn("TILEXR_CCU_PROBE_RANK=0", result.stdout) + self.assertNotIn("ccu_rank0.log", result.stdout) + + def test_runner_defaults_prepare_failure_fast_exit_and_allows_opt_out(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn( + '"TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE:-1}"', + source, + ) + self.assertNotIn("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE=0", source) + + def test_runner_dry_run_shows_rank_specific_endpoint_overrides_common(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE"] = "1" + env["TILEXR_CCU_DIRECT_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN"] = "99" + env["TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN_RANK0"] = "100" + env["TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN_RANK1"] = "200" + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_smoke_runner dryRun=1", result.stdout) + self.assertIn("dryRun rank0 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN=100", result.stdout) + self.assertIn("dryRun rank1 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN=200", result.stdout) + self.assertNotIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN=99", result.stdout) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + self.assertNotIn("ccu_rank0.log", result.stdout) + + def test_runner_dry_run_shows_rank_specific_resource_window_token_overrides_common(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE"] = "1" + env["TILEXR_CCU_DIRECT_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID"] = "0x1111" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK0"] = "0x2222" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK1"] = "0x3333" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID_RANK1"] = "0x4444" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_RANK1"] = "0x5555" + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_smoke_runner dryRun=1", result.stdout) + self.assertIn("dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID=0x2222", result.stdout) + self.assertIn("dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID=0x3333", result.stdout) + self.assertIn("dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID=0x4444", result.stdout) + self.assertIn("dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE=0x5555", result.stdout) + self.assertNotIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID=0x1111", result.stdout) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + self.assertNotIn("ccu_rank0.log", result.stdout) + + def test_runner_dry_run_shows_repository_install_diagnostic_variants(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE"] = "1" + env["TILEXR_CCU_DIRECT_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW"] = "full_repository" + env["TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE"] = "descriptor_bytes" + env["TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE"] = "acl_hccl_module" + env["TILEXR_CCU_DIRECT_INSTALL_ORDER"] = "lower_layer_first" + env["TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START"] = "489" + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("dryRun TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW=full_repository", result.stdout) + self.assertIn("dryRun TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE=descriptor_bytes", result.stdout) + self.assertIn("dryRun TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE=acl_hccl_module", result.stdout) + self.assertIn("dryRun TILEXR_CCU_DIRECT_INSTALL_ORDER=lower_layer_first", result.stdout) + self.assertIn("dryRun TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START=489", result.stdout) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + self.assertNotIn("ccu_rank0.log", result.stdout) + + def test_runner_dry_run_defaults_to_lower_layer_first_install_order(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE"] = "1" + env["TILEXR_CCU_DIRECT_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env.pop("TILEXR_CCU_DIRECT_INSTALL_ORDER", None) + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("dryRun TILEXR_CCU_DIRECT_INSTALL_ORDER=lower_layer_first", result.stdout) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + + def test_runner_dry_run_defaults_sync_xn_post_only_window_for_hcomm_style_task1_prelude(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE"] = "1" + env["TILEXR_CCU_DIRECT_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_BARRIER_MODE"] = "sync_xn_post_only" + env["TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT"] = "1" + env["TILEXR_CCU_PROBE_INSTRUCTION_START"] = "475" + env["TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START"] = "489" + env.pop("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT", None) + result = subprocess.run( + ["bash", str(RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("dryRun TILEXR_CCU_DIRECT_BARRIER_MODE=sync_xn_post_only", result.stdout) + self.assertIn("dryRun derived task0.instStartId=489 task0.instCnt=13", result.stdout) + self.assertIn("dryRun derived task1.instStartId=502 task1.instCnt=6", result.stdout) + self.assertIn( + "dryRun derived repositoryStartId=475 repositoryCount=33 missionInstructionStartId=489 " + "missionInstructionCount=19", + result.stdout, + ) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + + def test_npu_busy_guard_rejects_selected_device_processes(self): + sample = """ ++------------------+---------------+--------------+------------------+ +| NPU Chip | Process id | Process name | Process memory | ++==================+===============+==============+==================+ +| 0 0 | 31415 | python3.10 | 1024 | +| 2 0 | 27182 | train.py | 2048 | ++------------------+---------------+--------------+------------------+ +""" + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "npu-smi.log" + log_path.write_text(sample, encoding="utf-8") + busy = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "0,1"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + idle = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "1"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + + self.assertEqual(1, busy.returncode, busy.stdout + busy.stderr) + self.assertIn("device=0 pid=31415 process=python3.10", busy.stdout) + self.assertEqual(0, idle.returncode, idle.stdout + idle.stderr) + self.assertIn("no selected NPU processes", idle.stdout) + + def test_npu_busy_guard_ignores_device_status_table(self): + sample = """ ++------+-------------+--------+-------------+ +| NPU | Name | Health | Power(W) | ++======+=============+========+=============+ +| 0 | Ascend950PR | OK | 95 | +| 1 | Ascend950PR | OK | 93 | ++------+-------------+--------+-------------+ +""" + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "npu-smi.log" + log_path.write_text(sample, encoding="utf-8") + result = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "0,1"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("no selected NPU processes", result.stdout) + + def test_npu_busy_guard_rejects_selected_alarm_devices(self): + sample = """ ++------+-------------+--------+-------------+ +| NPU | Name | Health | Power(W) | ++======+=============+========+=============+ +| 0 | Ascend950PR | Alarm | 95 | +| 1 | Ascend950PR | OK | 93 | +| 2 | Ascend950PR | Alarm | 94 | ++------+-------------+--------+-------------+ +""" + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "npu-smi.log" + log_path.write_text(sample, encoding="utf-8") + unhealthy = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "0,1"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + healthy = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "1"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + + self.assertEqual(1, unhealthy.returncode, unhealthy.stdout + unhealthy.stderr) + self.assertIn("unhealthy selected NPU device=0 health=Alarm", unhealthy.stdout) + self.assertNotIn("device=2", unhealthy.stdout) + self.assertEqual(0, healthy.returncode, healthy.stdout + healthy.stderr) + self.assertIn("selected NPU devices healthy", healthy.stdout) + + def test_npu_busy_guard_can_allow_alarm_without_allowing_busy_processes(self): + sample = """ ++------+-------------+--------+-------------+ +| NPU | Name | Health | Power(W) | ++======+=============+========+=============+ +| 0 | Ascend950PR | Alarm | 95 | +| 1 | Ascend950PR | OK | 93 | +| 2 | Ascend950PR | Alarm | 94 | ++------+-------------+--------+-------------+ +| NPU Chip | Process id | Process name | Process memory | ++==================+===============+==============+==================+ +| 0 0 | 31415 | python3.10 | 1024 | ++------+-------------+--------+-------------+ +""" + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "npu-smi.log" + log_path.write_text(sample, encoding="utf-8") + allowed_health = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "2", "--allow-unhealthy"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + busy_still_blocked = subprocess.run( + ["python", str(BUSY_GUARD), "--log", str(log_path), "--devices", "0", "--allow-unhealthy"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + + self.assertEqual(0, allowed_health.returncode, allowed_health.stdout + allowed_health.stderr) + self.assertIn("unhealthy selected NPU devices allowed", allowed_health.stdout) + self.assertNotIn("selected NPU devices healthy", allowed_health.stdout) + self.assertEqual(1, busy_still_blocked.returncode, busy_still_blocked.stdout + busy_still_blocked.stderr) + self.assertIn("device=0 pid=31415 process=python3.10", busy_still_blocked.stdout) + + def test_runner_requires_submit_for_barrier_wait_expectation(self): + source = RUNNER.read_text(encoding="utf-8") + barrier_gate = source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]') + submit_guard = source.index( + 'if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then', + barrier_gate, + ) + timing_parse = source.index("wait_sync_ms=", barrier_gate) + + self.assertLess(barrier_gate, submit_guard) + self.assertLess(submit_guard, timing_parse) + self.assertIn("direct CCU barrier wait check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1", source) + + def test_evening_wrapper_defaults_to_process_mode_direct_ccu_only_prepare_first(self): + source = EVENING_RUNNER.read_text(encoding="utf-8") + + self.assertIn('--dry-run', source) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN", source) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES", source) + self.assertIn("source scripts/common_env.sh", source) + self.assertIn("cmake --build build --target tile-comm", source) + self.assertIn("TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1", source) + self.assertIn("TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI=1", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=0", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SUBMIT", source) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT", source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1', source) + self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE=sync_cke", source) + self.assertIn("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE=hcomm_cap", source) + self.assertIn("TILEXR_CCU_PROBE_SQE_ARG_COUNT", source) + self.assertIn("TILEXR_CCU_PROBE_MISSION_START", source) + self.assertIn("TILEXR_CCU_PROBE_INSTRUCTION_START", source) + self.assertIn("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START", source) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE", source) + self.assertIn("TILEXR_CCU_PROBE_RANK0_XN_START", source) + self.assertIn("TILEXR_CCU_PROBE_RANK1_XN_START", source) + self.assertIn(":-475", source) + self.assertIn(":-489", source) + self.assertIn(":-143", source) + self.assertIn(":-full_repository", source) + self.assertIn(":-instruction_bytes", source) + self.assertIn(":-acl", source) + self.assertIn("tests/ccu/run_tilexr_ccu_direct_smoke.sh", source) + self.assertIn('run_smoke_stage "prepare_${safe_prepare_mode}"', source) + self.assertIn("prepare_has_submit_ready", source) + self.assertIn("submitReady=1", source) + self.assertIn("run_smoke_stage submit", source) + self.assertIn("run_smoke_stage barrier", source) + self.assertIn("run_smoke_stage p2p", source) + self.assertLess(source.index('run_smoke_stage "prepare_${safe_prepare_mode}"'), source.index("run_smoke_stage submit")) + self.assertLess(source.index("prepare_has_submit_ready"), source.index("run_smoke_stage submit")) + + def test_evening_wrapper_dry_run_short_circuits_before_common_env(self): + source = EVENING_RUNNER.read_text(encoding="utf-8") + + dry_run_gate = source.index('if [ "${TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN:-0}" = "1" ]') + common_env_source = source.index("source scripts/common_env.sh") + cmake_build = source.index("cmake --build build --target tile-comm") + + self.assertLess(dry_run_gate, common_env_source) + self.assertLess(dry_run_gate, cmake_build) + + def test_evening_wrapper_dry_run_does_not_invoke_fake_npu_smi(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + marker = temp_path / "npu-smi-called" + fake_npu_smi = temp_path / "npu-smi" + fake_npu_smi.write_text( + f"#!/usr/bin/env bash\n" + f"echo called >> {str(marker).replace(os.sep, '/')!r}\n" + f"exit 0\n", + encoding="utf-8", + ) + fake_npu_smi.chmod(0o755) + + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN"] = "1" + env["PATH"] = str(temp_path) + os.pathsep + env.get("PATH", "") + result = subprocess.run( + ["bash", str(EVENING_RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_evening_smoke dryRun=1", result.stdout) + self.assertFalse(marker.exists(), result.stdout + result.stderr) + + def test_evening_wrapper_retries_prepare_with_module3_allocator_before_submit(self): + source = EVENING_RUNNER.read_text(encoding="utf-8") + + self.assertIn('TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES:-acl,acl_module3,rt_hbm', source) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES", source) + self.assertIn('prepare_alloc_modes', source) + self.assertIn('prepare_${safe_prepare_mode}', source) + self.assertIn('TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${prepare_profile_alloc}"', source) + self.assertIn('TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${prepare_profile_window}"', source) + self.assertIn('TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${prepare_profile_data_len_mode}"', source) + self.assertIn('TILEXR_CCU_DIRECT_INSTALL_ORDER="${prepare_profile_install_order}"', source) + self.assertIn('selected_prepare_alloc_mode', source) + self.assertIn('selected_prepare_window', source) + self.assertIn('prepareStatus=', source) + self.assertIn('prepare_status_summary', source) + self.assertIn('selectedPrepare alloc=', source) + self.assertIn('stopAfter=prepare reason="submitReady=1 missing for every prepare profile"', source) + + prepare_loop = source.index('for prepare_profile in "${prepare_profiles[@]}"') + self.assertLess( + source.index('TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${prepare_profile_alloc}"', prepare_loop), + source.index("prepare_has_submit_ready", prepare_loop), + ) + self.assertLess(source.index("selected_prepare_alloc_mode"), source.index("run_smoke_stage submit")) + submit_stage = source.index("run_smoke_stage submit") + self.assertLess( + submit_stage, + source.index('TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}"', submit_stage), + ) + self.assertLess( + submit_stage, + source.index('TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${selected_prepare_window}"', submit_stage), + ) + barrier_stage = source.index("run_smoke_stage barrier") + self.assertLess( + barrier_stage, + source.index('TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}"', barrier_stage), + ) + p2p_stage = source.index("run_smoke_stage p2p") + self.assertLess( + p2p_stage, + source.index('TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${selected_prepare_alloc_mode}"', p2p_stage), + ) + + def test_evening_wrapper_dry_run_lists_prepare_profiles(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES"] = ( + "full_acl:acl:full_repository:instruction_bytes:lower_layer_first," + "mission_desc:acl:mission:descriptor_bytes:repository_first" + ) + result = subprocess.run( + ["bash", str(EVENING_RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn( + "TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES=" + "full_acl:acl:full_repository:instruction_bytes:lower_layer_first," + "mission_desc:acl:mission:descriptor_bytes:repository_first", + result.stdout, + ) + self.assertIn( + "dryRun prepareProfile[0] name=full_acl alloc=acl window=full_repository " + "dataLenMode=instruction_bytes installOrder=lower_layer_first", + result.stdout, + ) + self.assertIn( + "dryRun prepareProfile[1] name=mission_desc alloc=acl window=mission " + "dataLenMode=descriptor_bytes installOrder=repository_first", + result.stdout, + ) + + def test_evening_wrapper_prepare_profiles_pass_repository_options_to_runner(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cmake = fake_bin / "cmake" + fake_cmake.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + fake_cmake.chmod(0o755) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + + fake_repo = temp_path / "repo" + fake_tests = fake_repo / "tests" / "ccu" + fake_scripts = fake_repo / "scripts" + fake_tests.mkdir(parents=True) + fake_scripts.mkdir() + (fake_scripts / "common_env.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + fake_runner = fake_tests / "run_tilexr_ccu_direct_smoke.sh" + fake_runner.write_text( + "#!/usr/bin/env bash\n" + "mkdir -p \"${TILEXR_CCU_SMOKE_WORK_DIR}\"\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" < \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" < \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" < \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" < \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submit ret=0 submitted=1 taskCount=1 submittedTaskCount=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submitTiming rank=0 preSubmitDelayMs=0 submitRet=0 syncRet=0 submitMs=1 syncMs=150\n" + "tilexr_ccu_direct_smoke p2pCcuCopy rank=0 passed=1\n" + "LOG\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submit ret=0 submitted=1 taskCount=1 submittedTaskCount=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submitTiming rank=1 preSubmitDelayMs=0 submitRet=0 syncRet=0 submitMs=1 syncMs=160\n" + "tilexr_ccu_direct_smoke p2pCcuCopy rank=1 passed=1\n" + "LOG\n" + "exit 0\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + fake_evening = fake_tests / "run_tilexr_ccu_direct_evening_smoke.sh" + fake_evening.write_text(EVENING_RUNNER.read_text(encoding="utf-8"), encoding="utf-8") + fake_evening.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES"] = "prepare,submit,barrier,p2p" + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_PROFILES"] = ( + "pfe_hcomm:acl:full_repository:instruction_bytes:lower_layer_first:hcomm_die:hcomm_fe_id" + ) + env["TILEXR_CCU_EVENING_WORK_ROOT"] = "work" + result = subprocess.run( + ["bash", str(fake_evening)], + cwd=fake_repo, + check=False, + text=True, + capture_output=True, + env=env, + ) + final_status = fake_repo / "work" / "final_status.log" + final_status_exists = final_status.exists() + final_status_text = final_status.read_text(encoding="utf-8") if final_status_exists else "" + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn( + "finalStatus prepare=pass submit=pass barrier=pass p2p=pass completionCandidate=1 failedStage=none selectedProfile=pfe_hcomm", + result.stdout, + ) + self.assertTrue(final_status_exists, result.stdout + result.stderr) + self.assertIn( + "finalStatus prepare=pass submit=pass barrier=pass p2p=pass completionCandidate=1 failedStage=none selectedProfile=pfe_hcomm", + final_status_text, + ) + self.assertIn("selectedPrepare", result.stdout) + self.assertIn("success workRoot=work", result.stdout) + + def test_evening_wrapper_five_field_profile_preserves_ambient_pfe_layout(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cmake = fake_bin / "cmake" + fake_cmake.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + fake_cmake.chmod(0o755) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + + fake_repo = temp_path / "repo" + fake_tests = fake_repo / "tests" / "ccu" + fake_scripts = fake_repo / "scripts" + fake_tests.mkdir(parents=True) + fake_scripts.mkdir() + (fake_scripts / "common_env.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + fake_runner = fake_tests / "run_tilexr_ccu_direct_smoke.sh" + fake_runner.write_text( + "#!/usr/bin/env bash\n" + "mkdir -p \"${TILEXR_CCU_SMOKE_WORK_DIR}\"\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" < \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke config rank=0 installOrder=1\n" + "tilexr_ccu_direct_smoke prepare ret=6 submitReady=0 message=\"failed to install CCU repository instruction image: CCU custom channel call failed op=251 driverRet=7 opRet=9\"\n" + "tilexr_ccu_direct_smoke preparedTasks count=0\n" + "LOG\n" + "exit 6\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + fake_evening = fake_tests / "run_tilexr_ccu_direct_evening_smoke.sh" + fake_evening.write_text(EVENING_RUNNER.read_text(encoding="utf-8"), encoding="utf-8") + fake_evening.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES"] = "prepare" + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES"] = "acl" + env["TILEXR_CCU_EVENING_WORK_ROOT"] = "work" + result = subprocess.run( + ["bash", str(fake_evening)], + cwd=fake_repo, + check=False, + text=True, + capture_output=True, + env=env, + ) + final_status = fake_repo / "work" / "final_status.log" + final_status_exists = final_status.exists() + final_status_text = final_status.read_text(encoding="utf-8") if final_status_exists else "" + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("prepareLogSummary mode=acl status=6", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke prepare ret=6 submitReady=0", result.stdout) + self.assertIn("CCU custom channel call failed op=251 driverRet=7 opRet=9", result.stdout) + self.assertIn('stopAfter=prepare reason="submitReady=1 missing for every prepare profile"', result.stdout) + self.assertIn( + "finalStatus prepare=fail submit=skipped barrier=skipped p2p=skipped " + "completionCandidate=0 failedStage=prepare", + result.stdout, + ) + self.assertTrue(final_status_exists, result.stdout + result.stderr) + self.assertIn( + "finalStatus prepare=fail submit=skipped barrier=skipped p2p=skipped " + "completionCandidate=0 failedStage=prepare", + final_status_text, + ) + + def test_evening_wrapper_extracts_submit_stage_summary_from_logs(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cmake = fake_bin / "cmake" + fake_cmake.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + fake_cmake.chmod(0o755) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + + fake_repo = temp_path / "repo" + fake_tests = fake_repo / "tests" / "ccu" + fake_scripts = fake_repo / "scripts" + fake_tests.mkdir(parents=True) + fake_scripts.mkdir() + (fake_scripts / "common_env.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + fake_runner = fake_tests / "run_tilexr_ccu_direct_smoke.sh" + fake_runner.write_text( + "#!/usr/bin/env bash\n" + "mkdir -p \"${TILEXR_CCU_SMOKE_WORK_DIR}\"\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "tilexr_ccu_direct_smoke submit ret=0 submitted=1 taskCount=1 submittedTaskCount=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submitTiming rank=0 preSubmitDelayMs=0 submitRet=0 syncRet=0 submitMs=1 syncMs=7\n" + "LOG\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "tilexr_ccu_direct_smoke submit ret=0 submitted=1 taskCount=1 submittedTaskCount=1 message=\"ok\"\n" + "tilexr_ccu_direct_smoke submitTiming rank=1 preSubmitDelayMs=0 submitRet=0 syncRet=0 submitMs=1 syncMs=8\n" + "LOG\n" + "exit 0\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + fake_evening = fake_tests / "run_tilexr_ccu_direct_evening_smoke.sh" + fake_evening.write_text(EVENING_RUNNER.read_text(encoding="utf-8"), encoding="utf-8") + fake_evening.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES"] = "prepare,submit" + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES"] = "acl" + env["TILEXR_CCU_EVENING_WORK_ROOT"] = "work" + result = subprocess.run( + ["bash", str(fake_evening)], + cwd=fake_repo, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("stageLogSummary stage=submit status=0", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke submit ret=0 submitted=1", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke submitTiming rank=1", result.stdout) + + def test_evening_wrapper_runs_p2p_ccu_copy_with_process_mode_direct_ccu_init(self): + source = EVENING_RUNNER.read_text(encoding="utf-8") + p2p_stage = source[source.index("if stage_enabled p2p;"):] + p2p_stage = p2p_stage[:p2p_stage.index("print_final_status none")] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1", p2p_stage) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY=1", p2p_stage) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=1", p2p_stage) + self.assertNotIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1", p2p_stage) + self.assertNotIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=0", p2p_stage) + + def test_evening_wrapper_keeps_submit_failure_diagnostics_before_long_trace_tail(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cmake = fake_bin / "cmake" + fake_cmake.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + fake_cmake.chmod(0o755) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + + fake_repo = temp_path / "repo" + fake_tests = fake_repo / "tests" / "ccu" + fake_scripts = fake_repo / "scripts" + fake_tests.mkdir(parents=True) + fake_scripts.mkdir() + (fake_scripts / "common_env.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + fake_runner = fake_tests / "run_tilexr_ccu_direct_smoke.sh" + fake_runner.write_text( + "#!/usr/bin/env bash\n" + "mkdir -p \"${TILEXR_CCU_SMOKE_WORK_DIR}\"\n" + "if [[ \"${TILEXR_CCU_SMOKE_WORK_DIR}\" != *submit* ]]; then\n" + " cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "LOG\n" + " cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "LOG\n" + " exit 0\n" + "fi\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "tilexr_ccu_direct_smoke submit ret=-2 submitted=0 taskCount=1 submittedTaskCount=0 message=\"direct CCU submit failed task=0 ret=-2 rtRet=507000 dieId=1 missionId=6 instStartId=489 instCnt=156 key=0x59b0f03 argSize=13 args[0]=0xfeed\"\n" + "LOG\n" + "for i in $(seq 1 40); do\n" + " echo \"TileXRDirectCcuTrace program.sync[$i] decoded=SyncXn\" >> \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\"\n" + "done\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "LOG\n" + "exit 7\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + fake_evening = fake_tests / "run_tilexr_ccu_direct_evening_smoke.sh" + fake_evening.write_text(EVENING_RUNNER.read_text(encoding="utf-8"), encoding="utf-8") + fake_evening.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES"] = "prepare,submit" + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES"] = "acl" + env["TILEXR_CCU_EVENING_WORK_ROOT"] = "work" + result = subprocess.run( + ["bash", str(fake_evening)], + cwd=fake_repo, + check=False, + text=True, + capture_output=True, + env=env, + ) + final_status = fake_repo / "work" / "final_status.log" + final_status_exists = final_status.exists() + final_status_text = final_status.read_text(encoding="utf-8") if final_status_exists else "" + + self.assertEqual(7, result.returncode, result.stdout + result.stderr) + self.assertIn("stageLogSummary stage=submit status=7", result.stdout) + self.assertIn("direct CCU submit failed task=0", result.stdout) + self.assertIn("rtRet=507000", result.stdout) + self.assertIn("args[0]=0xfeed", result.stdout) + self.assertIn( + "finalStatus prepare=pass submit=fail barrier=skipped p2p=skipped completionCandidate=0 failedStage=submit", + result.stdout, + ) + self.assertTrue(final_status_exists, result.stdout + result.stderr) + self.assertIn( + "finalStatus prepare=pass submit=fail barrier=skipped p2p=skipped completionCandidate=0 failedStage=submit", + final_status_text, + ) + + def test_evening_wrapper_stage_summary_keeps_decoded_direct_trace_lines(self): + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cmake = fake_bin / "cmake" + fake_cmake.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + fake_cmake.chmod(0o755) + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + "shift\n" + "exec \"$@\"\n", + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + + fake_repo = temp_path / "repo" + fake_tests = fake_repo / "tests" / "ccu" + fake_scripts = fake_repo / "scripts" + fake_tests.mkdir(parents=True) + fake_scripts.mkdir() + (fake_scripts / "common_env.sh").write_text("#!/usr/bin/env bash\n", encoding="utf-8") + fake_runner = fake_tests / "run_tilexr_ccu_direct_smoke.sh" + fake_runner.write_text( + "#!/usr/bin/env bash\n" + "mkdir -p \"${TILEXR_CCU_SMOKE_WORK_DIR}\"\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank0.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "TileXRDirectCcuTrace lowerLayerPfe[0] decoded=PfeCtx pfeOffset=18 startTaJettyId=1024\n" + "TileXRDirectCcuTrace lowerLayerJettyCtx[0,0] decoded=LocalJettyCtx inferredSqDepth=64 wqeBasicBlockStartId=0\n" + "TileXRDirectCcuTrace lowerLayerChannel[0] decoded=ChannelCtxV1 sourcePfeId=2 remoteCcuVa=0x12340000\n" + "TileXRDirectCcuTrace remoteXnBinding[0] localXn=1961 remoteXn=2361 endpointRouteVerified=1\n" + "TileXRDirectCcuTrace task[0] missionId=6 instStartId=489 instCnt=13 argSize=13\n" + "TileXRDirectCcuTrace task[1] missionId=6 instStartId=502 instCnt=143 argSize=13\n" + "TileXRDirectCcuTrace finalRuntimeTask[0] dieId=1 missionId=6 timeout=20 instStartId=489 instCnt=2 key=0x59b0f03 argSize=1 args[0]=0xabc000\n" + "TileXRDirectCcuTrace customChannel.return op=251 driverRet=328107 opRet=0\n" + "LOG\n" + "cat > \"${TILEXR_CCU_SMOKE_WORK_DIR}/ccu_rank1.log\" <<'LOG'\n" + "tilexr_ccu_direct_smoke prepare ret=0 installSucceeded=1 submitReady=1\n" + "LOG\n" + "exit 0\n", + encoding="utf-8", + ) + fake_runner.chmod(0o755) + fake_evening = fake_tests / "run_tilexr_ccu_direct_evening_smoke.sh" + fake_evening.write_text(EVENING_RUNNER.read_text(encoding="utf-8"), encoding="utf-8") + fake_evening.chmod(0o755) + + env = os.environ.copy() + env["PATH"] = str(fake_bin) + os.pathsep + env.get("PATH", "") + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES"] = "prepare" + env["TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES"] = "acl" + env["TILEXR_CCU_EVENING_WORK_ROOT"] = "work" + result = subprocess.run( + ["bash", str(fake_evening)], + cwd=fake_repo, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("decoded=PfeCtx", result.stdout) + self.assertIn("decoded=LocalJettyCtx", result.stdout) + self.assertIn("decoded=ChannelCtxV1", result.stdout) + self.assertIn("remoteXnBinding[0]", result.stdout) + self.assertIn("TileXRDirectCcuTrace task[0]", result.stdout) + self.assertIn("TileXRDirectCcuTrace finalRuntimeTask[0]", result.stdout) + self.assertIn("TileXRDirectCcuTrace customChannel.return op=251", result.stdout) + + def test_evening_wrapper_dry_run_does_not_touch_npu(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_EVENING_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK0"] = "0x2222" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK1"] = "0x3333" + result = subprocess.run( + ["bash", str(EVENING_RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_evening_smoke dryRun=1", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DRY_RUN=1", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_SMOKE_STAGES=prepare,submit,barrier,p2p", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_EVENING_PREPARE_ALLOC_MODES=acl,acl_module3,rt_hbm", result.stdout) + self.assertIn( + "dryRun prepareProfile[2] name=rt_hbm alloc=rt_hbm window=full_repository", + result.stdout, + ) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=1", result.stdout) + self.assertIn("TILEXR_CCU_PROBE_INSTRUCTION_START=475", result.stdout) + self.assertIn("TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START=489", result.stdout) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT=143", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE=acl", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_ORDER=lower_layer_first", result.stdout) + self.assertIn("dryRun TILEXR_CCU_DIRECT_INSTALL_ORDER=lower_layer_first", result.stdout) + self.assertIn("TILEXR_CCU_PROBE_SQE_ARG_COUNT=13", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE=sync_cke", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE=hcomm_cap", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK0=0x2222", result.stdout) + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK1=0x3333", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke_runner dryRun=1", result.stdout) + self.assertIn("dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID=0x2222", result.stdout) + self.assertIn("dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID=0x3333", result.stdout) + self.assertIn( + "dryRun derived repositoryStartId=475 repositoryCount=170 missionInstructionStartId=489", + result.stdout, + ) + self.assertIn("dryRun derived task0.instStartId=489 task0.instCnt=13", result.stdout) + self.assertIn("dryRun derived task1.instStartId=502 task1.instCnt=143", result.stdout) + self.assertIn("dryRun derived SET_INSTRUCTION offsetStartIdx=475 dataLen=5440", result.stdout) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + + def test_evening_wrapper_accepts_dry_run_argument_without_touching_npu(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + result = subprocess.run( + ["bash", str(EVENING_RUNNER), "--dry-run"], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_evening_smoke dryRun=1", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke_runner dryRun=1", result.stdout) + self.assertNotIn("cmake --build", result.stdout + result.stderr) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + + def test_evening_wrapper_accepts_generic_smoke_dry_run_without_touching_npu(self): + with tempfile.TemporaryDirectory() as temp_dir: + env = os.environ.copy() + env["TILEXR_CCU_SMOKE_DRY_RUN"] = "1" + env["TILEXR_CCU_SMOKE_WORK_DIR"] = temp_dir + result = subprocess.run( + ["bash", str(EVENING_RUNNER)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("tilexr_ccu_direct_evening_smoke dryRun=1", result.stdout) + self.assertIn("tilexr_ccu_direct_smoke_runner dryRun=1", result.stdout) + self.assertNotIn("stage=prepare", result.stdout + result.stderr) + self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py new file mode 100644 index 00000000..cf8fc80a --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -0,0 +1,992 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +import os +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +DRIVER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.h" +DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" +SPECS_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.h" +SPECS_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuDriverAdapterTest(unittest.TestCase): + def compile_and_run(self, code: str, extra_env=None): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "driver_adapter_test.cpp" + test_bin = temp_path / "driver_adapter_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(DRIVER_SOURCE), + str(SPECS_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + env = None + if extra_env: + env = {**os.environ, **extra_env} + return subprocess.run( + [str(test_bin)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env) + + def test_adapter_wraps_get_basic_info_and_reuses_tilexr_specs_decode(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + struct FakeState { + int calls = 0; + uint32_t observedDevice = 0; + uint32_t observedOp = 0; + uint32_t observedDie = 0; + }; + + int FakeCustomChannel( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + state->observedDevice = devicePhyId; + state->observedOp = in.op; + state->observedDie = in.data.dataInfo.udieIdx; + if (out == nullptr) { + return -1; + } + out->opRet = 0; + out->data.dataInfo.dataArray[0].baseinfo.msId = 0x45; + out->data.dataInfo.dataArray[0].baseinfo.tokenId = 0x1234; + out->data.dataInfo.dataArray[0].baseinfo.tokenValue = 0; + out->data.dataInfo.dataArray[0].baseinfo.tokenValid = 1; + out->data.dataInfo.dataArray[0].baseinfo.missionKey = 0x059b0f03U; + out->data.dataInfo.dataArray[0].baseinfo.resourceAddr = 0x200000000ULL; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap0 = (3U << 24) | (5U << 16) | 255U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap1 = (127U << 16) | 63U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap2 = (31U << 16) | 15U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap3 = (7U << 16) | 1U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap4 = 9U; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(4, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + TileXRCcuBasicInfo basic; + if (adapter.GetBasicInfo(1, &basic, &report) != TILEXR_SUCCESS) { + std::cerr << "get basic info failed: " << report.message << "\n"; + return 2; + } + if (state.calls != 1 || state.observedDevice != 4 || + state.observedOp != TILEXR_CCU_U_OP_GET_BASIC_INFO || state.observedDie != 1) { + std::cerr << "custom channel request mismatch\n"; + return 3; + } + if (basic.dieId != 1 || basic.msId != 0x45 || basic.missionKey != 0x059b0f03U || + basic.resourceAddr != 0x200000000ULL || basic.caps.cap0 == 0 || + basic.msidToken.tokenId != 0x1234 || basic.msidToken.tokenValue != 0 || + !basic.msidToken.valid) { + std::cerr << "basic info mismatch\n"; + return 4; + } + + TileXRCcuSpecInfo info; + TileXRCcuSpecsReport specsReport; + if (TileXRCcuDecodeBasicInfo(basic, &info, &specsReport) != TILEXR_SUCCESS) { + std::cerr << "decode failed: " << specsReport.message << "\n"; + return 5; + } + if (info.instructionNum != 256 || info.xnNum != 128 || info.channelNum != 2 || + info.missionNum != 6 || info.loopEngineNum != 4) { + std::cerr << "decoded info mismatch\n"; + return 6; + } + if (report.message != "ok" || report.opcode != TILEXR_CCU_U_OP_GET_BASIC_INFO || + report.dieId != 1 || report.devicePhyId != 4) { + std::cerr << "report mismatch\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_wraps_die_enable_and_reports_driver_errors(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + struct FakeState { + bool fail = false; + uint32_t observedOp = 0; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->observedOp = in.op; + if (state->fail) { + return -22; + } + out->opRet = 0; + out->data.dataInfo.dataArray[0].dieinfo.enableFlag = TILEXR_CCU_ENABLE_FLAG; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(7, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed\n"; + return 1; + } + + bool enabled = false; + if (adapter.GetDieEnabled(0, &enabled, &report) != TILEXR_SUCCESS || !enabled) { + std::cerr << "die enable query failed: " << report.message << "\n"; + return 2; + } + if (state.observedOp != TILEXR_CCU_U_OP_GET_DIE_WORKING) { + std::cerr << "die opcode mismatch\n"; + return 3; + } + + state.fail = true; + if (adapter.GetDieEnabled(0, &enabled, &report) != TILEXR_ERROR_MKIRT) { + std::cerr << "driver failure was accepted\n"; + return 4; + } + if (report.message.find("CCU custom channel call failed") == std::string::npos || + report.message.find("driverRet=-22") == std::string::npos || + report.message.find("opRet=0") == std::string::npos || + report.message.find("op=15") == std::string::npos || + report.driverRet != -22) { + std::cerr << "weak driver diagnostic: " << report.message << "\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_installs_instruction_repository_with_set_instruction_opcode(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + struct FakeState { + int calls = 0; + uint32_t observedDevice = 0; + uint32_t observedOp = 0; + uint32_t observedDie = 0; + uint32_t observedOffset = 0; + uint32_t observedDataLen = 0; + uint32_t observedArraySize = 0; + uint64_t observedResourceAddr = 0; + }; + + int FakeCustomChannel( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + state->observedDevice = devicePhyId; + state->observedOp = in.op; + state->observedDie = in.data.dataInfo.udieIdx; + state->observedOffset = in.offsetStartIdx; + state->observedDataLen = in.data.dataInfo.dataLen; + state->observedArraySize = in.data.dataInfo.dataArraySize; + state->observedResourceAddr = in.data.dataInfo.dataArray[0].insinfo.resourceAddr; + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + if (adapter.InstallInstructions(1, 489, 13, 0x100051152e00ULL, 13 * 32, &report) != + TILEXR_SUCCESS) { + std::cerr << "install failed: " << report.message << "\n"; + return 2; + } + if (state.calls != 1 || state.observedDevice != 5 || + state.observedOp != TILEXR_CCU_U_OP_SET_INSTRUCTION || + state.observedDie != 1 || state.observedOffset != 489 || + state.observedDataLen != 13 * 32 || state.observedArraySize != 1 || + state.observedResourceAddr != 0x100051152e00ULL) { + std::cerr << "SET_INSTRUCTION request mismatch\n"; + return 3; + } + if (report.message != "ok" || report.opcode != TILEXR_CCU_U_OP_SET_INSTRUCTION || + report.dieId != 1 || report.devicePhyId != 5) { + std::cerr << "report mismatch\n"; + return 4; + } + + if (adapter.InstallInstructions(1, 489, 13, 0, 13 * 32, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "zero device instruction address accepted\n"; + return 5; + } + if (adapter.InstallInstructions(1, 489, 0, 0x100051152e00ULL, 0, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty instruction image accepted\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_rejects_instruction_byte_mismatch_before_custom_channel(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + struct FakeState { + int calls = 0; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + const int ret = adapter.InstallInstructions( + 1, + 489, + 13, + 0x100051152e00ULL, + 12 * TILEXR_CCU_INSTRUCTION_BYTES, + &report); + if (ret != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "byte mismatch accepted ret=" << ret << "\n"; + return 2; + } + if (state.calls != 0 || + report.message.find("byte size mismatch") == std::string::npos) { + std::cerr << "byte mismatch should fail before custom channel: " + << report.message << " calls=" << state.calls << "\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_reads_each_instruction_from_its_own_data_array_slot(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + #include + + using namespace TileXR; + + struct InstructionWords { + uint64_t words[4]; + }; + + struct FakeState { + int calls = 0; + uint32_t observedDevice = 0; + uint32_t observedOp = 0; + uint32_t observedDie = 0; + uint32_t observedOffset = 0; + uint32_t observedDataLen = 0; + uint32_t observedArraySize = 0; + }; + + int FakeCustomChannel( + uint32_t devicePhyId, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + state->observedDevice = devicePhyId; + state->observedOp = in.op; + state->observedDie = in.data.dataInfo.udieIdx; + state->observedOffset = in.offsetStartIdx; + state->observedDataLen = in.data.dataInfo.dataLen; + state->observedArraySize = in.data.dataInfo.dataArraySize; + out->opRet = 0; + + InstructionWords first {{0x1014c00010802ULL, 0, 0, 0}}; + InstructionWords second {{0x10804ULL, 0x1014cULL, 0, 0}}; + std::memcpy(out->data.dataInfo.dataArray[0].byte32.raw, &first, sizeof(first)); + std::memcpy(out->data.dataInfo.dataArray[1].byte32.raw, &second, sizeof(second)); + out->data.dataInfo.dataArraySize = 2; + out->data.dataInfo.dataLen = 2 * TILEXR_CCU_INSTRUCTION_BYTES; + out->offsetNextIdx = 491; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + InstructionWords instructions[2] {}; + if (adapter.ReadInstructions( + 1, + 489, + instructions, + 2, + 2 * TILEXR_CCU_INSTRUCTION_BYTES, + &report) != TILEXR_SUCCESS) { + std::cerr << "read failed: " << report.message << "\n"; + return 2; + } + if (state.calls != 1 || state.observedDevice != 5 || + state.observedOp != TILEXR_CCU_U_OP_GET_INSTRUCTION || + state.observedDie != 1 || state.observedOffset != 489 || + state.observedDataLen != 2 * TILEXR_CCU_INSTRUCTION_BYTES || + state.observedArraySize != 2) { + std::cerr << "GET_INSTRUCTION request mismatch\n"; + return 3; + } + if (instructions[0].words[0] != 0x1014c00010802ULL || + instructions[0].words[1] != 0 || + instructions[1].words[0] != 0x10804ULL || + instructions[1].words[1] != 0x1014cULL) { + std::cerr << "readback slot copy mismatch first=0x" << std::hex + << instructions[0].words[0] << " second0=0x" + << instructions[1].words[0] << " second1=0x" + << instructions[1].words[1] << "\n"; + return 4; + } + if (report.message != "ok" || report.opcode != TILEXR_CCU_U_OP_GET_INSTRUCTION || + report.dieId != 1 || report.devicePhyId != 5) { + std::cerr << "report mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_set_instruction_trailer_wire_word_is_offset_then_opcode(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + #include + + using namespace TileXR; + + struct FakeState { + uint64_t trailer = 0; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + std::memcpy(&state->trailer, &in.offsetStartIdx, sizeof(state->trailer)); + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + if (adapter.InstallInstructions( + 1, 489, 13, 0x100051152e00ULL, 13 * TILEXR_CCU_INSTRUCTION_BYTES, &report) != + TILEXR_SUCCESS) { + std::cerr << "install failed: " << report.message << "\n"; + return 2; + } + const uint64_t expected = + (static_cast(TILEXR_CCU_U_OP_SET_INSTRUCTION) << 32U) | 489ULL; + if (state.trailer != expected) { + std::cerr << "trailer mismatch observed=0x" << std::hex << state.trailer + << " expected=0x" << expected << "\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_adapter_wraps_lower_layer_set_payloads_without_hcomm_runtime(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + uint32_t die = 0; + uint32_t offset = 0; + uint32_t dataLen = 0; + uint32_t arraySize = 0; + uint8_t raw[256] = {0}; + uint32_t msId = 0; + uint32_t tokenId = 0; + uint32_t tokenValue = 0; + }; + + struct FakeState { + int calls = 0; + ObservedCall observed[8]; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + if (state->calls >= 8) { + return -1; + } + auto& observed = state->observed[state->calls++]; + observed.op = in.op; + observed.die = in.data.dataInfo.udieIdx; + observed.offset = in.offsetStartIdx; + observed.dataLen = in.data.dataInfo.dataLen; + observed.arraySize = in.data.dataInfo.dataArraySize; + observed.msId = in.data.dataInfo.dataArray[0].baseinfo.msId; + observed.tokenId = in.data.dataInfo.dataArray[0].baseinfo.tokenId; + observed.tokenValue = in.data.dataInfo.dataArray[0].baseinfo.tokenValue; + std::memcpy(observed.raw, in.data.dataInfo.dataArray, sizeof(observed.raw)); + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + if (adapter.InstallMsidToken(1, 0x45, 0x1234, 0x5678, &report) != TILEXR_SUCCESS) { + std::cerr << "msid install failed: " << report.message << "\n"; + return 2; + } + if (state.observed[0].op != TILEXR_CCU_U_OP_SET_MSID_TOKEN || + state.observed[0].die != 1 || state.observed[0].offset != 0 || + state.observed[0].dataLen != 0 || state.observed[0].arraySize != 0 || + state.observed[0].msId != 0x45 || state.observed[0].tokenId != 0x1234 || + state.observed[0].tokenValue != 0x5678) { + std::cerr << "SET_MSID_TOKEN request mismatch\n"; + return 3; + } + + TileXRCcuPfeCtx pfe{}; + for (uint32_t i = 0; i < sizeof(pfe.raw); ++i) { + pfe.raw[i] = static_cast(0xa0 + i); + } + if (adapter.InstallPfeCtx(1, 7, pfe, &report) != TILEXR_SUCCESS) { + std::cerr << "pfe install failed: " << report.message << "\n"; + return 4; + } + if (state.observed[1].op != TILEXR_CCU_U_OP_SET_PFE || + state.observed[1].die != 1 || state.observed[1].offset != 7 || + state.observed[1].dataLen != TILEXR_CCU_PFE_CTX_BYTES || + state.observed[1].arraySize != 1 || + std::memcmp(state.observed[1].raw, pfe.raw, TILEXR_CCU_PFE_CTX_BYTES) != 0) { + std::cerr << "SET_PFE request mismatch\n"; + return 5; + } + + TileXRCcuLocalJettyCtxData jettys[2]{}; + for (uint32_t i = 0; i < sizeof(jettys[0].raw); ++i) { + jettys[0].raw[i] = static_cast(0x10 + i); + jettys[1].raw[i] = static_cast(0x50 + i); + } + if (adapter.InstallJettyCtx(1, 9, jettys, 2, &report) != TILEXR_SUCCESS) { + std::cerr << "jetty install failed: " << report.message << "\n"; + return 6; + } + if (state.observed[2].op != TILEXR_CCU_U_OP_SET_JETTY_CTX || + state.observed[2].die != 1 || state.observed[2].offset != 9 || + state.observed[2].dataLen != 2 * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES || + state.observed[2].arraySize != 2 || + std::memcmp(state.observed[2].raw, jettys[0].raw, TILEXR_CCU_LOCAL_JETTY_CTX_BYTES) != 0 || + std::memcmp( + state.observed[2].raw + TILEXR_CCU_DATA_ARRAY_SLOT_BYTES, + jettys[1].raw, + TILEXR_CCU_LOCAL_JETTY_CTX_BYTES) != 0) { + std::cerr << "SET_JETTY_CTX request mismatch\n"; + return 7; + } + + TileXRCcuChannelCtxDataV1 channel{}; + for (uint32_t i = 0; i < sizeof(channel.raw); ++i) { + channel.raw[i] = static_cast(0xc0 + i); + } + if (adapter.InstallChannelCtxV1(1, 11, channel, &report) != TILEXR_SUCCESS) { + std::cerr << "channel install failed: " << report.message << "\n"; + return 8; + } + if (state.observed[3].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.observed[3].die != 1 || state.observed[3].offset != 11 || + state.observed[3].dataLen != TILEXR_CCU_CHANNEL_CTX_V1_BYTES || + state.observed[3].arraySize != 1 || + std::memcmp(state.observed[3].raw, channel.raw, TILEXR_CCU_CHANNEL_CTX_V1_BYTES) != 0) { + std::cerr << "SET_CHANNEL request mismatch\n"; + return 9; + } + + if (adapter.ClearCkeRange(1, 16, 10, &report) != TILEXR_SUCCESS) { + std::cerr << "cke clear failed: " << report.message << "\n"; + return 10; + } + if (state.observed[4].op != TILEXR_CCU_U_OP_SET_CKE || + state.observed[4].offset != 16 || state.observed[4].arraySize != 8 || + state.observed[4].dataLen != 8 * TILEXR_CCU_CKE_SLOT_BYTES || + state.observed[5].op != TILEXR_CCU_U_OP_SET_CKE || + state.observed[5].offset != 24 || state.observed[5].arraySize != 2 || + state.observed[5].dataLen != 2 * TILEXR_CCU_CKE_SLOT_BYTES) { + std::cerr << "SET_CKE batching mismatch\n"; + return 11; + } + + if (adapter.InstallXnRange(1, 32, 10, &report) != TILEXR_SUCCESS) { + std::cerr << "xn install failed: " << report.message << "\n"; + return 12; + } + if (state.observed[6].op != TILEXR_CCU_U_OP_SET_XN || + state.observed[6].offset != 32 || state.observed[6].arraySize != 8 || + state.observed[6].dataLen != 8 * TILEXR_CCU_XN_SLOT_BYTES || + state.observed[7].op != TILEXR_CCU_U_OP_SET_XN || + state.observed[7].offset != 40 || state.observed[7].arraySize != 2 || + state.observed[7].dataLen != 2 * TILEXR_CCU_XN_SLOT_BYTES) { + std::cerr << "SET_XN batching mismatch\n"; + return 13; + } + + if (adapter.InstallJettyCtx(1, 9, jettys, 0, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || + adapter.InstallJettyCtx(1, 9, nullptr, 2, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || + adapter.ClearCkeRange(1, 0, 0, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || + adapter.InstallXnRange(1, 0, 0, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid lower-layer payload accepted\n"; + return 14; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_msid_token_envelope_intentionally_matches_hcomm_zero_length_reference(self): + hcomm_source = ( + REPO_ROOT + / "3rdparty" + / "hcomm" + / "src" + / "framework" + / "next" + / "comms" + / "ccu" + / "ccu_device" + / "ccu_comp" + / "ccu_comp.cc" + ).read_text(encoding="utf-8") + hcomm_body = hcomm_source[ + hcomm_source.index("HcclResult CcuComponent::ConfigMsIdToken()") : + hcomm_source.index("HcclResult CcuComponent::GetCcuResourceSpaceBufInfo") + ] + source = DRIVER_SOURCE.read_text(encoding="utf-8") + tilexr_body = source[ + source.index("int TileXRCcuDriverAdapter::InstallMsidToken(") : + source.index("int TileXRCcuDriverAdapter::InstallPfeCtx(") + ] + + for needle in [ + "CCU_U_OP_SET_MSID_TOKEN", + "baseinfo.msId", + "baseinfo.tokenId", + "baseinfo.tokenValue", + ]: + with self.subTest(reference=needle): + self.assertIn(needle, hcomm_body) + self.assertIn(needle, tilexr_body) + + self.assertNotIn("dataArraySize", hcomm_body) + self.assertNotIn("dataLen", hcomm_body) + self.assertNotIn("dataArraySize", tilexr_body) + self.assertNotIn("dataLen", tilexr_body) + + def test_direct_trace_dumps_custom_channel_envelope_and_payload_words(self): + source = DRIVER_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "TILEXR_CCU_DIRECT_TRACE", + "TraceCustomChannelRequest", + "TileXRDirectCcuTrace customChannel", + "devicePhyId=", + "op=", + "dieId=", + "offset=", + "dataLen=", + "arraySize=", + "payloadWords=", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + def test_direct_trace_dumps_custom_channel_return_and_trailer_fields(self): + source = DRIVER_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "TraceCustomChannelReturn", + "TileXRDirectCcuTrace customChannel.return", + "driverRet=", + "opRet=", + "offsetNext=", + "customChannel.requestTrailer", + "customChannel.response", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + def test_direct_trace_runtime_emits_custom_channel_request_payload(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + int main() + { + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, nullptr, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + TileXRCcuChannelCtxDataV1 channel{}; + for (uint32_t i = 0; i < TILEXR_CCU_CHANNEL_CTX_V1_BYTES; ++i) { + channel.raw[i] = static_cast(0x10 + i); + } + if (adapter.InstallChannelCtxV1(1, 11, channel, &report) != TILEXR_SUCCESS) { + std::cerr << "channel install failed: " << report.message << "\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code, {"TILEXR_CCU_DIRECT_TRACE": "1"}) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("TileXRDirectCcuTrace customChannel", result.stderr) + self.assertIn("devicePhyId=5", result.stderr) + self.assertIn("op=256", result.stderr) + self.assertIn("dieId=1", result.stderr) + self.assertIn("offset=11", result.stderr) + self.assertIn("dataLen=64", result.stderr) + self.assertIn("arraySize=1", result.stderr) + self.assertIn("payloadWords=8", result.stderr) + self.assertIn("customChannel.payloadWords=8", result.stderr) + self.assertIn("w0=0x1716151413121110", result.stderr) + + def test_direct_trace_runtime_emits_custom_channel_return_and_trailer_fields(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + + using namespace TileXR; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->offsetNextIdx = 99; + out->opRet = 7; + out->data.dataInfo.dataArray[0].dieinfo.enableFlag = 0; + return -22; + } + + int main() + { + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, nullptr, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + + bool enabled = true; + const int ret = adapter.GetDieEnabled(1, &enabled, &report); + if (ret != TILEXR_ERROR_MKIRT) { + std::cerr << "driver failure was accepted\n"; + return 2; + } + if (report.driverRet != -22 || report.opRet != 7 || + report.opcode != TILEXR_CCU_U_OP_GET_DIE_WORKING) { + std::cerr << "report did not retain driver diagnostics\n"; + return 3; + } + if (report.message.find("driverRet=-22") == std::string::npos || + report.message.find("opRet=7") == std::string::npos || + report.message.find("op=15") == std::string::npos) { + std::cerr << "message did not retain driver diagnostics: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code, {"TILEXR_CCU_DIRECT_TRACE": "1"}) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + self.assertIn("TileXRDirectCcuTrace customChannel.return", result.stderr) + self.assertIn("driverRet=-22", result.stderr) + self.assertIn("opRet=7", result.stderr) + self.assertIn("offsetNext=99", result.stderr) + self.assertIn("op=15", result.stderr) + self.assertIn("customChannel.requestTrailerWords=1", result.stderr) + self.assertIn("customChannel.responseWords=", result.stderr) + + def test_driver_adapter_failure_message_includes_opcode_and_driver_status(self): + source = DRIVER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("CcuCustomChannelFailureMessage", source) + self.assertIn('"CCU custom channel call failed"', source) + self.assertIn('"CCU custom channel operation failed"', source) + self.assertIn('" op="', source) + self.assertIn('" driverRet="', source) + self.assertIn('" opRet="', source) + self.assertIn( + "CcuCustomChannelFailureMessage(\"CCU custom channel call failed\", opcode, driverRet, out->opRet)", + source, + ) + self.assertIn( + "CcuCustomChannelFailureMessage(\"CCU custom channel operation failed\", opcode, driverRet, out->opRet)", + source, + ) + + def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = DRIVER_HEADER.read_text(encoding="utf-8") + source = DRIVER_SOURCE.read_text(encoding="utf-8") + specs_header = SPECS_HEADER.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_driver_adapter.h", cmake) + self.assertIn("ccu/tilexr_ccu_driver_adapter.cpp", cmake) + self.assertIn("TileXRCcuDriverAdapter", header) + self.assertIn("TileXRCcuMsidTokenInfo", specs_header) + self.assertIn("TileXRCcuMsidTokenInfo msidToken", specs_header) + self.assertIn("basicInfo->msidToken.tokenId = raw.tokenId", source) + self.assertIn("basicInfo->msidToken.tokenValue = raw.tokenValue", source) + self.assertIn("basicInfo->msidToken.valid = raw.tokenValid != 0", source) + self.assertIn("TileXRCcuCustomChannelIn", header) + self.assertIn("TILEXR_CCU_U_OP_GET_BASIC_INFO", header) + self.assertIn("TILEXR_CCU_U_OP_GET_DIE_WORKING", header) + self.assertIn("TILEXR_CCU_U_OP_SET_MSID_TOKEN", header) + self.assertIn("TILEXR_CCU_U_OP_SET_INSTRUCTION", header) + self.assertIn("TILEXR_CCU_U_OP_SET_XN", header) + self.assertIn("TILEXR_CCU_U_OP_SET_CKE", header) + self.assertIn("TILEXR_CCU_U_OP_SET_PFE", header) + self.assertIn("TILEXR_CCU_U_OP_SET_CHANNEL", header) + self.assertIn("TILEXR_CCU_U_OP_SET_JETTY_CTX", header) + self.assertIn("TILEXR_CCU_XN_SLOT_BYTES", header) + self.assertIn("TileXRCcuPfeCtx", header) + self.assertIn("TileXRCcuLocalJettyCtxData", header) + self.assertIn("TileXRCcuChannelCtxDataV1", header) + self.assertIn("TileXRCcuCustomChannelFn", header) + self.assertIn("GetBasicInfo", header) + self.assertIn("GetDieEnabled", header) + self.assertIn("InstallInstructions", header) + self.assertIn("InstallMsidToken", header) + self.assertIn("InstallPfeCtx", header) + self.assertIn("InstallJettyCtx", header) + self.assertIn("InstallChannelCtxV1", header) + self.assertIn("ClearCkeRange", header) + self.assertIn("InstallXnRange", header) + + combined = header + "\n" + source + for needle in [ + "#include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 10; + } + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = true; + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "default install provider unexpectedly succeeded\n"; + return 2; + } + if (installReport.installAttempted || installReport.installSucceeded) { + std::cerr << "default install provider reported an attempted/succeeded install\n"; + return 3; + } + if (installReport.message.find("no public no-hcomm CCU install provider") == std::string::npos) { + std::cerr << "weak install provider diagnostic: " << installReport.message << "\n"; + return 4; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "default install provider filled install evidence\n"; + return 5; + } + if (evidence.missionSource.kind != TileXRCcuEvidenceKind::Missing || + evidence.repositorySource.kind != TileXRCcuEvidenceKind::Missing || + evidence.localXnSource.kind != TileXRCcuEvidenceKind::Missing || + evidence.remoteXnSource.kind != TileXRCcuEvidenceKind::Missing || + evidence.notifyCkeSource.kind != TileXRCcuEvidenceKind::Missing || + evidence.channelBindingSource.kind != TileXRCcuEvidenceKind::Missing) { + std::cerr << "default install provider did not mark sources missing\n"; + return 6; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "unsupported install evidence became submit-ready\n"; + return 7; + } + if (!submitTasks.empty() || providerReport.submitReady) { + std::cerr << "failed prepare leaked submit-ready state\n"; + return 8; + } + if (providerReport.message.find("mission") == std::string::npos || + providerReport.message.find("missing") == std::string::npos) { + std::cerr << "weak provider diagnostic: " << providerReport.message << "\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_request_must_match_bound_launch_scope(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int ExpectScopeFailure( + const TileXRCcuInstallRequest& request, + const char* diagnostic) + { + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport report; + const int ret = TileXRCcuInstallHardware(request, &evidence, &report); + if (ret != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "scope mismatch was not rejected: " << diagnostic << "\n"; + return 1; + } + if (report.message.find(diagnostic) == std::string::npos) { + std::cerr << "weak scope mismatch diagnostic: " << report.message << "\n"; + return 2; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed scope mismatch left install evidence\n"; + return 3; + } + return 0; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallRequest request; + request.package = &package; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + + TileXRCcuInstallRequest wrongDevice = request; + wrongDevice.deviceId = 4; + if (ExpectScopeFailure(wrongDevice, "device scope mismatch") != 0) { + return 3; + } + + TileXRCcuInstallRequest wrongRank = request; + wrongRank.rank = 0; + if (ExpectScopeFailure(wrongRank, "rank scope mismatch") != 0) { + return 4; + } + + TileXRCcuInstallRequest wrongProvider = request; + wrongProvider.provider = "other-public-install-provider"; + if (ExpectScopeFailure(wrongProvider, "provider scope mismatch") != 0) { + return 5; + } + + TileXRCcuInstallRequest missingProvider = request; + missingProvider.provider.clear(); + if (ExpectScopeFailure(missingProvider, "provider scope mismatch") != 0) { + return 6; + } + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport report; + if (TileXRCcuInstallHardware(request, &evidence, &report) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "matching scope should reach unsupported provider: " << report.message << "\n"; + return 7; + } + if (report.message.find("no public no-hcomm CCU install provider") == std::string::npos) { + std::cerr << "matching scope diagnostic was weak: " << report.message << "\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_online_install_provider_installs_repository_and_keeps_remaining_bits_missing(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + uint32_t observedOp = 0; + uint32_t observedOffset = 0; + uint32_t observedDataLen = 0; + uint64_t observedResourceAddr = 0; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->observedOp = in.op; + state->observedOffset = in.offsetStartIdx; + state->observedDataLen = in.data.dataInfo.dataLen; + state->observedResourceAddr = in.data.dataInfo.dataArray[0].insinfo.resourceAddr; + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "partial hardware install should stop at missing lower-layer resources: " + << installReport.message << "\n"; + return 5; + } + if (!installReport.installAttempted || !installReport.installSucceeded || + installReport.installAttemptReceiptId == 0 || + installReport.publicVerifiedInstallSurfaceCount != 2 || + installReport.missingInstallSurfaceCount != 4) { + std::cerr << "repository install report mismatch\n"; + return 6; + } + if (!installReport.repository.satisfied || !installReport.mission.satisfied || + installReport.localXn.satisfied || installReport.remoteXn.satisfied || + installReport.notifyCke.satisfied || installReport.channelBinding.satisfied) { + std::cerr << "unexpected install step evidence bits\n"; + return 7; + } + if (!evidence.repositoryInstalled || !evidence.missionInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "unexpected hardware evidence bits\n"; + return 8; + } + if (evidence.repositorySource.kind != TileXRCcuEvidenceKind::PublicVerified || + evidence.repositorySource.surface != TileXRCcuEvidenceSurface::PublicInstallProvider || + evidence.repositorySource.packageFingerprint != TileXRCcuComputeLaunchPackageFingerprint(package) || + evidence.repositorySource.deviceId != 3 || evidence.repositorySource.rank != 1 || + evidence.repositorySource.provider != "unit-test-public-install-provider" || + evidence.repositorySource.installAttemptReceiptId != installReport.installAttemptReceiptId) { + std::cerr << "repository evidence scope mismatch\n"; + return 9; + } + if (evidence.missionSource.kind != TileXRCcuEvidenceKind::PublicVerified || + evidence.missionSource.surface != TileXRCcuEvidenceSurface::PublicInstallProvider || + evidence.missionSource.installAttemptReceiptId != installReport.installAttemptReceiptId || + evidence.missionSource.detail.find("launch task descriptor") == std::string::npos || + evidence.missionSource.detail.find("mission key") == std::string::npos) { + std::cerr << "mission descriptor evidence mismatch\n"; + return 12; + } + if (state.observedOp != TILEXR_CCU_U_OP_SET_INSTRUCTION || + state.observedOffset != 489 || + state.observedDataLen != 156U * sizeof(TileXRCcuInstr) || + state.observedResourceAddr != receipt.deviceInstructionAddr || + receipt.instructionStartId != 489 || receipt.instructionCount != 156 || + !receipt.uploaded || !receipt.installed) { + std::cerr << "SET_INSTRUCTION request or receipt mismatch\n"; + return 10; + } + + TileXRCcuRepositoryReport releaseReport; + if (TileXRCcuReleaseRepositoryInstallReceipt(receipt, memoryOps, &state, &releaseReport) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "release failed: " << releaseReport.message << "\n"; + return 11; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_online_install_provider_installs_lower_layer_payloads_without_unlocking_submit_gate(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + uint32_t die = 0; + uint32_t offset = 0; + uint32_t dataLen = 0; + uint32_t arraySize = 0; + uint8_t raw[160] = {0}; + uint32_t msId = 0; + uint32_t tokenId = 0; + uint32_t tokenValue = 0; + }; + + struct FakeState { + std::vector deviceBytes; + std::vector calls; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + ObservedCall call; + call.op = in.op; + call.die = in.data.dataInfo.udieIdx; + call.offset = in.offsetStartIdx; + call.dataLen = in.data.dataInfo.dataLen; + call.arraySize = in.data.dataInfo.dataArraySize; + call.msId = in.data.dataInfo.dataArray[0].baseinfo.msId; + call.tokenId = in.data.dataInfo.dataArray[0].baseinfo.tokenId; + call.tokenValue = in.data.dataInfo.dataArray[0].baseinfo.tokenValue; + std::memcpy(call.raw, in.data.dataInfo.dataArray, sizeof(call.raw)); + state->calls.push_back(call); + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 7; + for (uint32_t i = 0; i < TILEXR_CCU_PFE_CTX_BYTES; ++i) { + pfe.ctx.raw[i] = static_cast(0xa0 + i); + } + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(2); + for (uint32_t i = 0; i < TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; ++i) { + jetty.ctxs[0].raw[i] = static_cast(0x10 + i); + jetty.ctxs[1].raw[i] = static_cast(0x50 + i); + } + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 11; + for (uint32_t i = 0; i < TILEXR_CCU_CHANNEL_CTX_V1_BYTES; ++i) { + channel.ctx.raw[i] = static_cast(0xc0 + i); + } + lowerLayer.channels.push_back(channel); + lowerLayer.xnClears.push_back({1, 32, 10}); + lowerLayer.ckeClears.push_back({1, 16, 10}); + + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "partial lower-layer install should not unlock submit gate: " + << installReport.message << "\n"; + return 5; + } + if (!installReport.installAttempted || !installReport.installSucceeded || + installReport.publicVerifiedInstallSurfaceCount != 4 || + installReport.missingInstallSurfaceCount != 2) { + std::cerr << "lower-layer install report mismatch\n"; + return 6; + } + if (!evidence.repositoryInstalled || !evidence.missionInstalled || !evidence.localXnInstalled || + !evidence.notifyCkeInstalled || evidence.channelBindingsInstalled || + evidence.remoteXnBound) { + std::cerr << "unexpected evidence bits after lower-layer install\n"; + return 7; + } + if (evidence.repositorySource.installAttemptReceiptId != installReport.installAttemptReceiptId || + evidence.missionSource.installAttemptReceiptId != installReport.installAttemptReceiptId || + evidence.localXnSource.installAttemptReceiptId != installReport.installAttemptReceiptId || + evidence.notifyCkeSource.installAttemptReceiptId != installReport.installAttemptReceiptId) { + std::cerr << "lower-layer evidence receipts mismatch\n"; + return 8; + } + if (evidence.missionSource.detail.find("launch task descriptor") == std::string::npos || + evidence.missionSource.detail.find("mission key") == std::string::npos || + evidence.localXnSource.detail.find("SET_XN") == std::string::npos || + evidence.notifyCkeSource.detail.find("SET_CKE") == std::string::npos || + evidence.channelBindingSource.kind != TileXRCcuEvidenceKind::Missing || + evidence.channelBindingSource.detail.find("channel binding install evidence is missing") == + std::string::npos || + installReport.channelBinding.satisfied || + installReport.channelBinding.message.find("channel binding endpoint route provenance was not verified") == + std::string::npos) { + std::cerr << "lower-layer evidence detail is weak\n"; + return 9; + } + + if (state.calls.size() != 9 || + state.calls[0].op != TILEXR_CCU_U_OP_SET_INSTRUCTION || + state.calls[1].op != TILEXR_CCU_U_OP_SET_MSID_TOKEN || + state.calls[2].op != TILEXR_CCU_U_OP_SET_PFE || + state.calls[3].op != TILEXR_CCU_U_OP_SET_JETTY_CTX || + state.calls[4].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[5].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[6].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[7].op != TILEXR_CCU_U_OP_SET_CKE || + state.calls[8].op != TILEXR_CCU_U_OP_SET_CKE) { + std::cerr << "unexpected lower-layer call sequence\n"; + return 10; + } + if (state.calls[1].msId != 0x45 || state.calls[1].tokenId != 0x1234 || + state.calls[1].tokenValue != 0x5678) { + std::cerr << "MSID token request mismatch\n"; + return 11; + } + if (state.calls[2].offset != 7 || state.calls[2].dataLen != TILEXR_CCU_PFE_CTX_BYTES || + std::memcmp(state.calls[2].raw, pfe.ctx.raw, TILEXR_CCU_PFE_CTX_BYTES) != 0) { + std::cerr << "PFE request mismatch\n"; + return 12; + } + if (state.calls[3].offset != 9 || + state.calls[3].dataLen != 2 * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES || + std::memcmp(state.calls[3].raw, jetty.ctxs[0].raw, TILEXR_CCU_LOCAL_JETTY_CTX_BYTES) != 0 || + std::memcmp( + state.calls[3].raw + TILEXR_CCU_DATA_ARRAY_SLOT_BYTES, + jetty.ctxs[1].raw, + TILEXR_CCU_LOCAL_JETTY_CTX_BYTES) != 0) { + std::cerr << "Jetty request mismatch\n"; + return 13; + } + if (state.calls[4].offset != 11 || + state.calls[4].dataLen != TILEXR_CCU_CHANNEL_CTX_V1_BYTES || + std::memcmp(state.calls[4].raw, channel.ctx.raw, TILEXR_CCU_CHANNEL_CTX_V1_BYTES) != 0) { + std::cerr << "Channel request mismatch\n"; + return 14; + } + if (state.calls[5].offset != 32 || state.calls[5].arraySize != 8 || + state.calls[6].offset != 40 || state.calls[6].arraySize != 2) { + std::cerr << "XN batching mismatch\n"; + return 15; + } + if (state.calls[7].offset != 16 || state.calls[7].arraySize != 8 || + state.calls[8].offset != 24 || state.calls[8].arraySize != 2) { + std::cerr << "CKE batching mismatch\n"; + return 16; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "partial lower-layer evidence became submit-ready\n"; + return 17; + } + if (providerReport.message.find("remote XN") == std::string::npos || + providerReport.message.find("missing") == std::string::npos) { + std::cerr << "submit rejection diagnostic is weak: " << providerReport.message << "\n"; + return 18; + } + + TileXRCcuRepositoryReport releaseReport; + if (TileXRCcuReleaseRepositoryInstallReceipt(receipt, memoryOps, &state, &releaseReport) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "release failed: " << releaseReport.message << "\n"; + return 19; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_online_install_provider_promotes_remote_xn_when_peer_exchange_matches_sync_resources(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + uint32_t offset = 0; + }; + + struct FakeState { + std::vector deviceBytes; + std::vector calls; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void* userData) + { + auto* state = static_cast(userData); + state->freed = true; + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls.push_back({in.op, in.offsetStartIdx}); + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 3}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 365, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 366, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeLowerLayer(bool staleRemoteXn) + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + for (uint32_t i = 0; i < 3; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2 + i; + channel.ctx.raw[0] = 0xc0 + i; + lowerLayer.channels.push_back(channel); + } + + lowerLayer.xnClears.push_back({1, 1961, 3}); + lowerLayer.ckeClears.push_back({1, 332, 3}); + + lowerLayer.remoteXnBindings.push_back( + {1, 2, 1961, staleRemoteXn ? 2369 : 2361, 364, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 3, 1962, 2362, 365, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 4, 1963, 2364, 366, 0, true, 0, true, true, true}); + return lowerLayer; + } + + int RunInstall(const TileXRCcuLaunchPackage& package, + const TileXRCcuInstallManifest& manifest, + const TileXRCcuLowerLayerInstallPlan& lowerLayer, + FakeState* state, + TileXRCcuHardwareInstallEvidence* evidence, + TileXRCcuInstallProviderReport* installReport) + { + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 100; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + const int ret = TileXRCcuInstallHardware(request, evidence, installReport); + TileXRCcuRepositoryReport releaseReport; + if (receipt.deviceInstructionAddr != 0 && + TileXRCcuReleaseRepositoryInstallReceipt(receipt, memoryOps, state, &releaseReport) != + TILEXR_SUCCESS) { + std::cerr << "release failed: " << releaseReport.message << "\n"; + return 101; + } + return ret; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState staleState; + TileXRCcuHardwareInstallEvidence staleEvidence; + TileXRCcuInstallProviderReport staleReport; + TileXRCcuLowerLayerInstallPlan staleLowerLayer = MakeLowerLayer(true); + if (RunInstall(package, manifest, staleLowerLayer, &staleState, &staleEvidence, &staleReport) != + TILEXR_ERROR_NOT_FOUND) { + std::cerr << "stale remote XN proof should not complete install\n"; + return 4; + } + if (staleEvidence.remoteXnBound || staleReport.remoteXn.satisfied || + staleReport.publicVerifiedInstallSurfaceCount != 5 || + staleReport.missingInstallSurfaceCount != 1 || + staleReport.remoteXn.message.find("remote XN peer exchange proof") == std::string::npos) { + std::cerr << "stale remote XN proof was accepted or weakly diagnosed: " + << staleReport.remoteXn.message << "\n"; + return 5; + } + + FakeState state; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeLowerLayer(false); + if (RunInstall(package, manifest, lowerLayer, &state, &evidence, &installReport) != + TILEXR_SUCCESS) { + std::cerr << "complete remote XN proof should complete install: " + << installReport.message << "\n"; + return 6; + } + if (!installReport.installAttempted || !installReport.installSucceeded || + installReport.publicVerifiedInstallSurfaceCount != 6 || + installReport.missingInstallSurfaceCount != 0) { + std::cerr << "complete install report mismatch\n"; + return 7; + } + if (!evidence.repositoryInstalled || !evidence.missionInstalled || + !evidence.localXnInstalled || !evidence.remoteXnBound || + !evidence.notifyCkeInstalled || !evidence.channelBindingsInstalled) { + std::cerr << "complete install evidence mismatch\n"; + return 8; + } + if (evidence.remoteXnSource.source.find("ValidateRemoteXnExchangeBindingProof") == + std::string::npos || + evidence.remoteXnSource.detail.find("peer exchange") == std::string::npos || + evidence.remoteXnSource.detail.find("verified endpoint route channel contexts") == + std::string::npos || + evidence.remoteXnSource.detail.find("channel resource owner") == std::string::npos || + evidence.remoteXnSource.detail.find("transport resource exchange") == std::string::npos || + !evidence.remoteXnSource.endpointRouteVerified) { + std::cerr << "remote XN evidence detail is weak\n"; + return 9; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_SUCCESS) { + std::cerr << "submit preparation failed: " << providerReport.message << "\n"; + return 10; + } + if (!providerReport.submitReady || submitTasks.size() != package.tasks.size()) { + std::cerr << "submit readiness mismatch\n"; + return 11; + } + if (state.calls.size() != 9 || + state.calls[0].op != TILEXR_CCU_U_OP_SET_INSTRUCTION || + state.calls[4].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[5].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[6].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[7].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[8].op != TILEXR_CCU_U_OP_SET_CKE) { + std::cerr << "unexpected complete install call sequence\n"; + return 12; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_lower_layer_first_repository_failure_reports_installed_preconditions(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct ObservedCall { + uint32_t op = 0; + }; + + struct FakeState { + std::vector deviceBytes; + std::vector calls; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void* userData) + { + auto* state = static_cast(userData); + state->freed = true; + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls.push_back({in.op}); + out->opRet = in.op == TILEXR_CCU_U_OP_SET_INSTRUCTION ? 0x51 : 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 365, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 366, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + for (uint32_t i = 0; i < 3; ++i) { + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2 + i; + channel.ctx.raw[0] = static_cast(0xc0 + i); + lowerLayer.channels.push_back(channel); + } + + lowerLayer.xnClears.push_back({1, 1961, 3}); + lowerLayer.ckeClears.push_back({1, 332, 3}); + lowerLayer.remoteXnBindings.push_back({1, 2, 1961, 2361, 364, 0, true, 0, true}); + lowerLayer.remoteXnBindings.push_back({1, 3, 1962, 2362, 365, 0, true, 0, true}); + lowerLayer.remoteXnBindings.push_back({1, 4, 1963, 2364, 366, 0, true, 0, true}); + return lowerLayer; + } + + bool Contains(const std::string& text, const char* needle) + { + return text.find(needle) != std::string::npos; + } + + bool Contains(const std::string& text, const std::string& needle) + { + return text.find(needle) != std::string::npos; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeLowerLayer(); + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.installOrder = TileXRCcuInstallOrder::InstallLowerLayerFirst; + request.lowerLayerPlan = &lowerLayer; + + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_MKIRT) { + std::cerr << "repository failure should propagate MKIRT: " << installReport.message << "\n"; + return 5; + } + if (!installReport.installAttempted || installReport.installSucceeded || + installReport.publicVerifiedInstallSurfaceCount != 0 || + installReport.missingInstallSurfaceCount != 6) { + std::cerr << "repository failure report counts mismatch\n"; + return 6; + } + if (evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || + evidence.channelBindingsInstalled || evidence.missionInstalled) { + std::cerr << "failed repository install leaked hardware evidence\n"; + return 7; + } + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "failed repository install released submit gate\n"; + return 8; + } + if (providerReport.submitReady || !submitTasks.empty()) { + std::cerr << "failed repository install leaked submit-ready state\n"; + return 9; + } + if (state.calls.size() < 9 || + state.calls[0].op != TILEXR_CCU_U_OP_SET_MSID_TOKEN || + state.calls[1].op != TILEXR_CCU_U_OP_SET_PFE || + state.calls[2].op != TILEXR_CCU_U_OP_SET_JETTY_CTX || + state.calls[3].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[4].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[5].op != TILEXR_CCU_U_OP_SET_CHANNEL || + state.calls[6].op != TILEXR_CCU_U_OP_SET_XN || + state.calls[7].op != TILEXR_CCU_U_OP_SET_CKE || + state.calls[8].op != TILEXR_CCU_U_OP_SET_INSTRUCTION) { + std::cerr << "lower-layer-first call sequence mismatch\n"; + return 10; + } + for (size_t i = 9; i < state.calls.size(); ++i) { + if (state.calls[i].op != TILEXR_CCU_U_OP_GET_INSTRUCTION) { + std::cerr << "repository failure diagnostic readback sequence mismatch\n"; + return 10; + } + } + if (!state.freed || receipt.deviceInstructionPtr != nullptr || receipt.installed) { + std::cerr << "failed repository install did not release uploaded image\n"; + return 11; + } + const std::string summaryNeedles[] = { + "lowerLayerPreconditions{", + "msidTokenCount=1", + "pfeCount=1", + "jettyCount=1", + "channelCount=3", + "xnClearCount=1", + "ckeClearCount=1", + "localXnInstalled=1", + "notifyCkeInstalled=1", + "channelBindingInstalled=1", + "msidToken0{dieId=1 msId=69 tokenId=0x1234 tokenValue=0x5678}", + "pfe0{dieId=1 offset=2}", + "jetty0{dieId=1 startJettyCtxId=9 ctxCount=1}", + "channel0{dieId=1 channelId=2}", + "xnClear0{dieId=1 startXnId=1961 count=3}", + "ckeClear0{dieId=1 startCkeId=332 count=3}", + "remoteXn0{dieId=1 channelId=2 localXn=1961 remoteXn=2361 notifyCke=364", + }; + for (const auto& needle : summaryNeedles) { + if (!Contains(installReport.message, needle)) { + std::cerr << "missing lower-layer precondition summary: " << needle + << " in " << installReport.message << "\n"; + return 12; + } + } + if (!Contains(installReport.message, "CCU custom channel operation failed op=") || + !Contains(installReport.message, "opRet=81")) { + std::cerr << "repository failure context is missing: " << installReport.message << "\n"; + return 13; + } + if (!Contains(installReport.localXn.message, "lowerLayerPreconditions{") || + !Contains(installReport.remoteXn.message, "lowerLayerPreconditions{") || + !Contains(installReport.notifyCke.message, "lowerLayerPreconditions{") || + !Contains(installReport.channelBinding.message, "lowerLayerPreconditions{")) { + std::cerr << "lower-layer step messages lost precondition summary\n"; + return 14; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_remote_xn_proof_checks_remote_notify_and_local_wait_cke(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel(uint32_t, const TileXRCcuCustomChannelIn&, TileXRCcuCustomChannelOut* out, void*) + { + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 4}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 3; + resource.localWaitCke = 332; + resource.localWaitMask = 1; + resource.remoteNotifyMask = 1; + plan.syncResources.push_back(resource); + + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeLowerLayer(uint16_t remoteNotifyCke, uint16_t localWaitCke) + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2; + channel.ctx.raw[0] = 0xc0; + lowerLayer.channels.push_back(channel); + + lowerLayer.xnClears.push_back({1, 1961, 1}); + lowerLayer.ckeClears.push_back({1, 332, 4}); + + TileXRCcuRemoteXnBindingProof proof; + proof.dieId = 1; + proof.channelId = 2; + proof.localXn = 1961; + proof.remoteXn = 2361; + proof.notifyCke = remoteNotifyCke; + proof.localWaitCke = localWaitCke; + proof.peerRank = 0; + proof.peerExchangeObserved = true; + proof.endpointRouteVerified = true; + proof.channelResourceOwnerVerified = true; + proof.transportResourceExchangeVerified = true; + lowerLayer.remoteXnBindings.push_back(proof); + return lowerLayer; + } + + int RunInstall( + const TileXRCcuLaunchPackage& package, + const TileXRCcuInstallManifest& manifest, + const TileXRCcuLowerLayerInstallPlan& lowerLayer, + TileXRCcuInstallProviderReport* report) + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + return TILEXR_ERROR_INTERNAL; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + TileXRCcuHardwareInstallEvidence evidence; + return TileXRCcuInstallHardware(request, &evidence, report); + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + TileXRCcuInstallProviderReport staleLocalWait; + if (RunInstall(package, manifest, MakeLowerLayer(364, 333), &staleLocalWait) != + TILEXR_ERROR_NOT_FOUND) { + std::cerr << "stale local wait CKE proof was accepted\n"; + return 4; + } + if (staleLocalWait.remoteXn.satisfied || + staleLocalWait.remoteXn.message.find("local wait CKE") == std::string::npos) { + std::cerr << "local wait CKE diagnostic was weak: " + << staleLocalWait.remoteXn.message << "\n"; + return 5; + } + + TileXRCcuInstallProviderReport staleRemoteNotify; + if (RunInstall(package, manifest, MakeLowerLayer(365, 332), &staleRemoteNotify) != + TILEXR_ERROR_NOT_FOUND) { + std::cerr << "stale remote notify CKE proof was accepted\n"; + return 6; + } + if (staleRemoteNotify.remoteXn.satisfied || + staleRemoteNotify.remoteXn.message.find("remote notify CKE") == std::string::npos) { + std::cerr << "remote notify CKE diagnostic was weak: " + << staleRemoteNotify.remoteXn.message << "\n"; + return 7; + } + + TileXRCcuInstallProviderReport ok; + if (RunInstall(package, manifest, MakeLowerLayer(364, 332), &ok) != TILEXR_SUCCESS || + !ok.remoteXn.satisfied) { + std::cerr << "complete dual CKE proof was rejected: " << ok.message << "\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_online_install_provider_rejects_stale_mission_launch_descriptor_proof(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + uint32_t observedOp = 0; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->observedOp = in.op; + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + + package.tasks[0].key ^= 0x1U; + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "stale mission descriptor should leave install incomplete: " + << installReport.message << "\n"; + return 5; + } + if (!installReport.repository.satisfied || installReport.mission.satisfied || + !evidence.repositoryInstalled || evidence.missionInstalled) { + std::cerr << "stale mission descriptor was promoted\n"; + return 6; + } + if (installReport.publicVerifiedInstallSurfaceCount != 1 || + installReport.missingInstallSurfaceCount != 5) { + std::cerr << "stale descriptor install counts mismatch\n"; + return 7; + } + if (evidence.missionSource.kind != TileXRCcuEvidenceKind::Missing || + installReport.mission.message.find("launch task descriptor") == std::string::npos || + installReport.mission.message.find("mission key") == std::string::npos) { + std::cerr << "stale descriptor diagnostic is weak: " + << installReport.mission.message << "\n"; + return 8; + } + if (state.observedOp != TILEXR_CCU_U_OP_SET_INSTRUCTION) { + std::cerr << "repository install did not run before descriptor proof\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_evidence_adapter_keeps_offline_candidates_not_submit_ready(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuInstallStepEvidence Candidate(const char* source, const char* detail) + { + TileXRCcuInstallStepEvidence step; + step.satisfied = true; + step.source.kind = TileXRCcuEvidenceKind::PublicCandidate; + step.source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + step.source.source = source; + step.source.detail = detail; + step.message = detail; + return step; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 8; + } + + TileXRCcuInstallProviderReport installReport; + installReport.offlineOnly = true; + installReport.mission = Candidate("scan:mission", "public-looking mission candidate"); + installReport.repository = Candidate("scan:repository", "public-looking repository candidate"); + installReport.localXn = Candidate("scan:local-xn", "public-looking local XN candidate"); + installReport.remoteXn = Candidate("scan:remote-xn", "public-looking remote XN candidate"); + installReport.notifyCke = Candidate("scan:notify-cke", "public-looking notify CKE candidate"); + installReport.channelBinding = Candidate("scan:channel", "public-looking channel candidate"); + + TileXRCcuHardwareInstallEvidence evidence; + if (TileXRCcuBuildInstallEvidence(installReport, &evidence) != TILEXR_SUCCESS) { + std::cerr << "adapter rejected candidate report\n"; + return 2; + } + if (!evidence.missionInstalled || !evidence.repositoryInstalled || !evidence.localXnInstalled || + !evidence.remoteXnBound || !evidence.notifyCkeInstalled || !evidence.channelBindingsInstalled) { + std::cerr << "adapter dropped candidate evidence unexpectedly\n"; + return 3; + } + if (evidence.missionSource.kind != TileXRCcuEvidenceKind::PublicCandidate || + evidence.channelBindingSource.kind != TileXRCcuEvidenceKind::PublicCandidate) { + std::cerr << "adapter upgraded/downgraded candidate evidence unexpectedly\n"; + return 4; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "offline public candidates became submit-ready\n"; + return 5; + } + if (!submitTasks.empty() || providerReport.submitReady) { + std::cerr << "failed candidate prepare leaked submit-ready state\n"; + return 6; + } + if (providerReport.message.find("candidate") == std::string::npos || + providerReport.message.find("public install provider") == std::string::npos) { + std::cerr << "weak candidate diagnostic: " << providerReport.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_evidence_adapter_does_not_accept_offline_public_verified_claims(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuInstallStepEvidence ForgedVerified(const char* source, const char* detail) + { + TileXRCcuInstallStepEvidence step; + step.satisfied = true; + step.source.kind = TileXRCcuEvidenceKind::PublicVerified; + step.source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + step.source.source = source; + step.source.detail = detail; + step.message = detail; + return step; + } + + int main() + { + TileXRCcuInstallProviderReport installReport; + installReport.offlineOnly = true; + installReport.mission = ForgedVerified("forged:mission", "forged mission/key evidence"); + installReport.repository = ForgedVerified("forged:repository", "forged repository evidence"); + installReport.localXn = ForgedVerified("forged:local-xn", "forged local XN evidence"); + installReport.remoteXn = ForgedVerified("forged:remote-xn", "forged remote XN evidence"); + installReport.notifyCke = ForgedVerified("forged:notify-cke", "forged notify CKE evidence"); + installReport.channelBinding = ForgedVerified("forged:channel", "forged channel binding evidence"); + + TileXRCcuHardwareInstallEvidence evidence; + if (TileXRCcuBuildInstallEvidence(installReport, &evidence) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "offline public verified install claims were accepted\n"; + return 1; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed offline verified adapter left install evidence\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_evidence_adapter_requires_consistent_online_receipt(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuInstallStepEvidence VerifiedStep( + const char* source, + const char* detail, + uint64_t receipt) + { + TileXRCcuInstallStepEvidence step; + step.satisfied = true; + step.source.kind = TileXRCcuEvidenceKind::PublicVerified; + step.source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + step.source.packageFingerprint = 0xfeed001ULL; + step.source.deviceId = 3; + step.source.rank = 1; + step.source.provider = "unit-test-public-install-provider"; + step.source.source = source; + step.source.detail = detail; + step.source.installAttemptReceiptId = receipt; + step.message = detail; + return step; + } + + TileXRCcuInstallProviderReport VerifiedReport(uint64_t receipt) + { + TileXRCcuInstallProviderReport report; + report.offlineOnly = false; + report.installAttempted = true; + report.installSucceeded = true; + report.installAttemptReceiptId = receipt; + report.mission = VerifiedStep("provider:mission", "mission/key installed", receipt); + report.repository = VerifiedStep("provider:repository", "repository installed", receipt); + report.localXn = VerifiedStep("provider:local-xn", "local XN installed", receipt); + report.remoteXn = VerifiedStep("provider:remote-xn", "remote XN bound", receipt); + report.notifyCke = VerifiedStep("provider:notify-cke", "notify CKE installed", receipt); + report.channelBinding = VerifiedStep("provider:channel", "channel binding installed", receipt); + return report; + } + + int main() + { + TileXRCcuHardwareInstallEvidence evidence; + + TileXRCcuInstallProviderReport missingReceipt = VerifiedReport(0); + if (TileXRCcuBuildInstallEvidence(missingReceipt, &evidence) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "online public verified report with missing receipt was accepted\n"; + return 1; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed missing receipt adapter left install evidence\n"; + return 2; + } + + TileXRCcuInstallProviderReport mixedReceipt = VerifiedReport(0xabc001ULL); + mixedReceipt.remoteXn.source.installAttemptReceiptId = 0xabc002ULL; + if (TileXRCcuBuildInstallEvidence(mixedReceipt, &evidence) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "online public verified report with mixed receipts was accepted\n"; + return 3; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed mixed receipt adapter left install evidence\n"; + return 4; + } + + TileXRCcuInstallProviderReport notAttempted = VerifiedReport(0xabc001ULL); + notAttempted.installAttempted = false; + if (TileXRCcuBuildInstallEvidence(notAttempted, &evidence) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "online public verified report without attempted install was accepted\n"; + return 5; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed not-attempted adapter left install evidence\n"; + return 6; + } + + TileXRCcuInstallProviderReport notSucceeded = VerifiedReport(0xabc001ULL); + notSucceeded.installSucceeded = false; + if (TileXRCcuBuildInstallEvidence(notSucceeded, &evidence) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "online public verified report without successful install was accepted\n"; + return 7; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed not-succeeded adapter left install evidence\n"; + return 8; + } + + TileXRCcuInstallProviderReport matchingReceipt = VerifiedReport(0xabc001ULL); + if (TileXRCcuBuildInstallEvidence(matchingReceipt, &evidence) != TILEXR_SUCCESS) { + std::cerr << "matching receipt report was rejected\n"; + return 9; + } + if (evidence.missionSource.installAttemptReceiptId != 0xabc001ULL || + evidence.repositorySource.installAttemptReceiptId != 0xabc001ULL || + evidence.localXnSource.installAttemptReceiptId != 0xabc001ULL || + evidence.remoteXnSource.installAttemptReceiptId != 0xabc001ULL || + evidence.notifyCkeSource.installAttemptReceiptId != 0xabc001ULL || + evidence.channelBindingSource.installAttemptReceiptId != 0xabc001ULL) { + std::cerr << "adapter did not preserve matching receipt ids\n"; + return 10; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_manifest_from_launch_package_lists_all_required_surfaces(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + const TileXRCcuInstallRequirement* FindRequirement( + const TileXRCcuInstallManifest& manifest, + TileXRCcuInstallRequirementKind kind) + { + for (const auto& requirement : manifest.requirements) { + if (requirement.kind == kind) { + return &requirement; + } + } + return nullptr; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + const uint64_t fingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (manifest.deviceId != 3 || manifest.rank != 1 || + manifest.provider != "unit-test-public-install-provider" || + manifest.packageFingerprint != fingerprint || + !manifest.requiresHardwareInstall || + !manifest.installAttemptReceiptRequired || + manifest.requiredEvidenceKind != TileXRCcuEvidenceKind::PublicVerified || + manifest.requiredEvidenceSurface != TileXRCcuEvidenceSurface::PublicInstallProvider) { + std::cerr << "manifest did not preserve scope/fingerprint/evidence contract\n"; + return 4; + } + if (manifest.requirements.size() != 6 || manifestReport.requirementCount != 6 || + manifestReport.message != "ok") { + std::cerr << "manifest did not list the six required install surfaces\n"; + return 5; + } + + const TileXRCcuInstallRequirement* mission = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::MissionKey); + const TileXRCcuInstallRequirement* repository = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::RepositoryImage); + const TileXRCcuInstallRequirement* localXn = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::LocalXn); + const TileXRCcuInstallRequirement* remoteXn = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::RemoteXnBinding); + const TileXRCcuInstallRequirement* notifyCke = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::NotifyCke); + const TileXRCcuInstallRequirement* channel = + FindRequirement(manifest, TileXRCcuInstallRequirementKind::ChannelBinding); + if (mission == nullptr || repository == nullptr || localXn == nullptr || + remoteXn == nullptr || notifyCke == nullptr || channel == nullptr) { + std::cerr << "manifest missed one or more required install kinds\n"; + return 6; + } + if (mission->missionId != 6 || mission->missionKey != 0x059b0f03U || + mission->resourceStartId != 6 || mission->resourceCount != 1) { + std::cerr << "manifest mission/key requirement is incomplete\n"; + return 7; + } + if (repository->repositoryStartId != 475 || repository->repositoryCount != 170 || + repository->missionStartId != 489 || repository->missionCount != 156 || + repository->instructionCount != 170) { + std::cerr << "manifest repository requirement is incomplete\n"; + return 8; + } + if (localXn->resourceStartId != 1961 || localXn->resourceCount != 62) { + std::cerr << "manifest local XN requirement is incomplete\n"; + return 9; + } + if (remoteXn->syncResourceCount != 3 || notifyCke->syncResourceCount != 3 || + channel->syncResourceCount != 3 || channel->bindingCount != 9) { + std::cerr << "manifest sync resource requirements are incomplete\n"; + return 10; + } + for (const auto& requirement : manifest.requirements) { + if (requirement.packageFingerprint != fingerprint || + requirement.label.empty() || + requirement.detail.empty()) { + std::cerr << "manifest requirement missed audit metadata\n"; + return 11; + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_manifest_rejects_unbound_or_stale_scope_without_partial_output(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int ExpectManifestFailure( + const TileXRCcuLaunchPackage& package, + const char* diagnostic) + { + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport report; + const int ret = TileXRCcuBuildInstallManifest(package, &manifest, &report); + if (ret != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "manifest build unexpectedly succeeded for " << diagnostic << "\n"; + return 1; + } + if (report.message.find(diagnostic) == std::string::npos) { + std::cerr << "weak manifest diagnostic: " << report.message << "\n"; + return 2; + } + if (!manifest.requirements.empty() || report.requirementCount != 0) { + std::cerr << "failed manifest build left partial requirements\n"; + return 3; + } + return 0; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (ExpectManifestFailure(package, "launch install scope is not bound") != 0) { + return 2; + } + + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 3; + } + package.tasks[0].key ^= 0x1U; + if (ExpectManifestFailure(package, "launch install scope is stale") != 0) { + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_manifest_contract_does_not_make_default_provider_submit_ready(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "default install provider accepted manifest as installed evidence\n"; + return 4; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "manifest leaked into install evidence\n"; + return 5; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "manifest-only default provider became submit-ready\n"; + return 6; + } + if (!submitTasks.empty() || providerReport.submitReady || + providerReport.missingEvidenceCount == 0) { + std::cerr << "failed manifest-only prepare leaked submit-ready state\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_manifest_default_provider_reports_missing_required_surfaces(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + + if (TileXRCcuInstallHardware(request, &evidence, &installReport) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "default provider did not stop at unsupported install path\n"; + return 4; + } + if (installReport.requiredInstallSurfaceCount != 6 || + installReport.publicVerifiedInstallSurfaceCount != 0 || + installReport.missingInstallSurfaceCount != 6) { + std::cerr << "manifest requirement counts were not reported: required=" + << installReport.requiredInstallSurfaceCount + << " verified=" << installReport.publicVerifiedInstallSurfaceCount + << " missing=" << installReport.missingInstallSurfaceCount << "\n"; + return 5; + } + if (installReport.message.find("no public no-hcomm CCU install provider") == std::string::npos) { + std::cerr << "weak unsupported provider diagnostic: " << installReport.message << "\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_request_rejects_mismatched_manifest_scope(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int ExpectManifestFailure( + const TileXRCcuInstallRequest& request, + const char* diagnostic) + { + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport report; + const int ret = TileXRCcuInstallHardware(request, &evidence, &report); + if (ret != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "manifest mismatch was not rejected: " << diagnostic << "\n"; + return 1; + } + if (report.message.find(diagnostic) == std::string::npos) { + std::cerr << "weak manifest mismatch diagnostic: " << report.message << "\n"; + return 2; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed manifest mismatch left install evidence\n"; + return 3; + } + return 0; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + + TileXRCcuInstallManifest wrongDevice = manifest; + wrongDevice.deviceId = 4; + request.manifest = &wrongDevice; + if (ExpectManifestFailure(request, "install manifest device mismatch") != 0) { + return 4; + } + + TileXRCcuInstallManifest wrongFingerprint = manifest; + wrongFingerprint.packageFingerprint ^= 0x1ULL; + request.manifest = &wrongFingerprint; + if (ExpectManifestFailure(request, "install manifest fingerprint mismatch") != 0) { + return 5; + } + + TileXRCcuInstallManifest wrongRequirementKind = manifest; + wrongRequirementKind.requirements[5].kind = wrongRequirementKind.requirements[0].kind; + request.manifest = &wrongRequirementKind; + if (ExpectManifestFailure(request, "install manifest requirement kind mismatch") != 0) { + return 6; + } + + TileXRCcuInstallManifest wrongHardwareRequirement = manifest; + wrongHardwareRequirement.requiresHardwareInstall = false; + request.manifest = &wrongHardwareRequirement; + if (ExpectManifestFailure(request, "install manifest hardware requirement mismatch") != 0) { + return 7; + } + + TileXRCcuInstallManifest wrongMission = manifest; + wrongMission.requirements[0].missionKey ^= 0x1U; + request.manifest = &wrongMission; + if (ExpectManifestFailure(request, "install manifest mission requirement mismatch") != 0) { + return 8; + } + + TileXRCcuInstallManifest wrongChannel = manifest; + wrongChannel.requirements[5].bindingCount = 1; + request.manifest = &wrongChannel; + if (ExpectManifestFailure(request, "install manifest channel requirement mismatch") != 0) { + return 9; + } + + request.manifest = &manifest; + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport report; + if (TileXRCcuInstallHardware(request, &evidence, &report) != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "matching manifest should reach unsupported provider: " << report.message << "\n"; + return 10; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_evidence_adapter_rejects_public_verified_with_bad_surface_or_scope(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuInstallStepEvidence VerifiedStep( + const char* source, + const char* detail, + uint64_t receipt) + { + TileXRCcuInstallStepEvidence step; + step.satisfied = true; + step.source.kind = TileXRCcuEvidenceKind::PublicVerified; + step.source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + step.source.packageFingerprint = 0xfeed001ULL; + step.source.deviceId = 3; + step.source.rank = 1; + step.source.provider = "unit-test-public-install-provider"; + step.source.installAttemptReceiptId = receipt; + step.source.source = source; + step.source.detail = detail; + step.message = detail; + return step; + } + + TileXRCcuInstallProviderReport VerifiedReport(uint64_t receipt) + { + TileXRCcuInstallProviderReport report; + report.offlineOnly = false; + report.installAttempted = true; + report.installSucceeded = true; + report.installAttemptReceiptId = receipt; + report.mission = VerifiedStep("provider:mission", "mission/key installed", receipt); + report.repository = VerifiedStep("provider:repository", "repository installed", receipt); + report.localXn = VerifiedStep("provider:local-xn", "local XN installed", receipt); + report.remoteXn = VerifiedStep("provider:remote-xn", "remote XN bound", receipt); + report.notifyCke = VerifiedStep("provider:notify-cke", "notify CKE installed", receipt); + report.channelBinding = VerifiedStep("provider:channel", "channel binding installed", receipt); + return report; + } + + int ExpectAdapterFailure(const TileXRCcuInstallProviderReport& report, const char* diagnostic) + { + TileXRCcuHardwareInstallEvidence evidence; + const int ret = TileXRCcuBuildInstallEvidence(report, &evidence); + if (ret != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid public verified evidence was accepted: " << diagnostic << "\n"; + return 1; + } + if (evidence.missionInstalled || evidence.repositoryInstalled || evidence.localXnInstalled || + evidence.remoteXnBound || evidence.notifyCkeInstalled || evidence.channelBindingsInstalled) { + std::cerr << "failed adapter left invalid public verified evidence\n"; + return 2; + } + return 0; + } + + int main() + { + TileXRCcuInstallProviderReport wrongSurface = VerifiedReport(0xabc001ULL); + wrongSurface.mission.source.surface = TileXRCcuEvidenceSurface::LowerLayerResourceHelper; + if (ExpectAdapterFailure(wrongSurface, "surface mismatch") != 0) { + return 1; + } + + TileXRCcuInstallProviderReport missingScope = VerifiedReport(0xabc001ULL); + missingScope.repository.source.packageFingerprint = 0; + if (ExpectAdapterFailure(missingScope, "missing fingerprint") != 0) { + return 2; + } + + TileXRCcuInstallProviderReport missingProvider = VerifiedReport(0xabc001ULL); + missingProvider.localXn.source.provider.clear(); + if (ExpectAdapterFailure(missingProvider, "missing provider") != 0) { + return 3; + } + + TileXRCcuInstallProviderReport missingDetail = VerifiedReport(0xabc001ULL); + missingDetail.remoteXn.source.detail.clear(); + if (ExpectAdapterFailure(missingDetail, "missing source/detail") != 0) { + return 4; + } + + TileXRCcuHardwareInstallEvidence evidence; + if (TileXRCcuBuildInstallEvidence(VerifiedReport(0xabc001ULL), &evidence) != TILEXR_SUCCESS) { + std::cerr << "valid public verified evidence was rejected\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_peer_exchange_without_endpoint_route_provenance_does_not_submit(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 3}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 1}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeSyntheticLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2; + channel.ctx.raw[0] = 0xc0; + lowerLayer.channels.push_back(channel); + + lowerLayer.xnClears.push_back({1, 1961, 1}); + lowerLayer.ckeClears.push_back({1, 332, 1}); + + TileXRCcuRemoteXnBindingProof proof; + proof.dieId = 1; + proof.channelId = 2; + proof.localXn = 1961; + proof.remoteXn = 2361; + proof.notifyCke = 364; + proof.peerRank = 0; + proof.peerExchangeObserved = true; + lowerLayer.remoteXnBindings.push_back(proof); + return lowerLayer; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeSyntheticLowerLayer(); + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + const int ret = TileXRCcuInstallHardware(request, &evidence, &installReport); + if (ret != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "synthetic endpoint route unexpectedly completed install: " + << installReport.message << "\n"; + return 5; + } + if (evidence.remoteXnBound || evidence.channelBindingsInstalled || + installReport.remoteXn.satisfied || installReport.channelBinding.satisfied || + installReport.publicVerifiedInstallSurfaceCount != 4 || + installReport.missingInstallSurfaceCount != 2) { + std::cerr << "synthetic endpoint route produced submit evidence\n"; + return 6; + } + if (installReport.remoteXn.message.find("endpoint route") == std::string::npos || + installReport.channelBinding.message.find("endpoint route") == std::string::npos) { + std::cerr << "weak synthetic endpoint diagnostic remote=\"" + << installReport.remoteXn.message << "\" channel=\"" + << installReport.channelBinding.message << "\"\n"; + return 7; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "synthetic endpoint route became submit-ready\n"; + return 8; + } + if (!submitTasks.empty() || providerReport.submitReady) { + std::cerr << "submit-ready state leaked from synthetic endpoint route\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_peer_exchange_with_endpoint_route_but_without_channel_resource_binding_does_not_submit(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 3}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 1; + resource.localWaitCke = 332; + plan.syncResources.push_back(resource); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeEndpointOnlyLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2; + channel.ctx.raw[0] = 0xc0; + lowerLayer.channels.push_back(channel); + + lowerLayer.xnClears.push_back({1, 1961, 1}); + lowerLayer.ckeClears.push_back({1, 332, 3}); + + TileXRCcuRemoteXnBindingProof proof; + proof.dieId = 1; + proof.channelId = 2; + proof.localXn = 1961; + proof.remoteXn = 2361; + proof.notifyCke = 364; + proof.peerRank = 0; + proof.peerExchangeObserved = true; + proof.localWaitCke = 332; + proof.endpointRouteVerified = true; + lowerLayer.remoteXnBindings.push_back(proof); + return lowerLayer; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeEndpointOnlyLowerLayer(); + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + const int ret = TileXRCcuInstallHardware(request, &evidence, &installReport); + if (ret != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "endpoint-only remote XN proof unexpectedly completed install: " + << installReport.message << "\n"; + return 5; + } + if (evidence.remoteXnBound || evidence.channelBindingsInstalled || + installReport.remoteXn.satisfied || installReport.channelBinding.satisfied) { + std::cerr << "endpoint-only remote XN proof produced submit evidence\n"; + return 6; + } + if (installReport.remoteXn.message.find("channel resource owner") == std::string::npos || + installReport.channelBinding.message.find("channel resource owner") == std::string::npos) { + std::cerr << "weak endpoint-only diagnostic remote=\"" + << installReport.remoteXn.message << "\" channel=\"" + << installReport.channelBinding.message << "\"\n"; + return 7; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "endpoint-only remote XN proof became submit-ready\n"; + return 8; + } + if (!submitTasks.empty() || providerReport.submitReady) { + std::cerr << "submit-ready state leaked from endpoint-only remote XN proof\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_peer_exchange_with_channel_owner_but_without_transport_exchange_does_not_submit(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_install_provider.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void*, void*) + { + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn&, + TileXRCcuCustomChannelOut* out, + void*) + { + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 3}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 1; + resource.localWaitCke = 332; + plan.syncResources.push_back(resource); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + return plan; + } + + TileXRCcuLowerLayerInstallPlan MakeLowerLayer() + { + TileXRCcuLowerLayerInstallPlan lowerLayer; + lowerLayer.msidTokens.push_back({1, 0x45, 0x1234, 0x5678}); + + TileXRCcuPfeInstall pfe; + pfe.dieId = 1; + pfe.pfeOffset = 2; + pfe.ctx.raw[0] = 0xa0; + lowerLayer.pfes.push_back(pfe); + + TileXRCcuJettyInstall jetty; + jetty.dieId = 1; + jetty.startJettyCtxId = 9; + jetty.ctxs.resize(1); + jetty.ctxs[0].raw[0] = 0xb0; + lowerLayer.jettys.push_back(jetty); + + TileXRCcuChannelInstall channel; + channel.dieId = 1; + channel.channelId = 2; + channel.ctx.raw[0] = 0xc0; + lowerLayer.channels.push_back(channel); + + lowerLayer.xnClears.push_back({1, 1961, 1}); + lowerLayer.ckeClears.push_back({1, 332, 3}); + + TileXRCcuRemoteXnBindingProof proof; + proof.dieId = 1; + proof.channelId = 2; + proof.localXn = 1961; + proof.remoteXn = 2361; + proof.notifyCke = 364; + proof.peerRank = 0; + proof.peerExchangeObserved = true; + proof.localWaitCke = 332; + proof.endpointRouteVerified = true; + proof.channelResourceOwnerVerified = true; + proof.transportResourceExchangeVerified = false; + lowerLayer.remoteXnBindings.push_back(proof); + return lowerLayer; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuInstallManifest manifest; + TileXRCcuInstallManifestReport manifestReport; + if (TileXRCcuBuildInstallManifest(package, &manifest, &manifestReport) != TILEXR_SUCCESS) { + std::cerr << "manifest build failed: " << manifestReport.message << "\n"; + return 3; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 4; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuLowerLayerInstallPlan lowerLayer = MakeLowerLayer(); + TileXRCcuRepositoryInstallReceipt receipt; + TileXRCcuInstallRequest request; + request.package = &package; + request.manifest = &manifest; + request.deviceId = 3; + request.rank = 1; + request.provider = "unit-test-public-install-provider"; + request.offlineOnly = false; + request.driverAdapter = &adapter; + request.repositoryMemoryOps = memoryOps; + request.repositoryMemoryUserData = &state; + request.repositoryReceipt = &receipt; + request.lowerLayerPlan = &lowerLayer; + + TileXRCcuHardwareInstallEvidence evidence; + TileXRCcuInstallProviderReport installReport; + const int ret = TileXRCcuInstallHardware(request, &evidence, &installReport); + if (ret != TILEXR_ERROR_NOT_FOUND) { + std::cerr << "missing transport exchange proof unexpectedly completed install: " + << installReport.message << "\n"; + return 5; + } + if (evidence.remoteXnBound || evidence.channelBindingsInstalled || + installReport.remoteXn.satisfied || installReport.channelBinding.satisfied) { + std::cerr << "missing transport exchange proof produced submit evidence\n"; + return 6; + } + if (installReport.remoteXn.message.find("transport resource exchange") == std::string::npos || + installReport.channelBinding.message.find("transport resource exchange") == std::string::npos) { + std::cerr << "weak transport-exchange diagnostic remote=\"" + << installReport.remoteXn.message << "\" channel=\"" + << installReport.channelBinding.message << "\"\n"; + return 7; + } + + std::vector submitTasks; + TileXRCcuProviderReport providerReport; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &providerReport) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing transport exchange proof became submit-ready\n"; + return 8; + } + if (!submitTasks.empty() || providerReport.submitReady) { + std::cerr << "submit-ready state leaked from missing transport exchange proof\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_install_provider_layer_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = INSTALL_HEADER.read_text(encoding="utf-8") + source = INSTALL_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_install_provider.h", cmake) + self.assertIn("ccu/tilexr_ccu_install_provider.cpp", cmake) + self.assertIn("struct TileXRCcuInstallRequest", header) + self.assertIn("struct TileXRCcuInstallStepEvidence", header) + self.assertIn("struct TileXRCcuInstallProviderReport", header) + self.assertIn("TileXRCcuBuildInstallEvidence", header) + self.assertIn("TileXRCcuInstallHardware", header) + self.assertIn("TileXRCcuHardwareInstallEvidence", header) + self.assertIn("deviceId", header) + self.assertIn("rank", header) + self.assertIn("provider", header) + self.assertIn("offlineOnly", header) + self.assertIn("driverAdapter", header) + self.assertIn("repositoryMemoryOps", header) + self.assertIn("repositoryMemoryUserData", header) + self.assertIn("repositoryInstallOptions", header) + self.assertIn("repositoryReceipt", header) + self.assertIn("TileXRCcuInstallOrder", header) + self.assertIn("installOrder", header) + self.assertIn("TileXRCcuLowerLayerInstallPlan", header) + self.assertIn("TileXRCcuMsidTokenInstall", header) + self.assertIn("TileXRCcuPfeInstall", header) + self.assertIn("TileXRCcuJettyInstall", header) + self.assertIn("TileXRCcuChannelInstall", header) + self.assertIn("TileXRCcuCkeClearInstall", header) + self.assertIn("TileXRCcuXnClearInstall", header) + self.assertIn("TileXRCcuRemoteXnBindingProof", header) + self.assertIn("localWaitCke", header) + self.assertIn("lowerLayerPlan", header) + self.assertIn("xnClears", header) + self.assertIn("remoteXnBindings", header) + self.assertIn("installAttempted", header) + self.assertIn("installSucceeded", header) + self.assertIn("installAttemptReceiptId", header) + self.assertIn("TileXRCcuInstallRepositoryImage", source) + self.assertIn("TileXRCcuInstallRepositoryImageWithOptions", source) + self.assertIn("InstallLowerLayerResources", source) + self.assertIn("InstallLowerLayerFirst", source) + self.assertIn("ValidateRemoteXnExchangeBindingProof", source) + self.assertIn("local wait CKE", source) + self.assertIn("remote notify CKE", source) + self.assertIn("InstallMsidToken", source) + self.assertIn("InstallPfeCtx", source) + self.assertIn("InstallJettyCtx", source) + self.assertIn("InstallChannelCtxV1", source) + self.assertIn("const uint32_t expectedChannelCount", source) + self.assertIn("CountInstalledRemoteBindingChannels", source) + self.assertIn("installedChannelCount >= expectedChannelCount", source) + self.assertIn("InstallXnRange", source) + self.assertIn("ClearCkeRange", source) + self.assertIn("BuildRepositoryInstallReceiptId", source) + self.assertIn("repository instruction image installed via SET_INSTRUCTION", source) + self.assertIn("ValidateInstallRequestScope", source) + self.assertIn("ValidateInstallReceipt", source) + self.assertIn("install attempt did not succeed", source) + self.assertIn("device scope mismatch", source) + self.assertIn("rank scope mismatch", source) + self.assertIn("provider scope mismatch", source) + self.assertIn("no public no-hcomm CCU install provider", source) + self.assertIn("offline install evidence cannot be public verified", source) + self.assertIn("install attempt receipt", source) + self.assertIn("RejectOfflinePublicVerified", source) + self.assertIn("TileXRCcuInstallManifest", header) + self.assertIn("TileXRCcuInstallRequirement", header) + self.assertIn("TileXRCcuBuildInstallManifest", header) + self.assertIn("ValidateInstallManifestScope", source) + self.assertIn("ValidateInstallRequestManifest", source) + self.assertIn("install manifest fingerprint mismatch", source) + self.assertIn("install manifest requirement kind mismatch", source) + self.assertIn("install manifest hardware requirement mismatch", source) + self.assertIn("install manifest mission requirement mismatch", source) + self.assertIn("ValidateMissionLaunchDescriptorProof", source) + self.assertIn("launch task descriptor", source) + self.assertIn("install manifest channel requirement mismatch", source) + self.assertIn("ValidatePublicVerifiedStepScope", source) + self.assertIn("public verified evidence scope is incomplete", source) + self.assertIn("launch install scope is stale", source) + + combined = header + "\n" + source + self.assertNotIn("SET_MISSION_CTX", combined) + self.assertNotIn("TILEXR_CCU_U_OP_SET_MISSION_CTX", combined) + self.assertNotIn("258", combined) + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_launch_package.py b/tests/ccu/test_tilexr_ccu_launch_package.py new file mode 100644 index 00000000..dda0ace7 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_launch_package.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PACKAGE_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.h" +PACKAGE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.cpp" +REPOSITORY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_repository.cpp" +DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" +PRODUCER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.cpp" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL, 0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(plan, &package, &report) != TILEXR_SUCCESS) { + std::cerr << "launch package build failed: " << report.message << "\n"; + return 1; + } + + if (package.tasks.size() != 2 || package.program.sqeLoad.size() != 13 || + package.program.sync.size() != 11 || package.repository.instructions.size() != 170) { + std::cerr << "unexpected package sizes\n"; + return 2; + } + if (package.tasks[0].instStartId != 489 || package.tasks[0].instCnt != 13 || + package.tasks[0].argSize != 13 || package.tasks[0].key != 0x059b0f03U || + package.tasks[0].args[0] != 0x100051152e00ULL) { + std::cerr << "unexpected sqe-load mission task\n"; + return 3; + } + if (package.tasks[1].instStartId != 502 || package.tasks[1].instCnt != 143 || + package.tasks[1].argSize != 13 || package.tasks[1].key != 0x059b0f03U) { + std::cerr << "unexpected sync mission task\n"; + return 4; + } + if (package.repository.sqeLoadOffset != 14 || package.repository.syncOffset != 27 || + package.repository.instructions[14].words[0] != 0x0000000007a90001ULL || + package.repository.instructions[27].words[0] != 0x0000000007a90001ULL || + package.repository.instructions[31].words[0] != 0x0001016c00000802ULL || + package.repository.instructions[32].words[0] != 0x000007a90939100dULL || + package.repository.instructions[35].words[0] != 0x0000000000010802ULL) { + std::cerr << "unexpected repository image\n"; + return 5; + } + if (!package.requiresHardwareInstall) { + std::cerr << "package should still require hardware install\n"; + return 6; + } + if (report.taskCount != 2 || report.repositoryCount != 170 || + report.installedInstructionCount != 24 || report.message != "ok") { + std::cerr << "unexpected package report\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_rejects_invalid_inputs_without_partial_outputs(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_launch_package.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(plan, nullptr, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null package output accepted\n"; + return 1; + } + if (report.message.find("package") == std::string::npos) { + std::cerr << "null output diagnostic was weak: " << report.message << "\n"; + return 2; + } + + TileXRCcuLaunchPackage package; + plan.syncResources[0].bindingCount = 0; + if (TileXRCcuBuildLaunchPackage(plan, &package, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid producer state accepted\n"; + return 3; + } + if (!package.tasks.empty() || !package.repository.instructions.empty()) { + std::cerr << "failed build left partial package state\n"; + return 4; + } + if (report.message.find("binding") == std::string::npos) { + std::cerr << "invalid producer diagnostic was weak: " << report.message << "\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_builds_pure_barrier_without_sqe_load_task(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_launch_package.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {0, 1, 0x12345678U, true}; + plan.kernelLocalXn = {0, 1, 1}; + plan.kernelLocalCke = {0, 1, 1}; + plan.kernelLocalMission = {0, 1, 1}; + plan.instructionWindow = {0, 1, 2, 1, 2}; + plan.syncResources.push_back({0, 1, 2, 1, 1, 1, 1, 1, 1}); + plan.taskWindows.push_back({0, 1, 2, 13, {}}); + + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(plan, &package, &report) != TILEXR_SUCCESS) { + std::cerr << "pure barrier package build failed: " << report.message << "\n"; + return 1; + } + if (package.tasks.size() != 1 || !package.program.sqeLoad.empty() || + package.program.sync.size() != 2 || package.repository.instructions.size() != 2) { + std::cerr << "pure barrier package size mismatch\n"; + return 2; + } + if (package.tasks[0].instStartId != 1 || package.tasks[0].instCnt != 2 || + package.tasks[0].argSize != 13 || package.tasks[0].args[0] != 0) { + std::cerr << "pure barrier task mismatch\n"; + return 3; + } + if (package.repository.sqeLoadCount != 0 || package.repository.syncOffset != 0 || + package.repository.syncCount != 2 || + package.repository.instructions[0].words[0] != package.program.sync[0].words[0] || + package.repository.instructions[0].words[1] != package.program.sync[0].words[1] || + package.repository.instructions[1].words[0] != package.program.sync[1].words[0] || + package.repository.instructions[1].words[1] != package.program.sync[1].words[1]) { + std::cerr << "pure barrier repository mismatch\n"; + return 4; + } + if (report.taskCount != 1 || report.installedInstructionCount != 2 || + report.message != "ok") { + std::cerr << "pure barrier report mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_install_scope_records_current_package_fingerprint(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_launch_package.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + if (TileXRCcuBindLaunchPackageInstallScope( + nullptr, 3, 1, "unit-test-public-install-provider") != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null package scope bind was accepted\n"; + return 1; + } + + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &report) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << report.message << "\n"; + return 2; + } + if (TileXRCcuBindLaunchPackageInstallScope(&package, 3, 1, "") != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty provider scope bind was accepted\n"; + return 3; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 4; + } + + const uint64_t fingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (package.installScope.deviceId != 3 || + package.installScope.rank != 1 || + package.installScope.provider != "unit-test-public-install-provider" || + package.installScope.packageFingerprint != fingerprint) { + std::cerr << "unexpected install scope binding\n"; + return 5; + } + + package.tasks[0].key ^= 0x1U; + if (TileXRCcuComputeLaunchPackageFingerprint(package) == fingerprint) { + std::cerr << "mutated package kept old fingerprint\n"; + return 6; + } + if (package.installScope.packageFingerprint == TileXRCcuComputeLaunchPackageFingerprint(package)) { + std::cerr << "install scope fingerprint silently tracked mutation\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_fingerprint_changes_for_local_wait_cke(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_launch_package.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan(uint16_t localWaitCke) + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 4}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 3; + resource.localWaitCke = localWaitCke; + resource.localWaitMask = 1; + resource.remoteNotifyMask = 1; + plan.syncResources.push_back(resource); + + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage packageA; + TileXRCcuLaunchPackage packageB; + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(MakePlan(332), &packageA, &report) != TILEXR_SUCCESS || + TileXRCcuBuildLaunchPackage(MakePlan(333), &packageB, &report) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << report.message << "\n"; + return 1; + } + const uint64_t fingerprintA = TileXRCcuComputeLaunchPackageFingerprint(packageA); + const uint64_t fingerprintB = TileXRCcuComputeLaunchPackageFingerprint(packageB); + if (fingerprintA == 0 || fingerprintB == 0 || fingerprintA == fingerprintB) { + std::cerr << "local wait CKE did not affect launch package fingerprint\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_fingerprint_changes_for_kernel_local_gsa(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_launch_package.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan(uint16_t gsaStart) + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalGsa = {1, gsaStart, 1}; + plan.kernelLocalCke = {1, 332, 4}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3, 332, 1, 1}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage packageA; + TileXRCcuLaunchPackage packageB; + TileXRCcuLaunchPackageReport report; + if (TileXRCcuBuildLaunchPackage(MakePlan(510), &packageA, &report) != TILEXR_SUCCESS || + TileXRCcuBuildLaunchPackage(MakePlan(511), &packageB, &report) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << report.message << "\n"; + return 1; + } + const uint64_t fingerprintA = TileXRCcuComputeLaunchPackageFingerprint(packageA); + const uint64_t fingerprintB = TileXRCcuComputeLaunchPackageFingerprint(packageB); + if (fingerprintA == 0 || fingerprintB == 0 || fingerprintA == fingerprintB) { + std::cerr << "kernel-local GSA did not affect launch package fingerprint\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_launch_package_layer_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = PACKAGE_HEADER.read_text(encoding="utf-8") + source = PACKAGE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_launch_package.h", cmake) + self.assertIn("ccu/tilexr_ccu_launch_package.cpp", cmake) + self.assertIn("struct TileXRCcuLaunchPackage", header) + self.assertIn("struct TileXRCcuLaunchPackageReport", header) + self.assertIn("TileXRCcuBuildLaunchPackage", header) + self.assertIn("TileXRCcuLaunchInstallScope", header) + self.assertIn("installScope", header) + self.assertIn("TileXRCcuBindLaunchPackageInstallScope", header) + self.assertIn("requiresHardwareInstall", header) + self.assertIn("tilexr_ccu_repository.h", header) + self.assertIn("resource.localWaitCke", source) + self.assertIn("resource.localWaitMask", source) + self.assertIn("resource.remoteNotifyMask", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_payloads.py b/tests/ccu/test_tilexr_ccu_lower_layer_payloads.py new file mode 100644 index 00000000..cba54f76 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_lower_layer_payloads.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PAYLOAD_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_payloads.h" +PAYLOAD_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_payloads.cpp" +ABI_CONSTANTS_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_abi_constants.h" +HCOMM_ORACLE_SOURCE = REPO_ROOT / "tests" / "ccu" / "ccu_lower_layer_payload_hcomm_oracle.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuLowerLayerPayloadsTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "lower_layer_payloads_test.cpp" + test_bin = temp_path / "lower_layer_payloads_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(PAYLOAD_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_packers_match_lower_layer_pfe_jetty_and_channel_v1_layouts(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_payloads.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + int main() + { + TileXRCcuLowerLayerPayloadReport report; + + TileXRCcuPfeCtx pfe; + TileXRCcuPfeCtxSpec pfeSpec; + pfeSpec.startJettyId = 0x1234; + pfeSpec.jettyCount = 5; + pfeSpec.startLocalJettyCtxId = 0x22; + if (TileXRCcuBuildPfeCtx(pfeSpec, &pfe, &report) != TILEXR_SUCCESS) { + std::cerr << "pfe build failed: " << report.message << "\n"; + return 1; + } + if (Read16(pfe.raw, 0) != 0x1234 || + Read16(pfe.raw, 2) != static_cast(4U | (0x22U << 7U)) || + Read16(pfe.raw, 4) != 0 || Read16(pfe.raw, 6) != 0) { + std::cerr << "pfe layout mismatch\n"; + return 2; + } + + TileXRCcuLocalJettyCtxData jetty; + TileXRCcuLocalJettyCtxSpec jettySpec; + jettySpec.dieId = 1; + jettySpec.pfeId = 3; + jettySpec.doorbellVa = 0x1122334455667788ULL; + jettySpec.doorbellTokenId = 0x000abcdeU; + jettySpec.doorbellTokenValue = 0x89abcdefU; + jettySpec.sqDepth = 16; + jettySpec.wqeBasicBlockStartId = 0x9a; + if (TileXRCcuBuildLocalJettyCtx(jettySpec, &jetty, &report) != TILEXR_SUCCESS) { + std::cerr << "jetty build failed: " << report.message << "\n"; + return 3; + } + if (Read16(jetty.raw, 0) != 0x7788 || Read16(jetty.raw, 2) != 0x5566 || + Read16(jetty.raw, 4) != 0x3344 || Read16(jetty.raw, 6) != 0x1122 || + Read16(jetty.raw, 8) != 0xde73 || Read16(jetty.raw, 10) != 0xfabc || + Read16(jetty.raw, 12) != 0xbcde || Read16(jetty.raw, 14) != 0x689a || + Read16(jetty.raw, 16) != 0 || Read16(jetty.raw, 18) != 0 || + Read16(jetty.raw, 20) != 0 || Read16(jetty.raw, 22) != 0xa000 || + Read16(jetty.raw, 24) != 0x0009 || Read16(jetty.raw, 26) != 0 || + Read16(jetty.raw, 28) != 0 || Read16(jetty.raw, 30) != 0) { + std::cerr << "jetty layout mismatch\n"; + return 4; + } + + TileXRCcuChannelCtxDataV1 channel; + TileXRCcuChannelCtxV1Spec channelSpec; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + channelSpec.remoteEid[i] = static_cast(0x10 + i); + } + channelSpec.tpn = 0x00ab5678U; + channelSpec.sourcePfeId = 5; + channelSpec.startJettyId = 0x0234; + channelSpec.jettyCount = 7; + channelSpec.dieId = 1; + channelSpec.memoryTokenId = 0x000abcdeU; + channelSpec.memoryTokenValue = 0x89abcdefU; + channelSpec.remoteCcuVa = 0x000123456789ab00ULL; + if (TileXRCcuBuildChannelCtxV1(channelSpec, &channel, &report) != TILEXR_SUCCESS) { + std::cerr << "channel build failed: " << report.message << "\n"; + return 5; + } + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + if (channel.raw[i] != static_cast(0x10 + i)) { + std::cerr << "channel eid mismatch\n"; + return 6; + } + } + const uint64_t dstVa = channelSpec.remoteCcuVa >> TILEXR_CCU_REMOTE_CCU_VA_SHIFT; + if (Read16(channel.raw, 16) != 0x5678 || + Read16(channel.raw, 18) != 0x45ab || + Read16(channel.raw, 20) != 0x6023 || + Read16(channel.raw, 22) != 0xcde8 || + Read16(channel.raw, 24) != 0xefab || + Read16(channel.raw, 26) != 0xabcd || + Read16(channel.raw, 28) != static_cast(0x0089U | ((dstVa & 0xffU) << 8U)) || + Read16(channel.raw, 30) != static_cast((dstVa >> 8U) & 0xffffU) || + Read16(channel.raw, 32) != static_cast((dstVa >> 24U) & 0xffffU) || + Read16(channel.raw, 34) != static_cast(((dstVa >> 40U) & 0x1U) | 0x2U) || + Read16(channel.raw, 36) != 0 || Read16(channel.raw, 62) != 0) { + std::cerr << "channel layout mismatch\n"; + return 7; + } + + if (TileXRCcuBuildPfeCtx({0, 0, 0}, &pfe, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuBuildLocalJettyCtx({}, &jetty, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuBuildChannelCtxV1({}, &channel, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid lower-layer payload specs accepted\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_packers_accept_plaintext_zero_token_values_from_tilexr_udma(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_payloads.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + int main() + { + TileXRCcuLowerLayerPayloadReport report; + + TileXRCcuLocalJettyCtxData jetty; + TileXRCcuLocalJettyCtxSpec jettySpec; + jettySpec.dieId = 0; + jettySpec.pfeId = 2; + jettySpec.doorbellVa = 0x1020304050607080ULL; + jettySpec.doorbellTokenId = 0x12345U; + jettySpec.doorbellTokenValue = 0; + jettySpec.sqDepth = 8; + if (TileXRCcuBuildLocalJettyCtx(jettySpec, &jetty, &report) != TILEXR_SUCCESS) { + std::cerr << "zero doorbell token value rejected: " << report.message << "\n"; + return 1; + } + if (Read16(jetty.raw, 10) != 0x0123 || + Read16(jetty.raw, 12) != 0 || + (Read16(jetty.raw, 14) & 0x0fffU) != 0) { + std::cerr << "zero doorbell token value packed incorrectly\n"; + return 2; + } + + jettySpec.doorbellTokenId = 0; + if (TileXRCcuBuildLocalJettyCtx(jettySpec, &jetty, &report) != TILEXR_SUCCESS) { + std::cerr << "zero doorbell token id rejected: " << report.message << "\n"; + return 5; + } + if ((Read16(jetty.raw, 8) & 0x0040U) == 0 || + (Read16(jetty.raw, 8) & 0xff00U) != 0 || + (Read16(jetty.raw, 10) & 0x0fffU) != 0) { + std::cerr << "zero doorbell token id packed incorrectly\n"; + return 6; + } + + TileXRCcuChannelCtxDataV1 channel; + TileXRCcuChannelCtxV1Spec channelSpec; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + channelSpec.remoteEid[i] = static_cast(0xa0 + i); + } + channelSpec.tpn = 0x13579U; + channelSpec.sourcePfeId = 2; + channelSpec.startJettyId = 0x44; + channelSpec.jettyCount = 1; + channelSpec.dieId = 0; + channelSpec.memoryTokenId = 0x12345U; + channelSpec.memoryTokenValue = 0; + channelSpec.remoteCcuVa = 0x0000001234000000ULL; + if (TileXRCcuBuildChannelCtxV1(channelSpec, &channel, &report) != TILEXR_SUCCESS) { + std::cerr << "zero memory token value rejected: " << report.message << "\n"; + return 7; + } + if ((Read16(channel.raw, 24) & 0xff00U) != 0 || + Read16(channel.raw, 26) != 0 || + (Read16(channel.raw, 28) & 0x00ffU) != 0 || + (Read16(channel.raw, 34) & 0x2U) == 0) { + std::cerr << "zero memory token value packed incorrectly\n"; + return 8; + } + + channelSpec.memoryTokenId = 0; + if (TileXRCcuBuildChannelCtxV1(channelSpec, &channel, &report) != TILEXR_SUCCESS) { + std::cerr << "zero memory token id rejected: " << report.message << "\n"; + return 9; + } + if ((Read16(channel.raw, 22) & 0xfff0U) != 0 || + (Read16(channel.raw, 24) & 0x00ffU) != 0 || + (Read16(channel.raw, 34) & 0x2U) == 0) { + std::cerr << "zero memory token id packed incorrectly\n"; + return 10; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_lower_layer_payload_packers_are_wired_without_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = PAYLOAD_HEADER.read_text(encoding="utf-8") + source = PAYLOAD_SOURCE.read_text(encoding="utf-8") + oracle = HCOMM_ORACLE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_lower_layer_payloads.h", cmake) + self.assertIn("ccu/tilexr_ccu_lower_layer_payloads.cpp", cmake) + self.assertIn("TileXRCcuPfeCtxSpec", header) + self.assertIn("TileXRCcuLocalJettyCtxSpec", header) + self.assertIn("TileXRCcuChannelCtxV1Spec", header) + self.assertIn("TileXRCcuBuildPfeCtx", header) + self.assertIn("TileXRCcuBuildLocalJettyCtx", header) + self.assertIn("TileXRCcuBuildChannelCtxV1", header) + self.assertIn("tilexr_ccu_abi_constants.h", header) + self.assertIn("TILEXR_CCU_REMOTE_CCU_VA_SHIFT", ABI_CONSTANTS_HEADER.read_text(encoding="utf-8")) + self.assertIn("BuildHcommPfeCtx", oracle) + self.assertIn("BuildHcommLocalJettyCtx", oracle) + self.assertIn("BuildHcommChannelCtxV1", oracle) + + combined = header + "\n" + source + "\n" + oracle + for needle in [ + "#include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + int main() + { + TileXRCcuLowerLayerPlanSpec spec; + spec.msidToken.dieId = 1; + spec.msidToken.msId = 0x55; + spec.msidToken.tokenId = 0x45678; + spec.msidToken.tokenValue = 0; + spec.msidToken.valid = true; + + spec.pfe.dieId = 1; + spec.pfe.pfeOffset = 3; + spec.pfe.startJettyId = 0x120; + spec.pfe.startLocalJettyCtxId = 0x21; + + TileXRCcuLowerLayerJettySpec jetty0; + jetty0.dieId = 1; + jetty0.pfeId = 3; + jetty0.startJettyCtxId = 0x21; + jetty0.doorbellVa = 0x1122334455667788ULL; + jetty0.doorbellTokenId = 0x45678; + jetty0.doorbellTokenValue = 0; + jetty0.sqDepth = 16; + jetty0.wqeBasicBlockStartId = 0x40; + spec.jettys.push_back(jetty0); + + TileXRCcuLowerLayerJettySpec jetty1 = jetty0; + jetty1.startJettyCtxId = 0x22; + jetty1.doorbellVa = 0x2122334455667788ULL; + jetty1.wqeBasicBlockStartId = 0x44; + spec.jettys.push_back(jetty1); + + TileXRCcuLowerLayerChannelSpec channel; + channel.dieId = 1; + channel.channelId = 5; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + channel.remoteEid[i] = static_cast(0x20 + i); + } + channel.tpn = 0x123456; + channel.sourcePfeId = 3; + channel.startJettyId = 0x120; + channel.memoryTokenId = 0xabcde; + channel.memoryTokenValue = 0; + channel.remoteCcuVa = 0x0001234567800000ULL; + spec.channels.push_back(channel); + + spec.xnClear.dieId = 1; + spec.xnClear.startXnId = 0x1f0; + spec.xnClear.count = 3; + spec.xnClear.valid = true; + + spec.ckeClear.dieId = 1; + spec.ckeClear.startCkeId = 0x180; + spec.ckeClear.count = 2; + spec.ckeClear.valid = true; + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerInstallPlan(spec, &plan, &report) != TILEXR_SUCCESS) { + std::cerr << "plan build failed: " << report.message << "\n"; + return 1; + } + if (report.msidTokenCount != 1 || report.pfeCount != 1 || + report.jettyCount != 1 || report.localJettyCtxCount != 2 || + report.channelCount != 1 || report.ckeClearCount != 1) { + std::cerr << "unexpected plan report counts\n"; + return 2; + } + if (plan.msidTokens.size() != 1 || plan.pfes.size() != 1 || + plan.jettys.size() != 1 || plan.jettys[0].ctxs.size() != 2 || + plan.channels.size() != 1 || plan.xnClears.size() != 1 || + plan.ckeClears.size() != 1) { + std::cerr << "unexpected install plan shape\n"; + return 3; + } + if (plan.msidTokens[0].tokenValue != 0 || + plan.msidTokens[0].tokenId != 0x45678U || + plan.xnClears[0].startXnId != 0x1f0 || + plan.xnClears[0].count != 3 || + plan.ckeClears[0].startCkeId != 0x180 || + plan.ckeClears[0].count != 2) { + std::cerr << "scalar plan fields mismatch\n"; + return 4; + } + if (Read16(plan.pfes[0].ctx.raw, 0) != 0x120 || + Read16(plan.pfes[0].ctx.raw, 2) != static_cast(1U | (0x21U << 7U))) { + std::cerr << "pfe payload mismatch\n"; + return 5; + } + if (Read16(plan.jettys[0].ctxs[0].raw, 0) != 0x7788 || + Read16(plan.jettys[0].ctxs[1].raw, 0) != 0x7788 || + Read16(plan.jettys[0].ctxs[0].raw, 8) != 0x7873) { + std::cerr << "jetty payload mismatch\n"; + return 6; + } + if (plan.channels[0].ctx.raw[0] != 0x20 || + plan.channels[0].ctx.raw[15] != 0x2f || + Read16(plan.channels[0].ctx.raw, 16) != 0x3456) { + std::cerr << "channel payload mismatch\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_can_use_hcomm_compatible_wqe_stride_for_direct_ccu_experiment(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 2}; + allocation.localXn = {1, 0x120, 2}; + allocation.remoteXn = {1, 0x240, 2}; + allocation.notifyCke = {1, 0x330, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remote0.remoteNotifyCke = 0x360; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remote1.remoteNotifyCke = 0x361; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.routes.size() != 2 || + snapshot.routes[0].wqeBasicBlockStartId != 0 || + snapshot.routes[1].wqeBasicBlockStartId != 256) { + std::cerr << "hcomm-compatible WQE stride not applied\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE"] = "hcomm_cap" + + result = self.compile_and_run(code, env=env) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_preserves_explicit_peer_rank_from_remote_buffer(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 7, 1}; + allocation.localXn = {1, 0x120, 1}; + allocation.remoteXn = {1, 0x240, 1}; + allocation.notifyCke = {1, 0x330, 1}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.peerRank = 1; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + std::vector remoteCcuBuffers {remote}; + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.routes.size() != 1 || snapshot.routes[0].peerRank != 1) { + std::cerr << "explicit peer rank from remote buffer was not preserved\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_can_use_hcomm_die_pfe_offset_for_direct_ccu_experiment(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + basic.caps.cap4 = 15U; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 2}; + allocation.localXn = {1, 0x120, 2}; + allocation.remoteXn = {1, 0x240, 2}; + allocation.notifyCke = {1, 0x330, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.pfeId != 2 || snapshot.pfeOffset != 18) { + std::cerr << "hcomm die pfe offset not applied: pfeId=" << snapshot.pfeId + << " pfeOffset=" << snapshot.pfeOffset << "\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE"] = "hcomm_die" + + result = self.compile_and_run(code, env=env) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_can_use_hcomm_ordered_pfe_partition_for_direct_ccu_experiment(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 3, 2}; + allocation.localXn = {1, 0x120, 2}; + allocation.remoteXn = {1, 0x240, 2}; + allocation.notifyCke = {1, 0x330, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.pfeId != 3 || + snapshot.startLocalJettyCtxId != 0 || + snapshot.startJettyId != 1024 || + snapshot.pfeJettyCount != 23 || + snapshot.routes.size() != 2) { + std::cerr << "hcomm ordered pfe partition not applied: pfeId=" << snapshot.pfeId + << " startLocalJettyCtxId=" << snapshot.startLocalJettyCtxId + << " startJettyId=" << snapshot.startJettyId + << " pfeJettyCount=" << snapshot.pfeJettyCount << "\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION"] = "hcomm" + + result = self.compile_and_run(code, env=env) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_can_use_hcomm_fe_id_pfe_partition_for_direct_ccu_experiment(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 3, 2}; + allocation.localXn = {1, 0x120, 2}; + allocation.remoteXn = {1, 0x240, 2}; + allocation.notifyCke = {1, 0x330, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.pfeId != 3 || + snapshot.startLocalJettyCtxId != 69 || + snapshot.startJettyId != 1093 || + snapshot.pfeJettyCount != 23) { + std::cerr << "hcomm fe-id pfe partition not applied: pfeId=" << snapshot.pfeId + << " startLocalJettyCtxId=" << snapshot.startLocalJettyCtxId + << " startJettyId=" << snapshot.startJettyId + << " pfeJettyCount=" << snapshot.pfeJettyCount << "\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION"] = "hcomm_fe_id" + + result = self.compile_and_run(code, env=env) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builds_lower_layer_install_plan_from_transport_snapshot(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + int main() + { + TileXRCcuLowerLayerTransportSnapshot snapshot; + snapshot.msidToken.dieId = 0; + snapshot.msidToken.msId = 0x9; + snapshot.msidToken.tokenId = 0x12345; + snapshot.msidToken.tokenValue = 0; + snapshot.msidToken.valid = true; + snapshot.dieId = 0; + snapshot.pfeOffset = 2; + snapshot.pfeId = 2; + snapshot.startJettyId = 0x80; + snapshot.startLocalJettyCtxId = 0x10; + snapshot.xnStartId = 0x1a0; + snapshot.xnCount = 6; + snapshot.ckeStartId = 0x220; + snapshot.ckeCount = 2; + + TileXRCcuLowerLayerTransportRoute route0; + route0.channelId = 7; + route0.remoteXnId = 0x2a0; + route0.remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + route0.remoteEid[i] = static_cast(0x40 + i); + } + route0.tpn = 0x010203; + route0.doorbellVa = 0x1111222233334444ULL; + route0.doorbellTokenId = 0x12345; + route0.doorbellTokenValue = 0; + route0.sqDepth = 8; + route0.wqeBasicBlockStartId = 0x30; + route0.memoryTokenId = 0x23456; + route0.memoryTokenValue = 0; + route0.remoteCcuVa = 0x0000009234000000ULL; + route0.peerRank = 7; + route0.endpointRouteVerified = true; + snapshot.routes.push_back(route0); + + TileXRCcuLowerLayerTransportRoute route1 = route0; + route1.channelId = 8; + route1.remoteXnId = 0x2a1; + route1.remoteNotifyCke = 0x361; + route1.remoteEid[0] = 0x50; + route1.tpn = 0x010204; + route1.doorbellVa = 0x5555666677778888ULL; + route1.wqeBasicBlockStartId = 0x34; + route1.memoryTokenId = 0x23457; + route1.remoteCcuVa = 0x0000009334000000ULL; + route1.peerRank = 9; + route1.endpointRouteVerified = true; + snapshot.routes.push_back(route1); + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "transport snapshot build failed: " << report.message << "\n"; + return 1; + } + if (plan.msidTokens.size() != 1 || plan.pfes.size() != 1 || + plan.jettys.size() != 1 || plan.jettys[0].ctxs.size() != 2 || + plan.channels.size() != 2 || plan.xnClears.size() != 2 || + plan.ckeClears.size() != 2 || plan.remoteXnBindings.size() != 2) { + std::cerr << "unexpected transport-derived plan shape\n"; + return 2; + } + if (report.channelCount != 2 || report.localJettyCtxCount != 2 || + report.ckeClearCount != 2) { + std::cerr << "unexpected transport-derived report counts\n"; + return 3; + } + if (Read16(plan.pfes[0].ctx.raw, 0) != 0x80 || + Read16(plan.pfes[0].ctx.raw, 2) != static_cast(1U | (0x10U << 7U))) { + std::cerr << "transport pfe payload mismatch\n"; + return 4; + } + if (Read16(plan.jettys[0].ctxs[0].raw, 0) != 0x4444 || + Read16(plan.jettys[0].ctxs[1].raw, 0) != 0x8888) { + std::cerr << "transport jetty payload mismatch\n"; + return 5; + } + if (plan.channels[0].channelId != 7 || plan.channels[1].channelId != 8 || + plan.channels[0].ctx.raw[0] != 0x40 || plan.channels[1].ctx.raw[0] != 0x50 || + Read16(plan.channels[0].ctx.raw, 16) != 0x0203 || + Read16(plan.channels[1].ctx.raw, 16) != 0x0204) { + std::cerr << "transport channel payload mismatch\n"; + return 6; + } + if (Read16(plan.channels[0].ctx.raw, 18) != 0x0201 || + Read16(plan.channels[0].ctx.raw, 20) != 0x0008 || + Read16(plan.channels[1].ctx.raw, 18) != 0x1201 || + Read16(plan.channels[1].ctx.raw, 20) != 0x0008) { + std::cerr << "transport channel jetty window mismatch\n"; + return 9; + } + if (plan.xnClears[0].startXnId != 0x1a0 || plan.xnClears[0].count != 6 || + plan.xnClears[1].startXnId != 0x2a0 || plan.xnClears[1].count != 2 || + plan.ckeClears[0].startCkeId != 0x220 || plan.ckeClears[0].count != 2 || + plan.ckeClears[1].startCkeId != 0x360 || plan.ckeClears[1].count != 2) { + std::cerr << "transport local/channel XN or CKE clear mismatch\n"; + return 7; + } + if (!plan.remoteXnBindings[0].peerExchangeObserved || + plan.remoteXnBindings[0].channelId != 7 || + plan.remoteXnBindings[0].localXn != 0x1a0 || + plan.remoteXnBindings[0].remoteXn != 0x2a0 || + plan.remoteXnBindings[0].notifyCke != 0x360 || + plan.remoteXnBindings[0].peerRank != 7 || + plan.remoteXnBindings[0].localWaitCke != 0x220 || + !plan.remoteXnBindings[0].endpointRouteVerified || + plan.remoteXnBindings[1].channelId != 8 || + plan.remoteXnBindings[1].localXn != 0x1a1 || + plan.remoteXnBindings[1].remoteXn != 0x2a1 || + plan.remoteXnBindings[1].notifyCke != 0x361 || + plan.remoteXnBindings[1].peerRank != 9 || + plan.remoteXnBindings[1].localWaitCke != 0x221 || + !plan.remoteXnBindings[1].endpointRouteVerified) { + std::cerr << "transport remote XN proof mismatch\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_snapshot_pfe_jetty_count_reaches_pfe_ctx(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + int main() + { + TileXRCcuLowerLayerTransportSnapshot snapshot; + snapshot.msidToken.dieId = 1; + snapshot.msidToken.msId = 0x9; + snapshot.msidToken.tokenId = 0x12345; + snapshot.msidToken.valid = true; + snapshot.dieId = 1; + snapshot.pfeOffset = 18; + snapshot.pfeId = 2; + snapshot.startJettyId = 1024; + snapshot.pfeJettyCount = 23; + snapshot.startLocalJettyCtxId = 0; + snapshot.xnStartId = 0x1a0; + snapshot.xnCount = 1; + snapshot.ckeStartId = 0x220; + snapshot.ckeCount = 1; + + TileXRCcuLowerLayerTransportRoute route; + route.channelId = 7; + route.remoteXnId = 0x2a0; + route.remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + route.remoteEid[i] = static_cast(0x40 + i); + } + route.tpn = 0x010203; + route.doorbellVa = 0x1111222233334444ULL; + route.doorbellTokenId = 0x12345; + route.sqDepth = 8; + route.memoryTokenId = 0x23456; + route.remoteCcuVa = 0x0000009234000000ULL; + snapshot.routes.push_back(route); + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "transport snapshot build failed: " << report.message << "\n"; + return 1; + } + if (plan.pfes.size() != 1 || plan.pfes[0].pfeOffset != 18) { + std::cerr << "pfe install shape mismatch\n"; + return 2; + } + const uint16_t pfeWord = Read16(plan.pfes[0].ctx.raw, 2); + if ((pfeWord & 0x7fU) != 22U) { + std::cerr << "pfe jetty count did not reach ctx: word=" << pfeWord << "\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_snapshot_installs_remote_xn_range_separately(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuLowerLayerTransportSnapshot snapshot; + snapshot.msidToken.dieId = 1; + snapshot.msidToken.msId = 0x45; + snapshot.msidToken.tokenId = 0x1234; + snapshot.msidToken.valid = true; + snapshot.dieId = 1; + snapshot.pfeOffset = 0x80; + snapshot.pfeId = 2; + snapshot.startJettyId = 0x400; + snapshot.pfeJettyCount = 2; + snapshot.startLocalJettyCtxId = 0; + snapshot.xnStartId = 0x1a0; + snapshot.xnCount = 2; + snapshot.ckeStartId = 0x220; + snapshot.ckeCount = 2; + + snapshot.routes.resize(2); + snapshot.routes[0].channelId = 7; + snapshot.routes[0].remoteXnId = 0x2a0; + snapshot.routes[0].remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + snapshot.routes[0].remoteEid[i] = static_cast(0x40 + i); + } + snapshot.routes[0].tpn = 0x010203; + snapshot.routes[0].doorbellVa = 0x1111222233334444ULL; + snapshot.routes[0].doorbellTokenId = 0x12345; + snapshot.routes[0].sqDepth = 8; + snapshot.routes[0].remoteCcuVa = 0x90000000ULL; + snapshot.routes[0].memoryTokenId = 0x1234; + + snapshot.routes[1] = snapshot.routes[0]; + snapshot.routes[1].channelId = 8; + snapshot.routes[1].remoteXnId = 0x2a1; + snapshot.routes[1].remoteNotifyCke = 0x361; + snapshot.routes[1].tpn = 0x010204; + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "transport snapshot build failed: " << report.message << "\n"; + return 1; + } + if (plan.xnClears.size() != 2) { + std::cerr << "expected local and channel remote XN clears, got " << + plan.xnClears.size() << "\n"; + return 2; + } + if (plan.xnClears[0].startXnId != 0x1a0 || plan.xnClears[0].count != 2) { + std::cerr << "local XN clear mismatch\n"; + return 3; + } + if (plan.xnClears[1].startXnId != 0x2a0 || plan.xnClears[1].count != 2) { + std::cerr << "channel remote XN clear mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_snapshot_installs_remote_notify_cke_range_separately(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuLowerLayerTransportSnapshot snapshot; + snapshot.msidToken.dieId = 1; + snapshot.msidToken.msId = 0x45; + snapshot.msidToken.tokenId = 0x1234; + snapshot.msidToken.valid = true; + snapshot.dieId = 1; + snapshot.pfeOffset = 0x80; + snapshot.pfeId = 2; + snapshot.startJettyId = 0x400; + snapshot.pfeJettyCount = 2; + snapshot.startLocalJettyCtxId = 0; + snapshot.xnStartId = 0x1a0; + snapshot.xnCount = 2; + snapshot.ckeStartId = 0x220; + snapshot.ckeCount = 3; + + snapshot.routes.resize(2); + snapshot.routes[0].channelId = 7; + snapshot.routes[0].remoteXnId = 0x2a0; + snapshot.routes[0].remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + snapshot.routes[0].remoteEid[i] = static_cast(0x40 + i); + } + snapshot.routes[0].tpn = 0x010203; + snapshot.routes[0].doorbellVa = 0x1111222233334444ULL; + snapshot.routes[0].doorbellTokenId = 0x12345; + snapshot.routes[0].sqDepth = 8; + snapshot.routes[0].remoteCcuVa = 0x90000000ULL; + snapshot.routes[0].memoryTokenId = 0x1234; + snapshot.routes[1] = snapshot.routes[0]; + snapshot.routes[1].channelId = 8; + snapshot.routes[1].remoteXnId = 0x2a1; + snapshot.routes[1].remoteNotifyCke = 0x361; + snapshot.routes[1].tpn = 0x010204; + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "transport snapshot build failed: " << report.message << "\n"; + return 1; + } + if (plan.ckeClears.size() != 2 || report.ckeClearCount != 2) { + std::cerr << "expected local and remote notify CKE clears, got " << + plan.ckeClears.size() << " report=" << report.ckeClearCount << "\n"; + return 2; + } + if (plan.ckeClears[0].startCkeId != 0x220 || plan.ckeClears[0].count != 3) { + std::cerr << "local CKE clear mismatch\n"; + return 3; + } + if (plan.ckeClears[1].startCkeId != 0x360 || plan.ckeClears[1].count != 2) { + std::cerr << "remote notify CKE clear mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builds_transport_template_from_basic_info_and_resource_allocation(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0; + basic.msidToken.valid = true; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (5U << 16) | 255U; + basic.caps.cap1 = (127U << 16) | 63U; + basic.caps.cap2 = (31U << 16) | 15U; + basic.caps.cap3 = (7U << 16) | 1U; + basic.caps.cap4 = 9U; + + TileXRCcuSpecInfo specInfo; + TileXRCcuSpecsReport specsReport; + if (TileXRCcuDecodeBasicInfo(basic, &specInfo, &specsReport) != TILEXR_SUCCESS) { + std::cerr << "decode failed: " << specsReport.message << "\n"; + return 1; + } + + TileXRCcuResourceSpec resourceSpec; + if (TileXRCcuBuildResourceSpec(specInfo, 6, 475, 1961, 332, 2, &resourceSpec, &specsReport) != + TILEXR_SUCCESS) { + std::cerr << "resource spec failed: " << specsReport.message << "\n"; + return 2; + } + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(resourceSpec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 3; + } + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 2; + request.syncInstructionCount = 9; + request.bindingsPerSyncResource = 1; + TileXRCcuProducerPlan producerPlan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport allocatorReport; + if (allocator.Allocate(request, &producerPlan, &allocation, &allocatorReport) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << allocatorReport.message << "\n"; + return 4; + } + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = specInfo.xnBaseAddr + allocation.remoteXn.startId * 8ULL; + remote0.memoryTokenId = 0x23456; + remote0.memoryTokenValue = 0x5678; + remote0.remoteNotifyCke = 0x360; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = specInfo.xnBaseAddr + (allocation.remoteXn.startId + 1U) * 8ULL; + remote1.memoryTokenId = 0x23457; + remote1.memoryTokenValue = 0x5679; + remote1.remoteNotifyCke = 0x361; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 5; + } + if (!snapshot.msidToken.valid || snapshot.msidToken.dieId != 1 || + snapshot.msidToken.msId != 0x45 || snapshot.msidToken.tokenId != 0x1234 || + snapshot.msidToken.tokenValue != 0) { + std::cerr << "msid token template mismatch\n"; + return 6; + } + if (snapshot.dieId != 1 || snapshot.pfeOffset != allocation.channels.startId || + snapshot.pfeId != allocation.channels.startId || + snapshot.startJettyId != 1024 || snapshot.startLocalJettyCtxId != 0 || + snapshot.xnStartId != allocation.localXn.startId || + snapshot.xnCount != allocation.localXn.num || + snapshot.ckeStartId != allocation.notifyCke.startId || + snapshot.ckeCount != allocation.notifyCke.num) { + std::cerr << "template scalar mismatch\n"; + return 7; + } + if (snapshot.routes.size() != 2 || snapshot.routes[0].channelId != allocation.channels.startId || + snapshot.routes[1].channelId != allocation.channels.startId + 1U || + snapshot.routes[0].remoteXnId != allocation.remoteXn.startId || + snapshot.routes[1].remoteXnId != allocation.remoteXn.startId + 1U || + snapshot.routes[0].remoteNotifyCke != 0x360 || + snapshot.routes[1].remoteNotifyCke != 0x361 || + snapshot.routes[0].wqeBasicBlockStartId != 0 || + snapshot.routes[1].wqeBasicBlockStartId != 4 || + snapshot.routes[0].remoteCcuVa != remoteCcuBuffers[0].remoteCcuVa || + snapshot.routes[1].remoteCcuVa != remoteCcuBuffers[1].remoteCcuVa || + snapshot.routes[0].memoryTokenId != 0x23456 || + snapshot.routes[0].memoryTokenValue != 0x5678 || + snapshot.routes[1].memoryTokenId != 0x23457 || + snapshot.routes[1].memoryTokenValue != 0x5679) { + std::cerr << "route template mismatch\n"; + return 8; + } + if (report.message != "ok" || report.channelCount != 2 || report.ckeClearCount != 1) { + std::cerr << "report mismatch\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_accepts_basic_info_without_basic_msid_token(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0; + basic.msidToken.tokenValue = 0; + basic.msidToken.valid = false; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x1a0, 1}; + allocation.remoteXn = {1, 0x2a0, 1}; + allocation.notifyCke = {1, 0x220, 1}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.memoryTokenValue = 0x5678; + remote.remoteNotifyCke = 0x360; + remoteCcuBuffers.push_back(remote); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.msidToken.valid || snapshot.msidToken.tokenId != 0 || + snapshot.msidToken.tokenValue != 0 || snapshot.msidToken.dieId != 1 || + snapshot.msidToken.msId != 0x45) { + std::cerr << "basic-info token should remain absent in template\n"; + return 2; + } + if (report.msidTokenCount != 0 || report.channelCount != 1 || + snapshot.routes.size() != 1 || snapshot.routes[0].memoryTokenId != 0x23456) { + std::cerr << "template route/report mismatch\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_promotes_verified_endpoint_route_from_remote_buffer_info(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x1a0, 1}; + allocation.remoteXn = {1, 0x2a0, 1}; + allocation.notifyCke = {1, 0x220, 1}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.memoryTokenValue = 0x5678; + remote.remoteXnId = 0x2a7; + remote.remoteNotifyCke = 0x361; + for (uint32_t i = 0; i < remote.remoteEid.size(); ++i) { + remote.remoteEid[i] = static_cast(0x80 + i); + } + remote.tpn = 0x010203; + remote.doorbellVa = 0x1122334455667788ULL; + remote.doorbellTokenId = 0x3456; + remote.doorbellTokenValue = 0; + remote.sqDepth = 64; + remote.endpointRouteVerified = true; + std::vector remoteCcuBuffers {remote}; + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.routes.size() != 1 || + !snapshot.routes[0].endpointRouteVerified || + snapshot.routes[0].remoteEid[0] != 0x80 || + snapshot.routes[0].tpn != remote.tpn || + snapshot.routes[0].doorbellVa != remote.doorbellVa || + snapshot.routes[0].doorbellTokenId != remote.doorbellTokenId || + snapshot.routes[0].doorbellTokenValue != remote.doorbellTokenValue || + snapshot.routes[0].sqDepth != remote.sqDepth || + snapshot.routes[0].remoteXnId != remote.remoteXnId || + snapshot.routes[0].remoteNotifyCke != remote.remoteNotifyCke) { + std::cerr << "verified endpoint route was not promoted from remote buffer info\n"; + return 2; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "install plan build failed: " << report.message << "\n"; + return 3; + } + if (plan.remoteXnBindings.size() != 1 || + !plan.remoteXnBindings[0].endpointRouteVerified || + plan.remoteXnBindings[0].remoteXn != remote.remoteXnId || + plan.remoteXnBindings[0].notifyCke != remote.remoteNotifyCke) { + std::cerr << "verified endpoint proof was not propagated\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_rejects_incomplete_verified_endpoint_route_from_remote_buffer_info(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x1a0, 1}; + allocation.remoteXn = {1, 0x2a0, 1}; + allocation.notifyCke = {1, 0x220, 1}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.memoryTokenValue = 0x5678; + remote.remoteXnId = 0x2a7; + remote.remoteNotifyCke = 0x361; + remote.remoteEid[0] = 0x80; + remote.tpn = 0x010203; + remote.doorbellVa = 0x1122334455667788ULL; + remote.doorbellTokenId = 0; + remote.sqDepth = 64; + remote.endpointRouteVerified = true; + std::vector remoteCcuBuffers {remote}; + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.routes.size() != 1 || + snapshot.routes[0].endpointRouteVerified || + snapshot.routes[0].remoteEid[0] != 0 || + snapshot.routes[0].tpn != 0 || + snapshot.routes[0].doorbellVa != 0 || + snapshot.routes[0].sqDepth != 0) { + std::cerr << "incomplete endpoint route should fail closed\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_rejects_channel_allocation_smaller_than_routes(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x1a0, 2}; + allocation.remoteXn = {1, 0x2a0, 2}; + allocation.notifyCke = {1, 0x360, 2}; + allocation.localWaitCke = {1, 0x220, 2}; + allocation.remoteNotifyCke = {1, 0x360, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remote0.memoryTokenValue = 0x5678; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "template accepted an allocation with too few channels\n"; + return 1; + } + if (report.message.find("channel allocation count") == std::string::npos) { + std::cerr << "weak channel allocation diagnostic: " << report.message << "\n"; + return 2; + } + if (!snapshot.routes.empty()) { + std::cerr << "failed template should not retain routes\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_uses_local_wait_cke_for_clear_and_remote_notify_cke_for_routes(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 2}; + allocation.localXn = {1, 0x1a0, 2}; + allocation.remoteXn = {1, 0x2a0, 2}; + allocation.notifyCke = {1, 0x360, 2}; + allocation.localWaitCke = {1, 0x220, 2}; + allocation.remoteNotifyCke = {1, 0x360, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remote0.memoryTokenValue = 0x5678; + remote0.remoteNotifyCke = 0; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remote1.memoryTokenValue = 0x5679; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.ckeStartId != 0x220 || snapshot.ckeCount != 2) { + std::cerr << "local wait CKE clear range mismatch\n"; + return 2; + } + if (snapshot.routes.size() != 2 || + snapshot.routes[0].remoteNotifyCke != 0x360 || + snapshot.routes[1].remoteNotifyCke != 0x361) { + std::cerr << "remote notify CKE route fallback mismatch\n"; + return 3; + } + for (uint32_t i = 0; i < snapshot.routes.size(); ++i) { + for (uint32_t eidIndex = 0; eidIndex < TILEXR_CCU_EID_BYTES; ++eidIndex) { + snapshot.routes[i].remoteEid[eidIndex] = + static_cast(0x40 + i * 0x10 + eidIndex); + } + snapshot.routes[i].tpn = 0x010200 + i; + snapshot.routes[i].doorbellVa = 0x1111222233334444ULL + i * 0x1000ULL; + snapshot.routes[i].doorbellTokenId = 0x12345; + snapshot.routes[i].doorbellTokenValue = 0; + snapshot.routes[i].sqDepth = 8; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "install plan build failed: " << report.message << "\n"; + return 4; + } + if (plan.remoteXnBindings.size() != 2 || + plan.remoteXnBindings[0].notifyCke != 0x360 || + plan.remoteXnBindings[0].localWaitCke != 0x220 || + plan.remoteXnBindings[1].notifyCke != 0x361 || + plan.remoteXnBindings[1].localWaitCke != 0x221) { + std::cerr << "split CKE remote XN proof mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_overlay_verified_endpoint_routes_updates_only_matching_endpoint_fields(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuLowerLayerTransportSnapshot snapshot; + snapshot.routes.resize(2); + snapshot.routes[0].channelId = 0x20; + snapshot.routes[0].peerRank = 3; + snapshot.routes[0].remoteXnId = 0x1a0; + snapshot.routes[0].remoteNotifyCke = 0x360; + snapshot.routes[0].wqeBasicBlockStartId = 0x12; + snapshot.routes[0].memoryTokenId = 0x2001; + snapshot.routes[0].remoteCcuVa = 0x90000000ULL; + snapshot.routes[1].channelId = 0x21; + snapshot.routes[1].peerRank = 4; + snapshot.routes[1].remoteXnId = 0x1a1; + snapshot.routes[1].remoteNotifyCke = 0x361; + + TileXRCcuLowerLayerTransportRoute verified; + verified.channelId = 0x20; + verified.peerRank = 99; + verified.remoteXnId = 0x2b0; + verified.remoteNotifyCke = 0x470; + for (uint32_t i = 0; i < verified.remoteEid.size(); ++i) { + verified.remoteEid[i] = static_cast(0xc0 + i); + } + verified.tpn = 0x010203; + verified.doorbellVa = 0x1122334455667788ULL; + verified.doorbellTokenId = 0x3456; + verified.doorbellTokenValue = 0; + verified.sqDepth = 64; + verified.wqeBasicBlockStartId = 0x77; + verified.memoryTokenId = 0x9999; + verified.remoteCcuVa = 0xabcdefULL; + verified.endpointRouteVerified = true; + + TileXRCcuLowerLayerPlanBuilderReport report; + std::vector routes {verified}; + if (TileXRCcuOverlayVerifiedEndpointRoutes(routes, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "overlay failed: " << report.message << "\n"; + return 1; + } + + const auto& route0 = snapshot.routes[0]; + if (!route0.endpointRouteVerified || + route0.remoteEid[0] != 0xc0 || + route0.tpn != verified.tpn || + route0.doorbellVa != verified.doorbellVa || + route0.doorbellTokenId != verified.doorbellTokenId || + route0.doorbellTokenValue != verified.doorbellTokenValue || + route0.sqDepth != verified.sqDepth) { + std::cerr << "verified endpoint fields were not overlaid\n"; + return 2; + } + if (route0.peerRank != 3 || + route0.remoteXnId != 0x1a0 || + route0.remoteNotifyCke != 0x360 || + route0.wqeBasicBlockStartId != 0x12 || + route0.memoryTokenId != 0x2001 || + route0.remoteCcuVa != 0x90000000ULL) { + std::cerr << "overlay changed non-endpoint resource fields\n"; + return 3; + } + if (snapshot.routes[1].endpointRouteVerified || + snapshot.routes[1].remoteEid[0] != 0 || + snapshot.routes[1].tpn != 0 || + snapshot.routes[1].doorbellVa != 0) { + std::cerr << "overlay changed an unmatched route\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_uses_peer_exchanged_remote_xn_ids_when_present(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 2}; + allocation.localXn = {1, 0x2a0, 2}; + allocation.remoteXn = {1, 0x2b0, 2}; + allocation.notifyCke = {1, 0x360, 2}; + allocation.localWaitCke = {1, 0x220, 2}; + allocation.remoteNotifyCke = {1, 0x360, 2}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote0; + remote0.remoteCcuVa = 0x0000009234000000ULL; + remote0.memoryTokenId = 0x23456; + remote0.memoryTokenValue = 0x5678; + remote0.remoteXnId = 0x1a0; + remote0.remoteNotifyCke = 0x360; + remoteCcuBuffers.push_back(remote0); + TileXRCcuRemoteCcuBufferInfo remote1 = remote0; + remote1.remoteCcuVa = 0x0000009334000000ULL; + remote1.memoryTokenId = 0x23457; + remote1.memoryTokenValue = 0x5679; + remote1.remoteXnId = 0x1a1; + remote1.remoteNotifyCke = 0x361; + remoteCcuBuffers.push_back(remote1); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.routes.size() != 2 || + snapshot.routes[0].remoteXnId != 0x1a0 || + snapshot.routes[1].remoteXnId != 0x1a1) { + std::cerr << "peer exchanged remote XN IDs were not preserved\n"; + return 2; + } + for (uint32_t i = 0; i < snapshot.routes.size(); ++i) { + for (uint32_t eidIndex = 0; eidIndex < TILEXR_CCU_EID_BYTES; ++eidIndex) { + snapshot.routes[i].remoteEid[eidIndex] = + static_cast(0x40 + i * 0x10 + eidIndex); + } + snapshot.routes[i].tpn = 0x010200 + i; + snapshot.routes[i].doorbellVa = 0x1111222233334444ULL + i * 0x1000ULL; + snapshot.routes[i].doorbellTokenId = 0x12345; + snapshot.routes[i].sqDepth = 8; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "install plan build failed: " << report.message << "\n"; + return 3; + } + if (plan.remoteXnBindings.size() != 2 || + plan.remoteXnBindings[0].remoteXn != 0x1a0 || + plan.remoteXnBindings[1].remoteXn != 0x1a1 || + !plan.remoteXnBindings[0].peerExchangeObserved || + !plan.remoteXnBindings[1].peerExchangeObserved || + plan.remoteXnBindings[0].endpointRouteVerified || + plan.remoteXnBindings[1].endpointRouteVerified) { + std::cerr << "peer exchanged remote XN proof mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_carries_explicit_channel_owner_exchange_proof(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x2a0, 1}; + allocation.remoteXn = {1, 0x2b0, 1}; + allocation.notifyCke = {1, 0x360, 1}; + allocation.localWaitCke = {1, 0x220, 1}; + allocation.remoteNotifyCke = {1, 0x360, 1}; + + TileXRCcuRemoteCcuBufferInfo endpointOnly; + endpointOnly.remoteCcuVa = 0x0000009234000000ULL; + endpointOnly.memoryTokenId = 0x23456; + endpointOnly.memoryTokenValue = 0x5678; + endpointOnly.remoteXnId = 0x1a0; + endpointOnly.remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + endpointOnly.remoteEid[i] = static_cast(0x40 + i); + } + endpointOnly.tpn = 0x010200; + endpointOnly.doorbellVa = 0x1111222233334444ULL; + endpointOnly.doorbellTokenId = 0x12345; + endpointOnly.sqDepth = 8; + endpointOnly.endpointRouteVerified = true; + + TileXRCcuLowerLayerTransportSnapshot endpointOnlySnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, {endpointOnly}, &endpointOnlySnapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "endpoint-only template build failed: " << report.message << "\n"; + return 1; + } + TileXRCcuLowerLayerInstallPlan endpointOnlyPlan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + endpointOnlySnapshot, &endpointOnlyPlan, &report) != TILEXR_SUCCESS) { + std::cerr << "endpoint-only plan build failed: " << report.message << "\n"; + return 2; + } + if (!endpointOnlyPlan.remoteXnBindings[0].endpointRouteVerified || + endpointOnlyPlan.remoteXnBindings[0].channelResourceOwnerVerified || + endpointOnlyPlan.remoteXnBindings[0].transportResourceExchangeVerified) { + std::cerr << "endpoint route alone was promoted to owner/exchange proof\n"; + return 3; + } + + TileXRCcuRemoteCcuBufferInfo proven = endpointOnly; + proven.channelResourceOwnerVerified = true; + proven.transportResourceExchangeVerified = true; + + TileXRCcuLowerLayerTransportSnapshot provenSnapshot; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, {proven}, &provenSnapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "proven template build failed: " << report.message << "\n"; + return 4; + } + if (!provenSnapshot.routes[0].channelResourceOwnerVerified || + !provenSnapshot.routes[0].transportResourceExchangeVerified) { + std::cerr << "route did not preserve owner/exchange proof\n"; + return 5; + } + + TileXRCcuLowerLayerInstallPlan provenPlan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + provenSnapshot, &provenPlan, &report) != TILEXR_SUCCESS) { + std::cerr << "proven plan build failed: " << report.message << "\n"; + return 6; + } + const auto& proof = provenPlan.remoteXnBindings[0]; + if (!proof.endpointRouteVerified || + !proof.channelResourceOwnerVerified || + !proof.transportResourceExchangeVerified) { + std::cerr << "install proof did not preserve owner/exchange proof\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_plan_builder_rejects_incomplete_lower_layer_inputs(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + TileXRCcuLowerLayerPlanSpec spec; + if (TileXRCcuBuildLowerLayerInstallPlan(spec, &plan, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty lower-layer spec accepted\n"; + return 1; + } + if (report.message.find("missing lower-layer CCU MSID token") == std::string::npos) { + std::cerr << "weak empty-spec diagnostic: " << report.message << "\n"; + return 2; + } + + spec.msidToken.valid = true; + spec.msidToken.tokenId = 0; + spec.pfe.dieId = 0; + spec.pfe.pfeOffset = 1; + spec.pfe.startJettyId = 0x10; + spec.pfe.startLocalJettyCtxId = 0x2; + TileXRCcuLowerLayerJettySpec jetty; + jetty.startJettyCtxId = 0x2; + jetty.doorbellVa = 0x1000; + jetty.doorbellTokenId = 9; + jetty.sqDepth = 8; + spec.jettys.push_back(jetty); + TileXRCcuLowerLayerChannelSpec channel; + channel.channelId = 1; + channel.sourcePfeId = 1; + channel.startJettyId = 0x10; + channel.memoryTokenId = 7; + channel.remoteCcuVa = 0x200000; + spec.channels.push_back(channel); + spec.xnClear.valid = true; + spec.xnClear.count = 1; + spec.ckeClear.valid = true; + spec.ckeClear.count = 1; + if (TileXRCcuBuildLowerLayerInstallPlan(spec, &plan, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "channel with empty remote EID accepted\n"; + return 3; + } + if (report.message.find("invalid CCU channel context v1 spec") == std::string::npos) { + std::cerr << "weak channel diagnostic: " << report.message << "\n"; + return 4; + } + + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + spec.channels[0].remoteEid[i] = static_cast(0x40 + i); + } + if (TileXRCcuBuildLowerLayerInstallPlan(spec, &plan, &report) != TILEXR_SUCCESS) { + std::cerr << "valid zero-token lower-layer spec rejected: " << report.message << "\n"; + return 5; + } + if (plan.msidTokens.empty() || plan.msidTokens[0].tokenId != 0 || + plan.channels.empty() || plan.channels[0].ctx.raw[0] != 0x40) { + std::cerr << "valid zero-token lower-layer plan mismatch\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_plan_builder_surface_is_wired_into_tilexr_comm_without_udma_boundary(self): + header = BUILDER_HEADER.read_text(encoding="utf-8") + source = BUILDER_SOURCE.read_text(encoding="utf-8") + runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") + runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TileXRCcuLowerLayerTransportSnapshot", header) + self.assertIn("TileXRCcuLowerLayerTransportRoute", header) + self.assertIn("TileXRCcuRemoteCcuBufferInfo", header) + self.assertIn("TileXRCcuBuildLowerLayerTransportTemplate", header) + self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", header) + self.assertIn("remoteXnBindings", header) + self.assertIn("localWaitCke", source) + self.assertIn("TileXRCcuBuildPfeCtx", source) + self.assertIn("TileXRCcuBuildLocalJettyCtx", source) + self.assertIn("TileXRCcuBuildChannelCtxV1", source) + self.assertIn("allocation.channels.num < remoteCcuBuffers.size()", source) + self.assertIn("channel allocation count does not match lower-layer route count", source) + self.assertNotIn("TILEXR_CCU_DIRECT_SYNC_RESOURCE_MAP", source) + self.assertNotIn("UseHcommTraceSyncResourceMap", source) + self.assertIn("SelectLowerLayerWqeBasicBlockStride", source) + self.assertIn("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE", source) + self.assertIn("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE", source) + self.assertIn("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", source) + + self.assertIn("TileXRCcuDirectRuntime", runtime_header) + self.assertIn("ExportLowerLayerTransportSnapshot", runtime_header) + self.assertIn("RegisterCcuResourceRmaBuffer", runtime_header) + self.assertIn("ExportRemoteCcuRmaBuffers", runtime_header) + self.assertIn("TileXRCcuLocalResourceWindowInfo", runtime_header) + self.assertIn("TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot", runtime_source) + self.assertIn("TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE", runtime_source) + self.assertIn("UseImportedPeerEndpointRoute", runtime_source) + self.assertIn("useImportedPeerRoute &&", runtime_source) + self.assertIn("usePeerExportedRoute", runtime_source) + self.assertRegex( + runtime_source, + r"(?s)\} else \{\s*remote\.tpn = peerWindow\.tpn;.*?usePeerExportedRoute", + ) + self.assertNotIn( + "} else if (!useImportedPeerRoute && TraceEndpointRoute() && peerWindow.endpointRouteVerified)", + runtime_source, + ) + self.assertNotIn("udma/", runtime_header + "\n" + runtime_source) + + combined = header + "\n" + source + "\n" + runtime_header + "\n" + runtime_source + for needle in [ + "#include &remoteCcuBuffers", comm_source) + self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", comm_source) + self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr)", comm_source) + self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", comm_source) + self.assertIn("ccuDirectRuntime_->ExportLowerLayerTransportSnapshot", comm_source) + self.assertIn("RefreshDirectCcuLowerLayerPlan();", comm_source) + self.assertIn("direct CCU lower-layer template is not configured", comm_source) + self.assertIn("direct CCU lower-layer install plan cached", comm_source) + + register_body = comm_source[ + comm_source.index("int TileXRComm::RegisterUDMAMemory"): + comm_source.index("int TileXRComm::UnregisterUDMAMemory") + ] + self.assertIn("ret = UpdateCommArgsDev();", register_body) + self.assertNotIn("RefreshDirectCcuLowerLayerPlan();", register_body) + self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_body) + + unregister_body = comm_source[ + comm_source.index("int TileXRComm::UnregisterUDMAMemory"): + comm_source.index("GM_ADDR TileXRComm::GetUDMARegistryPtr") + ] + self.assertNotIn("ResetDirectCcuLowerLayerPlan();", unregister_body) + + init_udma_body = comm_source[ + comm_source.index("int TileXRComm::InitUDMA"): + comm_source.index("int TileXRComm::InitDirectCcuRuntime") + ] + self.assertNotIn("RefreshDirectCcuBasicInfo", init_udma_body) + self.assertNotIn("ResetDirectCcuBasicInfo", init_udma_body) + + forbidden_patterns = [ + ("udmaTransport_->", "RegisterCcuResourceRmaBuffer"), + ("udmaTransport_->", "ExportLocalCcuRmaBuffer"), + ("udmaTransport_->", "ExportRemoteCcuRmaBuffers"), + ("udmaTransport_->", "ExportLowerLayerTransportSnapshot"), + ] + for prefix, suffix in forbidden_patterns: + self.assertNotIn(prefix + suffix, comm_header + "\n" + comm_source) + for forbidden in [ + "rtCCULaunch", + "TileXRCcuSubmitTask", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "libhcomm", + "libhccl_v2", + ]: + self.assertNotIn(forbidden, comm_header + "\n" + comm_source) + + def test_direct_ccu_runtime_owns_resource_window_boundary(self): + runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") + runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + specs_header = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.h").read_text(encoding="utf-8") + comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + builder_header = BUILDER_HEADER.read_text(encoding="utf-8") + + self.assertIn("TileXRCcuLocalResourceWindowInfo", runtime_header) + self.assertIn("rawTokenId", runtime_header) + self.assertIn("rawMemoryTokenId", builder_header) + self.assertIn("TileXRCcuDirectAllGatherFn", runtime_header) + self.assertIn("localResourceWindow_", runtime_header) + self.assertIn("resourceWindowRegistered_", runtime_header) + self.assertIn("int RegisterCcuResourceRmaBuffer(", runtime_header) + self.assertIn("ExportLocalCcuRmaBuffer", runtime_header) + self.assertIn("int ExportRemoteCcuRmaBuffers(", runtime_header) + self.assertIn("int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer", runtime_source) + self.assertIn("int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer", runtime_source) + self.assertIn("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers", runtime_source) + self.assertIn("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot", runtime_source) + self.assertIn("localResourceWindow_.addr = resourceAddr", runtime_source) + self.assertIn("resourceWindowRegistered_ = true", runtime_source) + self.assertIn("TILEXR_CCU_RESOURCE_WINDOW_BYTES", specs_header) + self.assertIn("72ULL * 1024ULL * 1024ULL", specs_header) + self.assertIn("return TILEXR_CCU_RESOURCE_WINDOW_BYTES", runtime_source) + self.assertIn("options_.allGather", runtime_source) + self.assertIn("route.remoteEid", runtime_source) + self.assertIn("route.doorbellVa", runtime_source) + self.assertIn("route.sqDepth", runtime_source) + + self.assertIn("ConfigureDirectCcuLowerLayerTemplateFromAllocation(", comm_header) + self.assertIn("const std::vector &remoteCcuBuffers", comm_header) + self.assertIn("PrepareDirectCcuLowerLayerTemplateFromAllocation", comm_header) + self.assertIn("ExchangeDirectCcuRemoteNotifyCke", comm_header) + self.assertIn("DirectCcuAllGatherCallback", comm_header) + self.assertIn("DirectCcuThreadAllGather", comm_header) + self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer", comm_source) + self.assertIn("ccuDirectRuntime_->ExportLocalCcuRmaBuffer", comm_source) + self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", comm_source) + self.assertIn("options.allGather = &TileXRComm::DirectCcuAllGatherCallback", comm_source) + self.assertIn("options.allGatherUserData = this", comm_source) + self.assertIn("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke", comm_source) + self.assertIn("int TileXRComm::DirectCcuThreadAllGather", comm_source) + self.assertIn("DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this)", comm_source) + self.assertIn("comm->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf)", comm_source) + self.assertIn("InitThread", comm_source) + self.assertIn("ret = InitDirectCcuRuntime();", comm_source) + exchange_body = comm_source[ + comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): + comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + ] + self.assertNotIn("SelectDirectCcuRemoteBindingOverride", exchange_body) + self.assertIn("peerLocalWaitCkeOffset", exchange_body) + self.assertIn("peerResources.localWaitCkeStartId", exchange_body) + self.assertIn("peerResources.localWaitCkeCount", exchange_body) + self.assertNotIn("allocation.remoteNotifyCke.startId,\n routeIndex", exchange_body) + self.assertIn("allocation.localXn.startId", comm_source) + self.assertIn("remoteXnStartId", exchange_body) + self.assertIn("remoteXnCount", exchange_body) + self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET", exchange_body) + self.assertNotIn("TILEXR_CCU_XN_SLOT_BYTES", exchange_body) + self.assertIn("remoteXnId", comm_source) + self.assertIn("remoteNotifyCke", comm_source) + self.assertIn("templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId", comm_source) + self.assertIn("templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue", comm_source) + self.assertIn("templateSnapshot.msidToken.valid = true", comm_source) + self.assertIn("directCcuVerifiedEndpointRoutes_", comm_header) + self.assertIn("TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes", comm_source) + self.assertIn("directCcuLocalVerifiedEndpointRoute_", comm_header) + self.assertIn("TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute", comm_source) + self.assertIn("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute", comm_source) + self.assertIn("ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute", comm_source) + self.assertIn("TileXRCcuLocalEndpointRouteCollectorFn", runtime_header) + self.assertIn("localEndpointRouteCollector", runtime_header) + self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID", runtime_source) + self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA", runtime_source) + self.assertIn("direct CCU local endpoint route collected", runtime_source) + self.assertIn("TileXRCcuOverlayVerifiedEndpointRoutes", builder_header) + self.assertGreaterEqual(comm_source.count("TileXRCcuOverlayVerifiedEndpointRoutes("), 3) + init_runtime_body = comm_source[ + comm_source.index("int TileXRComm::InitDirectCcuRuntime"): + comm_source.index("int TileXRComm::InitSDMA") + ] + self.assertIn("logicDevId", init_runtime_body) + self.assertIn("devicePhyId", init_runtime_body) + self.assertIn("hdcType", init_runtime_body) + self.assertIn("raInitialized", init_runtime_body) + self.assertNotIn("udma/", runtime_header + "\n" + runtime_source) + + register_body = runtime_source[ + runtime_source.index("int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer"): + runtime_source.index("int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer") + ] + export_local_body = runtime_source[ + runtime_source.index("int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer"): + runtime_source.index("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers") + ] + export_remote_body = runtime_source[ + runtime_source.index("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers"): + runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot") + ] + prepare_from_allocation_body = comm_source[ + comm_source.index("int TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation"): + comm_source.index("int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation") + ] + export_snapshot_body = runtime_source[ + runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot"): + ] + self.assertNotIn("int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer(uint64_t)\n{\n return TILEXR_ERROR_NOT_FOUND;\n}", runtime_source) + self.assertNotIn("*info = TileXRCcuLocalResourceWindowInfo{};\n return TILEXR_ERROR_NOT_FOUND;", export_local_body) + self.assertNotIn("buffers->clear();\n return TILEXR_ERROR_NOT_FOUND;", export_remote_body) + self.assertNotIn("*snapshot = TileXRCcuLowerLayerTransportSnapshot{};\n return TILEXR_ERROR_NOT_FOUND;", export_snapshot_body) + self.assertLess( + prepare_from_allocation_body.index("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute"), + prepare_from_allocation_body.index("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers"), + ) + + register_memory_body = comm_source[ + comm_source.index("int TileXRComm::RegisterUDMAMemory"): + comm_source.index("int TileXRComm::UnregisterUDMAMemory") + ] + self.assertNotIn("RefreshDirectCcuLowerLayerPlan();", register_memory_body) + self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_memory_body) + + def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + exchange_body = comm_source[ + comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): + comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + ] + compact_body = " ".join(exchange_body.split()) + + self.assertIn( + "channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( peerResources.remoteXnStartId, peerLocalIndex, syncIndex, peerRouteCount)", + compact_body) + self.assertIn( + "peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset)", + compact_body) + self.assertIn("selectedRemoteXnOffset >= peerResources.remoteXnCount", compact_body) + self.assertNotIn("SelectDirectCcuRemoteBindingOverride", compact_body) + self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) + self.assertNotIn("(*remoteCcuBuffers)[routeIndex].remoteCcuVa +=", compact_body) + self.assertNotIn("static_cast(peerLocalXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) + self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET + static_cast(peerLocalXnId)", compact_body) + self.assertNotIn( + "uint16_t remoteXnId = static_cast(peerResources.localXnStartId + peerLocalIndex)", + compact_body) + self.assertNotIn( + "channelBoundRemoteXnId = static_cast(allocation.remoteXn.startId + routeIndex)", + compact_body) + self.assertNotIn( + "static_cast((*remoteCcuBuffers)[routeIndex].remoteXnId) * TILEXR_CCU_XN_SLOT_BYTES", + compact_body) + + def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + exchange_body = comm_source[ + comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): + comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + ] + compact_body = " ".join(exchange_body.split()) + + self.assertIn("peerLocalWaitCkeOffset", exchange_body) + self.assertIn("peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount", compact_body) + self.assertIn( + "remoteNotifyCke = static_cast(static_cast(peerResources.localWaitCkeStartId) + peerLocalWaitCkeOffset)", + compact_body) + self.assertNotIn( + "remoteNotifyCke = SelectDirectCcuRemoteNotifyCkeId( allocation.remoteNotifyCke.startId, routeIndex)", + compact_body) + self.assertIn("notifyCkeOwnerVerified", exchange_body) + self.assertIn("notifyCkeOwnerVerified &&", compact_body) + + def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + exchange_body = comm_source[ + comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): + comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + ] + compact_body = " ".join(exchange_body.split()) + + self.assertIn("const size_t peerRouteCount = static_cast(rankSize_ - 1)", compact_body) + self.assertIn("const size_t syncRouteCount = allocation.remoteXn.num", compact_body) + self.assertIn("allocation.remoteXn.num < static_cast(rankSize_ - 1)", compact_body) + self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize_ - 1)", compact_body) + self.assertIn("std::vector peerCcuBuffers = *remoteCcuBuffers", compact_body) + self.assertIn("remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{})", compact_body) + self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) + self.assertIn("const size_t peerBufferIndex = syncIndex % peerRouteCount", compact_body) + self.assertIn("(*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]", compact_body) + self.assertIn("channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) + + def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self): + runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + export_body = runtime_source[ + runtime_source.index("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers"): + runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot") + ] + compact_body = " ".join(export_body.split()) + + self.assertIn("ImportPeerEndpointRoute(", runtime_source) + self.assertIn( + "if (importedPeerRoute) { remote.remoteEid = importedRoute.remoteEid; remote.tpn = importedRoute.tpn", + compact_body) + self.assertIn("} else { remote.tpn = peerWindow.tpn", compact_body) + self.assertIn("TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET", runtime_source) + self.assertIn("const uint64_t remoteCcuVaOffset = SelectRemoteCcuVaOffset()", compact_body) + self.assertIn("remote.remoteCcuVa = peerWindow.addr + remoteCcuVaOffset", compact_body) + self.assertNotIn("remote.remoteCcuVa = peerWindow.addr;", compact_body) + self.assertIn("remote.localDoorbellVa = localVerifiedEndpointRoute_.doorbellVa", compact_body) + self.assertIn("remote.localDoorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId", compact_body) + self.assertIn("remote.localDoorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue", compact_body) + + def test_direct_ccu_runtime_can_override_resource_window_token_from_rank_env(self): + code = textwrap.dedent( + r''' + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int FakeRaCustomChannel(TileXRCcuRaInfo, TileXRCcuCustomChannelIn*, TileXRCcuCustomChannelOut*) + { + return 0; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.loader_.RaCustomChannel = FakeRaCustomChannel; + runtime.loader_.loaded_ = true; + runtime.options_.rank = 1; + runtime.cachedBasicInfoValid_ = true; + runtime.cachedBasicInfo_.resourceAddr = 0x10000000ULL; + runtime.cachedBasicInfo_.msidToken.tokenId = 0x1234; + runtime.cachedBasicInfo_.msidToken.tokenValue = 0x5678; + runtime.cachedBasicInfo_.msidToken.valid = true; + runtime.cachedBasicInfo_.caps.cap1 = 7U << 16U; + + if (runtime.RegisterCcuResourceRmaBuffer(0x10000000ULL) != TILEXR_SUCCESS) { + std::cerr << "register failed\n"; + return 1; + } + TileXRCcuLocalResourceWindowInfo local; + if (runtime.ExportLocalCcuRmaBuffer(&local) != TILEXR_SUCCESS) { + std::cerr << "export failed\n"; + return 2; + } + if (local.tokenId != 0x2222U || + local.rawTokenId != 0x3333U || + local.tokenValue != 0x4444U) { + std::cerr << "rank override was not applied tokenId=" << local.tokenId + << " rawTokenId=" << local.rawTokenId + << " tokenValue=" << local.tokenValue << "\n"; + return 3; + } + + TileXRCcuDirectRuntime defaultRuntime; + defaultRuntime.initialized_ = true; + defaultRuntime.loader_.RaCustomChannel = FakeRaCustomChannel; + defaultRuntime.loader_.loaded_ = true; + defaultRuntime.options_.rank = 0; + defaultRuntime.cachedBasicInfoValid_ = true; + defaultRuntime.cachedBasicInfo_ = runtime.cachedBasicInfo_; + if (defaultRuntime.RegisterCcuResourceRmaBuffer(0x10000000ULL) != TILEXR_SUCCESS || + defaultRuntime.ExportLocalCcuRmaBuffer(&local) != TILEXR_SUCCESS) { + std::cerr << "default register/export failed\n"; + return 4; + } + if (local.tokenId != 0x1111U || + local.rawTokenId != 0x1111U || + local.tokenValue != 0x7777U) { + std::cerr << "common override fallback was not applied\n"; + return 5; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID"] = "0x1111" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE"] = "0x7777" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID_RANK1"] = "0x2222" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_RAW_TOKEN_ID_RANK1"] = "0x3333" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_RANK1"] = "0x4444" + + result = self.compile_and_run( + code, + env=env, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_can_register_resource_window_with_public_ra_ctx(self): + code = textwrap.dedent( + r''' + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + constexpr uint64_t kResourceAddr = 0x10001234ULL; + constexpr uint64_t kAlignedResourceAddr = 0x10001000ULL; + constexpr uint64_t kAlignedResourceBytes = + TILEXR_CCU_RESOURCE_WINDOW_BYTES + (kResourceAddr - kAlignedResourceAddr); + + int FakeRaCustomChannel(TileXRCcuRaInfo, TileXRCcuCustomChannelIn*, TileXRCcuCustomChannelOut*) + { + return 0; + } + + int FakeRaGetDevEidInfoNum(TileXRCcuRaInfo, uint32_t* num) + { + *num = 1; + return 0; + } + + int FakeRaGetDevEidInfoList(TileXRCcuRaInfo, TileXRCcuHccpDevEidInfo* list, uint32_t* num) + { + if (list == nullptr || num == nullptr || *num != 1) { + return -1; + } + list[0].eidIndex = 3; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + list[0].eid.raw[i] = static_cast(0xa0 + i); + } + return 0; + } + + int FakeRaCtxInit(TileXRCcuHccpCtxInitCfg*, TileXRCcuHccpCtxInitAttr* attr, void** ctx) + { + if (attr == nullptr || attr->phyId != 0x55 || attr->ub.eidIndex != 3) { + return -1; + } + *ctx = reinterpret_cast(0x1000); + return 0; + } + + int FakeRaCtxDeinit(void*) + { + return 0; + } + + int FakeRaCtxTokenIdAlloc(void* ctx, TileXRCcuHccpTokenId* token, void** tokenHandle) + { + if (ctx != reinterpret_cast(0x1000)) { + return -1; + } + token->tokenId = 0x12345600U; + *tokenHandle = reinterpret_cast(0x2000); + return 0; + } + + int FakeRaCtxTokenIdFree(void*, void*) + { + return 0; + } + + int FakeRaGetSecRandom(TileXRCcuRaInfo* info, uint32_t* value) + { + if (info == nullptr || info->phyId != 0x55 || info->mode != TILEXR_CCU_NETWORK_OFFLINE) { + return -1; + } + *value = 0xabcdef01U; + return 0; + } + + int FakeRaCtxLmemRegister(void* ctx, TileXRCcuHccpMrRegInfo* mr, void** handle) + { + if (ctx != reinterpret_cast(0x1000) || mr == nullptr || + mr->in.mem.addr != kAlignedResourceAddr || mr->in.mem.size != kAlignedResourceBytes || + mr->in.ub.tokenValue != 0xabcdef01U || + mr->in.ub.tokenIdHandle != reinterpret_cast(0x2000) || + mr->in.ub.flags.bs.tokenIdValid != 1 || + mr->in.ub.flags.bs.nonPin != 1) { + return -1; + } + mr->out.ub.tokenId = 0x12345600U; + mr->out.ub.targetSegHandle = 0x4455667788ULL; + mr->out.key.size = 5; + *handle = reinterpret_cast(0x3000); + return 0; + } + + int FakeRaCtxLmemUnregister(void*, void*) + { + return 0; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.loader_.RaCustomChannel = FakeRaCustomChannel; + runtime.loader_.loaded_ = true; + runtime.loader_.RaGetDevEidInfoNum = FakeRaGetDevEidInfoNum; + runtime.loader_.RaGetDevEidInfoList = FakeRaGetDevEidInfoList; + runtime.loader_.RaCtxInit = FakeRaCtxInit; + runtime.loader_.RaCtxDeinit = FakeRaCtxDeinit; + runtime.loader_.RaCtxTokenIdAlloc = FakeRaCtxTokenIdAlloc; + runtime.loader_.RaCtxTokenIdFree = FakeRaCtxTokenIdFree; + runtime.loader_.RaGetSecRandom = FakeRaGetSecRandom; + runtime.loader_.RaCtxLmemRegister = FakeRaCtxLmemRegister; + runtime.loader_.RaCtxLmemUnregister = FakeRaCtxLmemUnregister; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 0; + runtime.cachedBasicInfoValid_ = true; + runtime.cachedBasicInfo_.resourceAddr = kResourceAddr; + runtime.cachedBasicInfo_.msidToken.tokenId = 0x1111; + runtime.cachedBasicInfo_.msidToken.tokenValue = 0x2222; + runtime.cachedBasicInfo_.msidToken.valid = true; + runtime.cachedBasicInfo_.caps.cap1 = 7U << 16U; + + if (runtime.RegisterCcuResourceRmaBuffer(kResourceAddr) != TILEXR_SUCCESS) { + std::cerr << "ra ctx resource window register failed\n"; + return 1; + } + + TileXRCcuLocalResourceWindowInfo local; + if (runtime.ExportLocalCcuRmaBuffer(&local) != TILEXR_SUCCESS) { + std::cerr << "export failed\n"; + return 2; + } + if (local.tokenId != 0x123456U || + local.rawTokenId != 0x12345600U || + local.tokenValue != 0xabcdef01U || + local.addr != kResourceAddr || + local.bytes != TILEXR_CCU_RESOURCE_WINDOW_BYTES) { + std::cerr << "unexpected registered resource window tokenId=" << local.tokenId + << " rawTokenId=" << local.rawTokenId + << " tokenValue=" << local.tokenValue + << " bytes=" << local.bytes << "\n"; + return 3; + } + + runtime.Shutdown(); + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"] = "ra_ctx" + + result = self.compile_and_run( + code, + env=env, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_collects_ra_ctx_endpoint_route_when_resource_window_uses_ra_ctx(self): + code = textwrap.dedent( + r''' + #include + #include + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + constexpr uint32_t kExpectedSqDepth = 8; + + int EchoLocalExchangeAsPeer(const void* sendBuf, size_t sendBytes, void* recvBuf, void*) + { + if (sendBuf == nullptr || recvBuf == nullptr || + sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + if (!local->endpointRouteVerified || + local->remoteEid[0] != 0xb0 || + local->tpn != 0x010203 || + local->doorbellVa != 0x1122334455667788ULL || + local->doorbellTokenId != 0x3456 || + local->sqDepth != kExpectedSqDepth) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = *local; + return TILEXR_SUCCESS; + } + + int FakeRaCtxCqCreate(void* ctx, TileXRCcuHccpCqInfo*, void** cqHandle) + { + if (ctx != reinterpret_cast(0x1000)) { + return -1; + } + *cqHandle = reinterpret_cast(0x2000); + return 0; + } + + int FakeRaCtxCqDestroy(void*, void*) + { + return 0; + } + + int FakeRaCtxQpCreate( + void* ctx, + TileXRCcuHccpQpCreateAttr* attr, + TileXRCcuHccpQpCreateInfo* info, + void** qpHandle) + { + if (ctx != reinterpret_cast(0x1000) || + attr == nullptr || + attr->ub.tokenIdHandle != reinterpret_cast(0x1100) || + attr->ub.tokenValue != 0xabcdef01U || + info == nullptr) { + return -1; + } + info->key.size = 4; + info->ub.dbAddr = 0x1122334455667788ULL; + info->ub.dbTokenId = 0x345600U; + *qpHandle = reinterpret_cast(0x3000); + return 0; + } + + int FakeRaCtxQpDestroy(void*) + { + return 0; + } + + int FakeRaGetTpInfoListAsync( + void* ctx, + TileXRCcuHccpGetTpCfg*, + TileXRCcuHccpTpInfo infoList[], + uint32_t* num, + void** reqHandle) + { + if (ctx != reinterpret_cast(0x1000) || + infoList == nullptr || + num == nullptr || + *num == 0) { + return -1; + } + infoList[0].tpHandle = 0x99887766ULL; + *num = 1; + *reqHandle = reinterpret_cast(0x4000); + return 0; + } + + int FakeRaGetAsyncReqResult(void* reqHandle, int* reqResult) + { + if (reqHandle != reinterpret_cast(0x4000) || reqResult == nullptr) { + return -1; + } + *reqResult = 0; + return 0; + } + + int FakeRaCtxQpImport(void* ctx, TileXRCcuHccpQpImportInfo* info, void** remoteQpHandle) + { + if (ctx != reinterpret_cast(0x1000) || + info == nullptr || + info->in.ub.expImportCfg.tpHandle != 0x99887766ULL) { + return -1; + } + info->out.ub.tpn = 0x010203; + *remoteQpHandle = reinterpret_cast(0x5000); + return 0; + } + + int FakeRaCtxQpUnimport(void*, void*) + { + return 0; + } + + int main() + { + unsetenv("TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE"); + unsetenv("TILEXR_CCU_ENDPOINT_ROUTE_PROVIDER"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_ID"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_VALUE"); + unsetenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_SQ_DEPTH"); + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = EchoLocalExchangeAsPeer; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = TILEXR_CCU_RESOURCE_WINDOW_BYTES; + runtime.localResourceWindow_.tokenId = 0x123456U; + runtime.localResourceWindow_.rawTokenId = 0x12345600U; + runtime.localResourceWindow_.tokenValue = 0xabcdef01U; + runtime.localResourceWindow_.raCtxHandle = reinterpret_cast(0x1000); + runtime.localResourceWindow_.tokenIdHandle = reinterpret_cast(0x1100); + runtime.localResourceWindow_.raCtxRegistered = true; + for (uint32_t i = 0; i < runtime.localResourceWindow_.eid.size(); ++i) { + runtime.localResourceWindow_.eid[i] = static_cast(0xb0 + i); + } + runtime.loader_.RaCtxCqCreate = FakeRaCtxCqCreate; + runtime.loader_.RaCtxCqDestroy = FakeRaCtxCqDestroy; + runtime.loader_.RaCtxQpCreate = FakeRaCtxQpCreate; + runtime.loader_.RaCtxQpDestroy = FakeRaCtxQpDestroy; + runtime.loader_.RaCtxQpImport = FakeRaCtxQpImport; + runtime.loader_.RaCtxQpUnimport = FakeRaCtxQpUnimport; + runtime.loader_.RaGetTpInfoListAsync = FakeRaGetTpInfoListAsync; + runtime.loader_.RaGetAsyncReqResult = FakeRaGetAsyncReqResult; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_SUCCESS) { + std::cerr << "ra ctx endpoint route was not collected by default: " + << report.message << "\n"; + return 1; + } + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed after ra ctx route collection\n"; + return 2; + } + if (buffers.size() != 1 || + !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0xb0 || + buffers[0].tpn != 0x010203 || + buffers[0].doorbellVa != 0x1122334455667788ULL || + buffers[0].doorbellTokenId != 0x3456 || + buffers[0].doorbellTokenValue != 0xabcdef01U || + buffers[0].sqDepth != kExpectedSqDepth) { + std::cerr << "ra ctx collected endpoint route was not exported\n"; + return 3; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"] = "ra_ctx" + + result = self.compile_and_run( + code, + env=env, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_rejects_zero_resource_window_token_override(self): + code = textwrap.dedent( + r''' + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int FakeRaCustomChannel(TileXRCcuRaInfo, TileXRCcuCustomChannelIn*, TileXRCcuCustomChannelOut*) + { + return 0; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.loader_.RaCustomChannel = FakeRaCustomChannel; + runtime.loader_.loaded_ = true; + runtime.options_.rank = 0; + runtime.cachedBasicInfoValid_ = true; + runtime.cachedBasicInfo_.resourceAddr = 0x10000000ULL; + runtime.cachedBasicInfo_.msidToken.tokenId = 0x1234; + runtime.cachedBasicInfo_.msidToken.valid = true; + + if (runtime.RegisterCcuResourceRmaBuffer(0x10000000ULL) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "zero token override was accepted\n"; + return 1; + } + TileXRCcuLocalResourceWindowInfo local; + if (runtime.ExportLocalCcuRmaBuffer(&local) != TILEXR_ERROR_NOT_INITIALIZED) { + std::cerr << "resource window remained registered after invalid override\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID"] = "0" + + result = self.compile_and_run( + code, + env=env, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_preserves_verified_endpoint_routes_from_template(self): + code = textwrap.dedent( + r''' + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x1234; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.localResourceWindow_.addr = 0x100000000ULL; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.tokenValue = 0x5678; + + TileXRCcuLowerLayerTransportSnapshot templ; + templ.dieId = 1; + templ.xnStartId = 1961; + TileXRCcuLowerLayerTransportRoute route; + route.peerRank = 1; + route.channelId = 9; + for (uint32_t i = 0; i < route.remoteEid.size(); ++i) { + route.remoteEid[i] = static_cast(0xa0 + i); + } + route.tpn = 0; + route.doorbellVa = 0x1122334455667788ULL; + route.doorbellTokenId = 0x2345; + route.doorbellTokenValue = 0; + route.sqDepth = 64; + route.endpointRouteVerified = true; + templ.routes.push_back(route); + + TileXRCcuLowerLayerTransportRoute synthetic = route; + synthetic.peerRank = 2; + synthetic.channelId = 10; + synthetic.remoteEid = {}; + synthetic.tpn = 0; + synthetic.doorbellVa = 0; + synthetic.doorbellTokenId = 0; + synthetic.doorbellTokenValue = 0; + synthetic.sqDepth = 0; + synthetic.endpointRouteVerified = true; + templ.routes.push_back(synthetic); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + if (runtime.ExportLowerLayerTransportSnapshot(templ, &snapshot) != TILEXR_SUCCESS) { + std::cerr << "export snapshot failed\n"; + return 4; + } + if (snapshot.routes.size() != 2 || !snapshot.routes[0].endpointRouteVerified || + snapshot.routes[0].remoteEid[0] != 0xa0 || + snapshot.routes[0].tpn != route.tpn || + snapshot.routes[0].doorbellVa != route.doorbellVa || + snapshot.routes[0].doorbellTokenId != route.doorbellTokenId || + snapshot.routes[0].doorbellTokenValue != route.doorbellTokenValue || + snapshot.routes[0].sqDepth != route.sqDepth) { + std::cerr << "verified endpoint route was not preserved\n"; + return 5; + } + if (snapshot.routes[1].endpointRouteVerified || + snapshot.routes[1].remoteEid[0] == 0 || + snapshot.routes[1].tpn == 0 || + snapshot.routes[1].doorbellVa == 0 || + snapshot.routes[1].doorbellTokenId != runtime.localResourceWindow_.tokenId || + snapshot.routes[1].doorbellTokenValue != runtime.localResourceWindow_.tokenValue || + snapshot.routes[1].sqDepth == 0) { + std::cerr << "synthetic endpoint route did not fail closed\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_can_trust_synthetic_endpoint_routes_only_with_diagnostic_env(self): + code = textwrap.dedent( + r''' + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int main() + { + setenv("TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE", "1", 1); + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x1234; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.localResourceWindow_.addr = 0x100000000ULL; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.tokenValue = 0x5678; + + TileXRCcuLowerLayerTransportSnapshot templ; + templ.dieId = 1; + templ.xnStartId = 1961; + TileXRCcuLowerLayerTransportRoute route; + route.peerRank = 1; + route.channelId = 9; + route.endpointRouteVerified = true; + templ.routes.push_back(route); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + if (runtime.ExportLowerLayerTransportSnapshot(templ, &snapshot) != TILEXR_SUCCESS) { + std::cerr << "export snapshot failed\n"; + return 1; + } + if (snapshot.routes.size() != 1 || + !snapshot.routes[0].endpointRouteVerified || + snapshot.routes[0].remoteEid[0] == 0 || + snapshot.routes[0].tpn == 0 || + snapshot.routes[0].doorbellVa == 0 || + snapshot.routes[0].doorbellTokenId != runtime.localResourceWindow_.tokenId || + snapshot.routes[0].doorbellTokenValue != runtime.localResourceWindow_.tokenValue || + snapshot.routes[0].sqDepth == 0) { + std::cerr << "synthetic endpoint route was not trusted under diagnostic env\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_exchanges_verified_endpoint_route_with_resource_window(self): + code = textwrap.dedent( + r''' + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int FakeAllGather(const void* sendBuf, size_t sendBytes, void* recvBuf, void* userData) + { + (void)sendBuf; + if (sendBytes != sizeof(TileXRCcuResourceWindowExchange) || userData == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* peer = static_cast(userData); + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = *peer; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuResourceWindowExchange peer {}; + peer.addr = 0x0000009234000000ULL; + peer.bytes = 0x2000; + peer.tokenId = 0x23456; + peer.rawTokenId = 0x33456; + peer.tokenValue = 0x5678; + for (uint32_t i = 0; i < peer.remoteEid.size(); ++i) { + peer.remoteEid[i] = static_cast(0x90 + i); + } + peer.tpn = 0x010203; + peer.doorbellVa = 0x1122334455667788ULL; + peer.doorbellTokenId = 0x3456; + peer.doorbellTokenValue = 0; + peer.sqDepth = 64; + peer.endpointRouteVerified = true; + peer.channelResourceOwnerVerified = true; + peer.transportResourceExchangeVerified = true; + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = FakeAllGather; + runtime.options_.allGatherUserData = &peer; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed\n"; + return 1; + } + if (buffers.size() != 1 || + buffers[0].remoteCcuVa != peer.addr || + buffers[0].memoryTokenId != peer.tokenId || + buffers[0].rawMemoryTokenId != peer.rawTokenId || + buffers[0].memoryTokenValue != peer.tokenValue || + !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0x9f || + buffers[0].remoteEid[15] != 0x90 || + buffers[0].tpn != peer.tpn || + buffers[0].doorbellVa != peer.doorbellVa || + buffers[0].doorbellTokenId != peer.doorbellTokenId || + buffers[0].doorbellTokenValue != peer.doorbellTokenValue || + buffers[0].sqDepth != peer.sqDepth || + !buffers[0].channelResourceOwnerVerified || + !buffers[0].transportResourceExchangeVerified) { + std::cerr << "verified endpoint route was not exchanged with resource window\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_imported_peer_route_uses_hcomm_reverse_channel_eid(self): + code = textwrap.dedent( + r''' + #include + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + struct TpExchange { + uint64_t tpHandles[128] = {}; + uint32_t psn = 0; + }; + + struct ExchangeState { + TileXRCcuResourceWindowExchange peer; + }; + + int FakeAllGather(const void* sendBuf, size_t sendBytes, void* recvBuf, void* userData) + { + if (sendBuf == nullptr || recvBuf == nullptr || userData == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + auto* state = static_cast(userData); + if (sendBytes == sizeof(TileXRCcuResourceWindowExchange)) { + const auto* local = static_cast(sendBuf); + if (!local->endpointRouteVerified || local->qpKey.size == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = state->peer; + return TILEXR_SUCCESS; + } + if (sendBytes == sizeof(TpExchange)) { + const auto* local = static_cast(sendBuf); + if (local->tpHandles[1] != 0x1111222233334444ULL || local->psn != 5) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TpExchange) * 2); + out[1].tpHandles[0] = 0x5555666677778888ULL; + out[1].psn = 9; + return TILEXR_SUCCESS; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + int FakeGetTpInfoListAsync( + void* ctx, + TileXRCcuHccpGetTpCfg* cfg, + TileXRCcuHccpTpInfo* tpInfo, + uint32_t* tpInfoNum, + void** reqHandle) + { + if (ctx != reinterpret_cast(0x1000) || cfg == nullptr || + tpInfo == nullptr || tpInfoNum == nullptr || reqHandle == nullptr || + cfg->peerEid.raw[0] != 0xa0 || cfg->peerEid.raw[15] != 0xaf) { + return -1; + } + tpInfo->tpHandle = 0x1111222233334444ULL; + *tpInfoNum = 1; + *reqHandle = reinterpret_cast(0x2000); + return 0; + } + + int FakeGetAsyncReqResult(void* reqHandle, int* reqResult) + { + if (reqHandle != reinterpret_cast(0x2000) || reqResult == nullptr) { + return -1; + } + *reqResult = 0; + return 0; + } + + int FakeQpImport(void* ctx, TileXRCcuHccpQpImportInfo* info, void** remoteQpHandle) + { + if (ctx != reinterpret_cast(0x1000) || info == nullptr || remoteQpHandle == nullptr || + info->in.ub.expImportCfg.tpHandle != 0x1111222233334444ULL || + info->in.ub.expImportCfg.peerTpHandle != 0x5555666677778888ULL || + info->in.ub.expImportCfg.txPsn != 5 || + info->in.ub.expImportCfg.rxPsn != 9 || + info->in.key.size == 0) { + return -1; + } + info->out.ub.tpn = 0x47; + *remoteQpHandle = reinterpret_cast(0x3000); + return 0; + } + + int main() + { + ExchangeState state; + state.peer.addr = 0x0000009234000000ULL; + state.peer.bytes = 0x2000; + state.peer.tokenId = 0x23456; + state.peer.rawTokenId = 0x33456; + state.peer.tokenValue = 0x5678; + for (uint32_t i = 0; i < state.peer.remoteEid.size(); ++i) { + state.peer.remoteEid[i] = static_cast(0xa0 + i); + state.peer.qpKey.value[i] = static_cast(0x40 + i); + } + state.peer.qpKey.size = TILEXR_CCU_HCCP_QP_KEY_BYTES; + state.peer.endpointRouteVerified = true; + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = FakeAllGather; + runtime.options_.allGatherUserData = &state; + runtime.localResourceWindow_.raCtxHandle = reinterpret_cast(0x1000); + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + runtime.localResourceWindow_.eid[0] = 0x10; + runtime.localResourceWindow_.eid[15] = 0x1f; + runtime.loader_.RaGetTpInfoListAsync = FakeGetTpInfoListAsync; + runtime.loader_.RaGetAsyncReqResult = FakeGetAsyncReqResult; + runtime.loader_.RaCtxQpImport = FakeQpImport; + runtime.endpointPsn_ = 5; + runtime.endpointQpKeyValid_ = true; + runtime.endpointQpKey_.size = TILEXR_CCU_HCCP_QP_KEY_BYTES; + runtime.localVerifiedEndpointRouteValid_ = true; + runtime.localVerifiedEndpointRoute_.endpointRouteVerified = true; + runtime.localVerifiedEndpointRoute_.remoteEid[0] = 0x10; + runtime.localVerifiedEndpointRoute_.tpn = 0x22; + runtime.localVerifiedEndpointRoute_.doorbellVa = 0x1122334455667788ULL; + runtime.localVerifiedEndpointRoute_.doorbellTokenId = 0x3456; + runtime.localVerifiedEndpointRoute_.doorbellTokenValue = 0x4567; + runtime.localVerifiedEndpointRoute_.sqDepth = 8; + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed\n"; + return 1; + } + if (buffers.size() != 1 || buffers[0].tpn != 0x47 || + buffers[0].remoteEid[0] != 0xaf || buffers[0].remoteEid[15] != 0xa0 || + buffers[0].doorbellVa != runtime.localVerifiedEndpointRoute_.doorbellVa || + buffers[0].doorbellTokenId != runtime.localVerifiedEndpointRoute_.doorbellTokenId || + !buffers[0].endpointRouteVerified) { + std::cerr << "imported peer route did not use hcomm-style reverse EID and imported TPN" + << " size=" << buffers.size(); + if (!buffers.empty()) { + std::cerr << " tpn=0x" << std::hex << buffers[0].tpn + << " eid0=0x" << static_cast(buffers[0].remoteEid[0]) + << " eid15=0x" << static_cast(buffers[0].remoteEid[15]) + << " doorbellVa=0x" << buffers[0].doorbellVa + << " doorbellTokenId=0x" << buffers[0].doorbellTokenId + << std::dec + << " endpointRouteVerified=" << buffers[0].endpointRouteVerified; + } + std::cerr << "\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_exports_global_peer_rank_with_remote_buffer(self): + code = textwrap.dedent( + r''' + #include + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int FakeAllGather(const void* sendBuf, size_t sendBytes, void* recvBuf, void*) + { + if (sendBuf == nullptr || recvBuf == nullptr || + sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = *local; + out[1].addr = 0x20000000ULL; + out[1].bytes = 0x2000; + out[1].tokenId = 0x2222; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = FakeAllGather; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed\n"; + return 1; + } + if (buffers.size() != 1 || buffers[0].peerRank != 1) { + std::cerr << "global peer rank was not exported with remote buffer\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_exports_configured_local_verified_endpoint_route(self): + code = textwrap.dedent( + r''' + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int EchoLocalExchangeAsPeer(const void* sendBuf, size_t sendBytes, void* recvBuf, void* userData) + { + (void)userData; + if (sendBuf == nullptr || sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + if (!local->endpointRouteVerified || + local->remoteEid[0] != 0x70 || + local->tpn != 0x010203 || + local->doorbellVa != 0x1122334455667788ULL || + local->doorbellTokenId != 0x3456 || + local->sqDepth != 64) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = *local; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuLowerLayerTransportRoute localRoute; + for (uint32_t i = 0; i < localRoute.remoteEid.size(); ++i) { + localRoute.remoteEid[i] = static_cast(0x70 + i); + } + localRoute.tpn = 0x010203; + localRoute.doorbellVa = 0x1122334455667788ULL; + localRoute.doorbellTokenId = 0x3456; + localRoute.doorbellTokenValue = 0; + localRoute.sqDepth = 64; + localRoute.endpointRouteVerified = true; + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = EchoLocalExchangeAsPeer; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + + if (runtime.ConfigureLocalVerifiedEndpointRoute(localRoute) != TILEXR_SUCCESS) { + std::cerr << "failed to configure local verified endpoint route\n"; + return 1; + } + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed\n"; + return 2; + } + if (buffers.size() != 1 || + !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0x70 || + buffers[0].tpn != localRoute.tpn || + buffers[0].doorbellVa != localRoute.doorbellVa || + buffers[0].doorbellTokenId != localRoute.doorbellTokenId || + buffers[0].doorbellTokenValue != localRoute.doorbellTokenValue || + buffers[0].sqDepth != localRoute.sqDepth) { + std::cerr << "configured local verified endpoint route was not exported\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_collects_local_verified_endpoint_route_before_exchange(self): + code = textwrap.dedent( + r''' + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + struct CollectorState { + int calls = 0; + uint32_t observedDevicePhyId = 0; + uint64_t observedResourceWindow = 0; + }; + + int FakeCollector( + uint32_t devicePhyId, + const TileXRCcuLocalResourceWindowInfo& localResourceWindow, + TileXRCcuLowerLayerTransportRoute* route, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + state->observedDevicePhyId = devicePhyId; + state->observedResourceWindow = localResourceWindow.addr; + for (uint32_t i = 0; i < route->remoteEid.size(); ++i) { + route->remoteEid[i] = static_cast(0x80 + i); + } + route->tpn = 0x010203; + route->doorbellVa = 0x1122334455667788ULL; + route->doorbellTokenId = 0x3456; + route->doorbellTokenValue = 0; + route->sqDepth = 64; + route->endpointRouteVerified = true; + return TILEXR_SUCCESS; + } + + int EchoLocalExchangeAsPeer(const void* sendBuf, size_t sendBytes, void* recvBuf, void*) + { + if (sendBuf == nullptr || sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + if (!local->endpointRouteVerified || local->remoteEid[0] != 0x80 || + local->tpn != 0x010203 || local->doorbellVa != 0x1122334455667788ULL || + local->doorbellTokenId != 0x3456 || local->sqDepth != 64) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[1] = *local; + return TILEXR_SUCCESS; + } + + int main() + { + CollectorState state; + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x1234; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.allGather = EchoLocalExchangeAsPeer; + runtime.options_.localEndpointRouteCollector = FakeCollector; + runtime.options_.localEndpointRouteCollectorUserData = &state; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_SUCCESS) { + std::cerr << "failed to refresh local endpoint route: " << report.message << "\n"; + return 1; + } + if (state.calls != 1 || state.observedDevicePhyId != 0x1234 || + state.observedResourceWindow != runtime.localResourceWindow_.addr) { + std::cerr << "collector did not receive runtime context\n"; + return 2; + } + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed after collector refresh\n"; + return 3; + } + if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0x80 || + buffers[0].tpn != 0x010203 || + buffers[0].doorbellVa != 0x1122334455667788ULL || + buffers[0].doorbellTokenId != 0x3456 || + buffers[0].doorbellTokenValue != 0 || + buffers[0].sqDepth != 64) { + std::cerr << "collected local endpoint route was not exported\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_endpoint_collector_failure_fails_closed(self): + code = textwrap.dedent( + r''' + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int IncompleteCollector( + uint32_t, + const TileXRCcuLocalResourceWindowInfo&, + TileXRCcuLowerLayerTransportRoute* route, + void*) + { + route->remoteEid[0] = 0x90; + route->tpn = 0; + route->doorbellVa = 0x1122334455667788ULL; + route->doorbellTokenId = 0; + route->sqDepth = 64; + route->endpointRouteVerified = true; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x1234; + runtime.options_.rank = 0; + runtime.options_.rankSize = 2; + runtime.options_.localEndpointRouteCollector = IncompleteCollector; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "incomplete collected route was accepted\n"; + return 1; + } + if (runtime.localVerifiedEndpointRouteValid_) { + std::cerr << "incomplete collected route remained verified\n"; + return 2; + } + + TileXRCcuLowerLayerTransportSnapshot templ; + templ.dieId = 1; + templ.xnStartId = 1961; + TileXRCcuLowerLayerTransportRoute route; + route.peerRank = 1; + route.channelId = 9; + templ.routes.push_back(route); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + if (runtime.ExportLowerLayerTransportSnapshot(templ, &snapshot) != TILEXR_SUCCESS) { + std::cerr << "export snapshot failed\n"; + return 3; + } + if (snapshot.routes.size() != 1 || snapshot.routes[0].endpointRouteVerified || + snapshot.routes[0].remoteEid[0] == 0 || snapshot.routes[0].tpn == 0 || + snapshot.routes[0].doorbellVa == 0 || snapshot.routes[0].doorbellTokenId == 0 || + snapshot.routes[0].sqDepth == 0) { + std::cerr << "collector failure did not fail closed to synthetic unverified route\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_collects_ranked_env_local_verified_endpoint_route(self): + code = textwrap.dedent( + r''' + #include + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + + int EchoLocalExchangeAsPeer(const void* sendBuf, size_t sendBytes, void* recvBuf, void*) + { + if (sendBuf == nullptr || sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + if (!local->endpointRouteVerified || local->remoteEid[0] != 0xa0 || + local->tpn != 0x010203 || local->doorbellVa != 0x1122334455667788ULL || + local->doorbellTokenId != 0x3456 || local->doorbellTokenValue != 0 || + local->sqDepth != 64) { + return TILEXR_ERROR_NOT_FOUND; + } + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[0] = *local; + return TILEXR_SUCCESS; + } + + int main() + { + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID", "00112233445566778899aabbccddeeff", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN", "7", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA", "0x1111111111111111", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_ID", "0x1111", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_VALUE", "0x22", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_SQ_DEPTH", "8", 1); + setenv( + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID_RANK1", + "a0:a1:a2:a3:a4:a5:a6:a7:a8:a9:aa:ab:ac:ad:ae:af", + 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN_RANK1", "0x010203", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA_RANK1", "0x1122334455667788", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_ID_RANK1", "0x3456", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_VALUE_RANK1", "0", 1); + setenv("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_SQ_DEPTH_RANK1", "64", 1); + + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x1234; + runtime.options_.rank = 1; + runtime.options_.rankSize = 2; + runtime.options_.allGather = EchoLocalExchangeAsPeer; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0x4567; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_SUCCESS) { + std::cerr << "ranked env route was not collected: " << report.message << "\n"; + return 1; + } + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed after env route collection\n"; + return 2; + } + if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0xa0 || + buffers[0].remoteEid[15] != 0xaf || + buffers[0].tpn != 0x010203 || + buffers[0].doorbellVa != 0x1122334455667788ULL || + buffers[0].doorbellTokenId != 0x3456 || + buffers[0].doorbellTokenValue != 0 || + buffers[0].sqDepth != 64) { + std::cerr << "ranked env route was not exported\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_tilexr_comm_caches_direct_ccu_basic_info_without_submit_readiness(self): + comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + + self.assertIn("RefreshDirectCcuBasicInfo", comm_header) + self.assertIn("HasDirectCcuBasicInfo", comm_header) + self.assertIn("GetDirectCcuBasicInfoStatus", comm_header) + self.assertIn("GetDirectCcuBasicInfo", comm_header) + self.assertIn("GetDirectCcuBasicInfoReport", comm_header) + self.assertIn("directCcuBasicInfo_", comm_header) + self.assertIn("directCcuBasicInfoReport_", comm_header) + + self.assertIn("int TileXRComm::RefreshDirectCcuBasicInfo", comm_source) + self.assertIn("bool TileXRComm::HasDirectCcuBasicInfo", comm_source) + self.assertIn("ccuDirectRuntime_->QueryBasicInfo", comm_source) + self.assertIn("direct CCU basic info cached", comm_source) + self.assertIn("ResetDirectCcuBasicInfo", comm_source) + self.assertIn("ResetDirectCcuBasicInfo();", comm_source) + self.assertNotIn("udmaTransport_->" + "QueryCcuBasicInfo", comm_source) + + for forbidden in [ + "TileXRCcuPrepareSubmitTasks", + "TileXRCcuSubmitTask", + "rtCCULaunch", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "libhcomm", + "libhccl_v2", + ]: + self.assertNotIn(forbidden, comm_header + "\n" + comm_source) + + def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self): + comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") + runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', comm_header) + self.assertIn('ccu/tilexr_ccu_direct_runtime.h', comm_header) + self.assertIn("PrepareDirectCcuInstallAttempt", comm_header) + self.assertIn("FillDirectCcuLowerLayerPlanFromAllocation", comm_header) + self.assertIn("PrepareDirectCcuLowerLayerPlanCallback", comm_header) + self.assertIn("int CreateDriverAdapter(", runtime_header) + + self.assertIn("int TileXRComm::PrepareDirectCcuInstallAttempt", comm_source) + self.assertIn("int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation", comm_source) + self.assertIn("int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback", comm_source) + self.assertIn("ccuDirectRuntime_->CreateDriverAdapter", comm_source) + self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)", comm_source) + self.assertIn("next.lowerLayerPlan = nullptr", comm_source) + self.assertIn( + "next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback", + comm_source, + ) + self.assertIn("next.lowerLayerPlanUserData = this", comm_source) + self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", comm_source) + self.assertIn("int TileXRCcuDirectRuntime::CreateDriverAdapter", runtime_source) + self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source) + + combined = comm_header + "\n" + comm_source + "\n" + runtime_header + "\n" + runtime_source + for forbidden in [ + "TileXRCcuPrepareSubmitTasks", + "TileXRCcuSubmitTask", + "rtCCULaunch", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "libhcomm", + "libhccl_v2", + ]: + self.assertNotIn(forbidden, combined) + + def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + + self.assertIn("g_ccuDirectRuntimeUnavailableMessage", comm_source) + init_body = comm_source[ + comm_source.index("int TileXRComm::InitDirectCcuRuntime"): + comm_source.index("int TileXRComm::InitSDMA") + ] + prepare_body = comm_source[ + comm_source.index("int TileXRComm::PrepareDirectCcuInstallAttempt"): + comm_source.index("int TileXRComm::RefreshDirectCcuLowerLayerPlan") + ] + + self.assertIn("g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message", init_body) + self.assertIn("direct CCU runtime unavailable after process-level init failure", comm_source) + self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", prepare_body) + self.assertLess( + prepare_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), + prepare_body.index("RefreshDirectCcuBasicInfo(installDieId)"), + ) + + def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + init_body = comm_source[ + comm_source.index("int TileXRComm::InitDirectCcuRuntime"): + comm_source.index("int TileXRComm::InitSDMA") + ] + + lock_pos = init_body.index("lock_guard lock(g_ccuDirectRuntimeMtx);") + allocation_pos = init_body.index("ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime())") + runtime_init_pos = init_body.index("ccuDirectRuntime_->Init(options, &runtimeReport)") + unavailable_set_pos = init_body.index("g_ccuDirectRuntimeUnavailable = true") + + self.assertLess(lock_pos, allocation_pos) + self.assertLess(allocation_pos, runtime_init_pos) + self.assertLess(runtime_init_pos, unavailable_set_pos) + self.assertEqual(1, init_body.count("lock_guard lock(g_ccuDirectRuntimeMtx);")) + + def test_tilexr_comm_direct_ccu_prepare_can_select_install_die_for_diagnostics(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + prepare_body = comm_source[ + comm_source.index("int TileXRComm::PrepareDirectCcuInstallAttempt"): + comm_source.index("int TileXRComm::RefreshDirectCcuLowerLayerPlan") + ] + + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", comm_source) + self.assertIn("SelectDirectCcuInstallDieId", comm_source) + self.assertIn("RefreshDirectCcuBasicInfo(installDieId)", prepare_body) + self.assertIn("directCcuBasicInfo_.dieId != installDieId", prepare_body) + self.assertNotIn("RefreshDirectCcuBasicInfo(0)", prepare_body) + + def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_process_init_failure(self): + comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + thread_allgather_body = comm_source[ + comm_source.index("int TileXRComm::DirectCcuThreadAllGather"): + comm_source.index("int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback") + ] + + self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", thread_allgather_body) + self.assertIn("direct CCU thread allgather abort", thread_allgather_body) + self.assertLess( + thread_allgather_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), + thread_allgather_body.index("TILEXR_INIT_TIMEOUT"), + ) + + def test_tilexr_comm_direct_ccu_lower_layer_plan_api_is_header_visible(self): + code = textwrap.dedent( + r''' + #include "tilexr_comm.h" + + #include + + using namespace TileXR; + + int main() + { + using ConfigureFn = int (TileXRComm::*)(const TileXRCcuLowerLayerTransportSnapshot&); + using ConfigureFromAllocationFn = int (TileXRComm::*)( + const TileXRCcuResourceAllocation&, + const std::vector&); + using PrepareFromAllocationFn = int (TileXRComm::*)(const TileXRCcuResourceAllocation&); + using RefreshFn = int (TileXRComm::*)(); + using HasFn = bool (TileXRComm::*)() const; + using StatusFn = int (TileXRComm::*)() const; + using ReportFn = const TileXRCcuLowerLayerPlanBuilderReport& (TileXRComm::*)() const; + using PlanFn = const TileXRCcuLowerLayerInstallPlan* (TileXRComm::*)() const; + using BasicRefreshFn = int (TileXRComm::*)(uint8_t); + using BasicHasFn = bool (TileXRComm::*)() const; + using BasicStatusFn = int (TileXRComm::*)() const; + using BasicInfoFn = const TileXRCcuBasicInfo* (TileXRComm::*)() const; + using BasicReportFn = const TileXRCcuDriverAdapterReport& (TileXRComm::*)() const; + using ConfigureVerifiedEndpointRoutesFn = int (TileXRComm::*)( + const std::vector&); + using ConfigureLocalVerifiedEndpointRouteFn = int (TileXRComm::*)( + const TileXRCcuLowerLayerTransportRoute&); + using InstallAttemptFn = int (TileXRComm::*)( + const TileXRCcuDirectInstallOptions&, + TileXRCcuDirectInstallAttempt*, + TileXRCcuDirectInstallReport*); + + ConfigureFn configure = &TileXRComm::ConfigureDirectCcuLowerLayerTemplate; + ConfigureFromAllocationFn configureFromAllocation = + &TileXRComm::ConfigureDirectCcuLowerLayerTemplateFromAllocation; + PrepareFromAllocationFn prepareFromAllocation = + &TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation; + RefreshFn refresh = &TileXRComm::RefreshDirectCcuLowerLayerPlan; + HasFn has = &TileXRComm::HasDirectCcuLowerLayerPlan; + StatusFn status = &TileXRComm::GetDirectCcuLowerLayerPlanStatus; + ReportFn report = &TileXRComm::GetDirectCcuLowerLayerPlanReport; + PlanFn plan = &TileXRComm::GetDirectCcuLowerLayerPlan; + BasicRefreshFn basicRefresh = &TileXRComm::RefreshDirectCcuBasicInfo; + BasicHasFn basicHas = &TileXRComm::HasDirectCcuBasicInfo; + BasicStatusFn basicStatus = &TileXRComm::GetDirectCcuBasicInfoStatus; + BasicInfoFn basicInfo = &TileXRComm::GetDirectCcuBasicInfo; + BasicReportFn basicReport = &TileXRComm::GetDirectCcuBasicInfoReport; + ConfigureVerifiedEndpointRoutesFn configureVerifiedEndpointRoutes = + &TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes; + ConfigureLocalVerifiedEndpointRouteFn configureLocalVerifiedEndpointRoute = + &TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute; + InstallAttemptFn installAttempt = &TileXRComm::PrepareDirectCcuInstallAttempt; + (void)configure; + (void)configureFromAllocation; + (void)prepareFromAllocation; + (void)refresh; + (void)has; + (void)status; + (void)report; + (void)plan; + (void)basicRefresh; + (void)basicHas; + (void)basicStatus; + (void)basicInfo; + (void)basicReport; + (void)configureVerifiedEndpointRoutes; + (void)configureLocalVerifiedEndpointRoute; + (void)installAttempt; + return 0; + } + ''' + ) + + result = self.compile_only(code) + + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_memory_program.py b/tests/ccu/test_tilexr_ccu_memory_program.py new file mode 100644 index 00000000..29868acb --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_memory_program.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MEMORY_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_memory_program.h" +MEMORY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_memory_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + #include + + using namespace TileXR; + + int main() + { + if (TileXRCcuPackMemoryToken(0x12345, 0x89abcdefU, true) != + 0x0011234589abcdefULL) { + std::cerr << "packed token mismatch\n"; + return 1; + } + + TileXRCcuMemoryCopySpec spec; + spec.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; + spec.localGsa = 0x101; + spec.localXn = 0x102; + spec.remoteGsa = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x301; + spec.localAddr = 0x1122334455667788ULL; + spec.localToken = TileXRCcuPackMemoryToken(0x12345, 0x89abcdefU, true); + spec.remoteAddr = 0x8877665544332211ULL; + spec.remoteToken = TileXRCcuPackMemoryToken(0x23456, 0x76543210U, true); + spec.lengthBytes = 0x80; + spec.channelId = 0x12; + spec.completionCke = 0x401; + spec.completionMask = 1; + + std::vector program; + TileXRCcuMemoryProgramReport report; + if (TileXRCcuBuildMemoryCopyProgram(spec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "memory copy program build failed: " << report.message << "\n"; + return 2; + } + if (program.size() != 7 || report.loadInstructionCount != 5 || + report.transferInstructionCount != 1 || report.waitInstructionCount != 1 || + report.totalInstructionCount != 7 || report.message != "ok") { + std::cerr << "unexpected memory program report\n"; + return 3; + } + if (program[0].words[0] != 0x5566778801010002ULL || + program[0].words[1] != 0x0000000011223344ULL || + program[1].words[0] != 0x89abcdef01020003ULL || + program[1].words[1] != 0x0000000100112345ULL || + program[2].words[0] != 0x4433221102010002ULL || + program[2].words[1] != 0x0000000088776655ULL || + program[3].words[0] != 0x7654321002020003ULL || + program[3].words[1] != 0x0000000100123456ULL || + program[4].words[0] != 0x0000008003010003ULL || + program[4].words[1] != 0) { + std::cerr << "unexpected load immediate program\n"; + return 4; + } + if (program[5].words[0] != 0x0101020202011009ULL || + program[5].words[1] != 0x0000001203010102ULL || + program[5].words[2] != 0x0003000000000000ULL || + program[5].words[3] != 0x0000000000010401ULL) { + std::cerr << "unexpected loc->rmt transfer instruction\n"; + return 5; + } + if (program[6].words[0] != 0x0000000000010804ULL || + program[6].words[1] != 0x0000000000010401ULL || + program[6].words[2] != 0 || + program[6].words[3] != 0) { + std::cerr << "unexpected completion wait instruction\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_memory_program_builds_hcomm_style_rmt_to_loc_copy(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_memory_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuMemoryCopySpec spec; + spec.direction = TileXRCcuMemoryCopyDirection::RemoteToLocal; + spec.localGsa = 0x101; + spec.localXn = 0x102; + spec.remoteGsa = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x301; + spec.localAddr = 0x1122334455667788ULL; + spec.localToken = TileXRCcuPackMemoryToken(0x12345, 0x89abcdefU, true); + spec.remoteAddr = 0x8877665544332211ULL; + spec.remoteToken = TileXRCcuPackMemoryToken(0x23456, 0x76543210U, true); + spec.lengthBytes = 0x80; + spec.channelId = 0x12; + spec.completionCke = 0x401; + spec.completionMask = 1; + + std::vector program; + TileXRCcuMemoryProgramReport report; + if (TileXRCcuBuildMemoryCopyProgram(spec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "memory copy program build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 7 || + program[5].words[0] != 0x0201010201011008ULL || + program[5].words[1] != 0x0000001203010202ULL || + program[5].words[2] != 0x0003000000000000ULL || + program[5].words[3] != 0x0000000000010401ULL) { + std::cerr << "unexpected rmt->loc transfer instruction\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_memory_program_rejects_missing_required_fields(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_memory_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + std::vector program(1); + TileXRCcuMemoryProgramReport report; + TileXRCcuMemoryCopySpec empty; + if (TileXRCcuBuildMemoryCopyProgram(empty, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty memory copy spec accepted\n"; + return 1; + } + if (!program.empty() || report.message.find("missing") == std::string::npos) { + std::cerr << "weak empty spec diagnostic: " << report.message << "\n"; + return 2; + } + + TileXRCcuMemoryCopySpec spec; + spec.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; + spec.localGsa = 1; + spec.localXn = 2; + spec.remoteGsa = 3; + spec.remoteXn = 4; + spec.lengthXn = 5; + spec.localAddr = 0x1000; + spec.localToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.remoteAddr = 0x2000; + spec.remoteToken = TileXRCcuPackMemoryToken(3, 4, true); + spec.lengthBytes = 128; + spec.channelId = 6; + spec.completionCke = 7; + spec.completionMask = 1; + + if (TileXRCcuBuildMemoryCopyProgram(spec, nullptr, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null output program accepted\n"; + return 3; + } + + spec.lengthBytes = 0; + if (TileXRCcuBuildMemoryCopyProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("length") == std::string::npos) { + std::cerr << "zero length accepted or weak diagnostic: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_memory_program_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = MEMORY_HEADER.read_text(encoding="utf-8") + source = MEMORY_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_memory_program.h", cmake) + self.assertIn("ccu/tilexr_ccu_memory_program.cpp", cmake) + self.assertIn("enum class TileXRCcuMemoryCopyDirection", header) + self.assertIn("struct TileXRCcuMemoryCopySpec", header) + self.assertIn("struct TileXRCcuMemoryProgramReport", header) + self.assertIn("TileXRCcuPackMemoryToken", header) + self.assertIn("TileXRCcuBuildMemoryCopyProgram", header) + self.assertIn("TileXRCcuEncodeLoadImdToGsa", source) + self.assertIn("TileXRCcuEncodeLoadImdToXn", source) + self.assertIn("TileXRCcuEncodeTransRmtMemToLocMem", source) + self.assertIn("TileXRCcuEncodeTransLocMemToRmtMem", source) + self.assertIn("TileXRCcuEncodeClearCke", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_microcode.py b/tests/ccu/test_tilexr_ccu_microcode.py new file mode 100644 index 00000000..bf06d5e5 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_microcode.py @@ -0,0 +1,572 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +MICROCODE_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.h" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr load; + if (TileXRCcuEncodeLoadSqeArgsToX(1961, 0, &load) != TILEXR_SUCCESS) { + std::cerr << "load encode failed\n"; + return 1; + } + if (load.words[0] != 0x0000000007a90001ULL || load.words[1] != 0 || + load.words[2] != 0 || load.words[3] != 0) { + std::cerr << "unexpected load word\n"; + return 2; + } + + TileXRCcuInstr sync; + TileXRCcuSyncXnSpec spec; + spec.remoteXn = 2361; + spec.localXn = 1961; + spec.channelId = 2; + spec.notifyCke = 364; + spec.notifyMask = 1; + spec.setCkeId = 0; + spec.setCkeMask = 0; + if (TileXRCcuEncodeSyncXn(spec, &sync) != TILEXR_SUCCESS) { + std::cerr << "sync encode failed\n"; + return 3; + } + if (sync.words[0] != 0x000007a90939100dULL || + sync.words[1] != 0x00000001016c0002ULL || + sync.words[2] != 0x0001000000000000ULL || + sync.words[3] != 0x0000000000000000ULL) { + std::cerr << "unexpected sync words\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_microcode_encodes_hcomm_v1_load_immediate_to_xn_words(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr load; + if (TileXRCcuEncodeLoadImdToXn(1961, 0x1122334455667788ULL, 0, &load) != TILEXR_SUCCESS) { + std::cerr << "load immediate encode failed\n"; + return 1; + } + if (load.words[0] != 0x5566778807a90003ULL || + load.words[1] != 0x0000000011223344ULL || + load.words[2] != 0 || + load.words[3] != 0) { + std::cerr << "unexpected load immediate words\n"; + return 2; + } + + if (TileXRCcuEncodeLoadImdToXn(0, 1, 0, &load) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeLoadImdToXn(1961, 1, 0, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid load immediate arguments accepted\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_microcode_encodes_hcomm_v1_load_immediate_to_gsa_words(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr load; + if (TileXRCcuEncodeLoadImdToGsa(510, 0x1122334455667788ULL, &load) != TILEXR_SUCCESS) { + std::cerr << "load immediate to GSA encode failed\n"; + return 1; + } + if (load.words[0] != 0x5566778801fe0002ULL || + load.words[1] != 0x0000000011223344ULL || + load.words[2] != 0 || + load.words[3] != 0) { + std::cerr << "unexpected load immediate to GSA words\n"; + return 2; + } + + if (TileXRCcuEncodeLoadImdToGsa(0, 1, &load) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeLoadImdToGsa(510, 1, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "invalid load immediate to GSA arguments accepted\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_microcode_rejects_invalid_arguments(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + if (TileXRCcuEncodeLoadSqeArgsToX(1961, 13, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null output accepted\n"; + return 1; + } + + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadSqeArgsToX(1961, TILEXR_CCU_SQE_ARGS_LEN, &instr) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "out-of-range sqe arg accepted\n"; + return 2; + } + + TileXRCcuSyncXnSpec missingRemote; + missingRemote.localXn = 1961; + missingRemote.channelId = 2; + missingRemote.notifyCke = 364; + missingRemote.notifyMask = 1; + if (TileXRCcuEncodeSyncXn(missingRemote, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing remote xn accepted\n"; + return 3; + } + + TileXRCcuSyncXnSpec missingNotify; + missingNotify.remoteXn = 2361; + missingNotify.localXn = 1961; + missingNotify.channelId = 2; + if (TileXRCcuEncodeSyncXn(missingNotify, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing notify accepted\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_syncxn_setcke_fields_match_hcomm_v1_layout(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr sync; + TileXRCcuSyncXnSpec spec; + spec.remoteXn = 2361; + spec.localXn = 1961; + spec.channelId = 2; + spec.notifyCke = 364; + spec.notifyMask = 1; + spec.setCkeId = 400; + spec.setCkeMask = 5; + spec.waitCkeId = 401; + spec.waitCkeMask = 6; + if (TileXRCcuEncodeSyncXn(spec, &sync) != TILEXR_SUCCESS) { + std::cerr << "sync encode failed\n"; + return 1; + } + if (sync.words[0] != 0x000007a90939100dULL || + sync.words[1] != 0x00000001016c0002ULL || + sync.words[2] != 0x0001000000000000ULL || + sync.words[3] != 0x0006019100050190ULL) { + std::cerr << "sync local set/wait CKE fields are not hcomm v1 packed layout\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_synccke_fields_match_hcomm_v1_layout(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr sync; + TileXRCcuSyncCkeSpec spec; + spec.remoteCke = 0x330; + spec.localCke = 0x221; + spec.localCkeMask = 1; + spec.channelId = 2; + spec.setCkeId = 0x401; + spec.setCkeMask = 2; + spec.waitCkeId = 0x402; + spec.waitCkeMask = 3; + if (TileXRCcuEncodeSyncCke(spec, &sync) != TILEXR_SUCCESS) { + std::cerr << "synccke encode failed\n"; + return 1; + } + if (sync.words[0] != 0x000102210330100bULL || + sync.words[1] != 0x0000000000000002ULL || + sync.words[2] != 0x0001000000000000ULL || + sync.words[3] != 0x0003040200020401ULL) { + std::cerr << "synccke fields are not hcomm v1 packed layout\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_set_and_clear_cke_microcode_encoders_match_hcomm_v1_layout(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuCkeSpec setSpec; + setSpec.ckeId = 332; + setSpec.mask = 3; + setSpec.waitCkeId = 364; + setSpec.waitMask = 1; + setSpec.clearWait = true; + + TileXRCcuInstr setInstr; + if (TileXRCcuEncodeSetCke(setSpec, &setInstr) != TILEXR_SUCCESS) { + std::cerr << "set cke encode failed\n"; + return 1; + } + if (setInstr.words[0] != 0x0003014c00010802ULL || + setInstr.words[1] != 0x000000000001016cULL || + setInstr.words[2] != 0 || + setInstr.words[3] != 0) { + std::cerr << "unexpected set cke words\n"; + return 2; + } + + TileXRCcuCkeSpec clearSpec; + clearSpec.ckeId = 0; + clearSpec.mask = 0; + clearSpec.waitCkeId = 364; + clearSpec.waitMask = 1; + clearSpec.clearWait = true; + + TileXRCcuInstr clearInstr; + if (TileXRCcuEncodeClearCke(clearSpec, &clearInstr) != TILEXR_SUCCESS) { + std::cerr << "clear cke encode failed\n"; + return 3; + } + if (clearInstr.words[0] != 0x0000000000010804ULL || + clearInstr.words[1] != 0x000000000001016cULL || + clearInstr.words[2] != 0 || + clearInstr.words[3] != 0) { + std::cerr << "unexpected clear cke words\n"; + return 4; + } + + TileXRCcuCkeSpec invalid; + if (TileXRCcuEncodeSetCke(invalid, &setInstr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeClearCke(invalid, &clearInstr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty cke operation accepted\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_memory_transfer_microcode_encoders_match_hcomm_v1_layout(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuMemTransferSpec spec; + spec.localGsa = 0x101; + spec.localXn = 0x102; + spec.remoteGsa = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x301; + spec.channelId = 0x12; + spec.reduceDataType = 0x0a; + spec.reduceOpCode = 0x05; + spec.setCkeId = 0x401; + spec.setCkeMask = 0x2; + spec.waitCkeId = 0x402; + spec.waitCkeMask = 0x3; + spec.clearWait = true; + spec.lengthFromXn = true; + spec.reduceEnabled = true; + + TileXRCcuInstr read; + if (TileXRCcuEncodeTransRmtMemToLocMem(spec, &read) != TILEXR_SUCCESS) { + std::cerr << "trans rmt->loc encode failed\n"; + return 1; + } + if (read.words[0] != 0x0201010201011008ULL || + read.words[1] != 0x5a00001203010202ULL || + read.words[2] != 0x0007000000000000ULL || + read.words[3] != 0x0003040200020401ULL) { + std::cerr << "unexpected trans rmt->loc words\n"; + return 2; + } + + TileXRCcuInstr write; + if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &write) != TILEXR_SUCCESS) { + std::cerr << "trans loc->rmt encode failed\n"; + return 3; + } + if (write.words[0] != 0x0101020202011009ULL || + write.words[1] != 0x5a00001203010102ULL || + write.words[2] != 0x0007000000000000ULL || + write.words[3] != 0x0003040200020401ULL) { + std::cerr << "unexpected trans loc->rmt words\n"; + return 4; + } + + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_memory_transfer_microcode_rejects_missing_required_fields(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_microcode.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuInstr instr; + TileXRCcuMemTransferSpec empty; + if (TileXRCcuEncodeTransRmtMemToLocMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToRmtMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty transfer accepted\n"; + return 1; + } + + TileXRCcuMemTransferSpec spec; + spec.localGsa = 1; + spec.localXn = 2; + spec.remoteGsa = 3; + spec.remoteXn = 4; + spec.lengthXn = 5; + spec.channelId = 6; + spec.setCkeId = 7; + spec.setCkeMask = 8; + + if (TileXRCcuEncodeTransRmtMemToLocMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToRmtMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null output accepted\n"; + return 2; + } + + spec.reduceDataType = 0x10; + if (TileXRCcuEncodeTransRmtMemToLocMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "out-of-range reduce data type accepted\n"; + return 3; + } + + spec.reduceDataType = 0; + spec.reduceOpCode = 0x10; + if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "out-of-range reduce op code accepted\n"; + return 4; + } + + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_microcode_builder_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = MICROCODE_HEADER.read_text(encoding="utf-8") + source = MICROCODE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_microcode.h", cmake) + self.assertIn("ccu/tilexr_ccu_microcode.cpp", cmake) + self.assertIn("struct TileXRCcuInstr", header) + self.assertIn("struct TileXRCcuSyncXnSpec", header) + self.assertIn("TileXRCcuEncodeLoadSqeArgsToX", header) + self.assertIn("TileXRCcuEncodeLoadImdToXn", header) + self.assertIn("TileXRCcuEncodeLoadImdToGsa", header) + self.assertIn("TileXRCcuEncodeSyncXn", header) + self.assertIn("TileXRCcuEncodeSyncCke", header) + self.assertIn("TileXRCcuEncodeSetCke", header) + self.assertIn("TileXRCcuEncodeClearCke", header) + self.assertIn("struct TileXRCcuMemTransferSpec", header) + self.assertIn("TileXRCcuEncodeTransRmtMemToLocMem", header) + self.assertIn("TileXRCcuEncodeTransLocMemToRmtMem", header) + self.assertIn("0x0001U", source) + self.assertIn("0x0002U", source) + self.assertIn("0x0003U", source) + self.assertIn("0x0802U", source) + self.assertIn("0x0804U", source) + self.assertIn("0x1008U", source) + self.assertIn("0x1009U", source) + self.assertIn("0x100bU", source) + self.assertIn("0x100dU", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_producer_plan.py b/tests/ccu/test_tilexr_ccu_producer_plan.py new file mode 100644 index 00000000..fc61ea29 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_producer_plan.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import subprocess +import tempfile +import textwrap +import unittest +import shutil +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +CCU_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.h" +CCU_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.cpp" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime.h" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalGsa = {1, 510, 20}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL, 0x100051152e00ULL, 0x0010017f86b7d29aULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuProducerPlanReport report; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_SUCCESS) { + std::cerr << "valid plan rejected: " << report.message << "\n"; + return 1; + } + if (report.syncResourceCount != 3 || report.taskCount != 2 || report.instructionCount != 170) { + std::cerr << "unexpected report counts\n"; + return 2; + } + + std::vector tasks; + if (TileXRCcuBuildTasks(plan, &tasks, &report) != TILEXR_SUCCESS) { + std::cerr << "task build failed: " << report.message << "\n"; + return 3; + } + if (tasks.size() != 2) { + std::cerr << "unexpected task size\n"; + return 4; + } + if (tasks[0].dieId != 1 || tasks[0].missionId != 6 || tasks[0].key != 0x059b0f03U || + tasks[0].instStartId != 489 || tasks[0].instCnt != 13 || tasks[0].argSize != 13 || + tasks[0].args[0] != 0x100051152e00ULL || tasks[0].args[2] != 0x0010017f86b7d29aULL) { + std::cerr << "unexpected sqe-load mission task\n"; + return 5; + } + if (tasks[1].dieId != 1 || tasks[1].missionId != 6 || tasks[1].key != 0x059b0f03U || + tasks[1].instStartId != 502 || tasks[1].instCnt != 143 || tasks[1].argSize != 13 || + tasks[1].args[0] != 0 || tasks[1].args[12] != 0) { + std::cerr << "unexpected sync mission task\n"; + return 6; + } + if (tasks[0].timeout != TILEXR_CCU_DEFAULT_TASK_TIMEOUT_SEC || + tasks[1].timeout != TILEXR_CCU_DEFAULT_TASK_TIMEOUT_SEC) { + std::cerr << "unexpected task timeout " << tasks[0].timeout + << " " << tasks[1].timeout << "\n"; + return 12; + } + TileXRCcuProgram program; + if (TileXRCcuBuildMicrocode(plan, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << report.message << "\n"; + return 7; + } + if (program.sqeLoad.size() != 13 || program.sync.size() != 11) { + std::cerr << "unexpected microcode sizes\n"; + return 8; + } + if (program.sqeLoad[0].words[0] != 0x0000000007a90001ULL || + program.sqeLoad[12].words[0] != 0x0000000c07b50001ULL) { + std::cerr << "unexpected sqe load microcode\n"; + return 9; + } + if (program.sync[0].words[0] != 0x0000000007a90001ULL || + program.sync[1].words[0] != 0x0000000107aa0001ULL || + program.sync[2].words[0] != 0x0000000007b60003ULL || + program.sync[3].words[0] != 0x0000000001fe0002ULL || + program.sync[4].words[0] != 0x0001016c00000802ULL || + program.sync[4].words[1] != 0) { + std::cerr << "unexpected hcomm-style task1 prelude\n"; + return 10; + } + if (program.sync[5].words[0] != 0x000007a90939100dULL || + program.sync[5].words[1] != 0x00000001016c0002ULL || + program.sync[5].words[2] != 0x0001000000000000ULL) { + std::cerr << "unexpected first sync microcode after prelude\n"; + return 10; + } + if (program.sync[8].words[0] != 0x0000000000010802ULL || + program.sync[8].words[1] != 0x000000000001016cULL) { + std::cerr << "unexpected sync wait microcode\n"; + return 11; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_plan_rejects_missing_producer_state(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_producer_plan.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + + TileXRCcuProducerPlanReport report; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing mission/key was accepted\n"; + return 1; + } + if (report.message.find("mission") == std::string::npos) { + std::cerr << "missing mission/key diagnostic was weak: " << report.message << "\n"; + return 2; + } + + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.syncResources[0].bindingCount = 0; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing channel binding was accepted\n"; + return 3; + } + if (report.message.find("binding") == std::string::npos) { + std::cerr << "missing binding diagnostic was weak: " << report.message << "\n"; + return 4; + } + + plan.syncResources[0].bindingCount = 1; + plan.syncResources[0].channelId = 0; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing channel id was accepted\n"; + return 5; + } + if (report.message.find("channel id") == std::string::npos) { + std::cerr << "missing channel id diagnostic was weak: " << report.message << "\n"; + return 6; + } + + plan.syncResources[0].channelId = 2; + plan.syncResources.push_back({1, 1962, 2362, 365, 2, 1}); + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "duplicate channel id was accepted\n"; + return 7; + } + if (report.message.find("duplicate channel id") == std::string::npos) { + std::cerr << "duplicate channel id diagnostic was weak: " << report.message << "\n"; + return 8; + } + plan.syncResources.pop_back(); + + plan.syncResources[0].localXn = 2040; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "out-of-range local xn was accepted\n"; + return 9; + } + if (report.message.find("local XN") == std::string::npos) { + std::cerr << "local xn diagnostic was weak: " << report.message << "\n"; + return 10; + } + + plan.syncResources[0].localXn = 1961; + plan.taskWindows[0].instStartId = 646; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "out-of-window task was accepted\n"; + return 11; + } + if (report.message.find("instruction") == std::string::npos) { + std::cerr << "instruction diagnostic was weak: " << report.message << "\n"; + return 12; + } + + plan.taskWindows[0].instStartId = 489; + plan.kernelLocalCke = {1, 0, 0}; + if (TileXRCcuValidateProducerPlan(plan, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing kernel-local cke repository was accepted\n"; + return 13; + } + if (report.message.find("CKE") == std::string::npos) { + std::cerr << "kernel-local cke diagnostic was weak: " << report.message << "\n"; + return 14; + } + + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_producer_plan_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = CCU_HEADER.read_text(encoding="utf-8") + source = CCU_SOURCE.read_text(encoding="utf-8") + runtime_header = RUNTIME_HEADER.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_producer_plan.h", cmake) + self.assertIn("ccu/tilexr_ccu_producer_plan.cpp", cmake) + self.assertIn('#include "ccu/tilexr_ccu_runtime.h"', header) + self.assertIn("TileXRCcuProducerPlan", header) + self.assertIn("TileXRCcuValidateProducerPlan", header) + self.assertIn("TileXRCcuBuildTasks", header) + self.assertIn("TileXRCcuBuildMicrocode", header) + self.assertIn("TileXRCcuProgram", header) + self.assertIn("localWaitCke", header) + self.assertIn("localWaitMask", header) + self.assertIn("remoteNotifyMask", header) + self.assertIn("tilexr_ccu_barrier_program.h", header) + self.assertIn("TileXRCcuBuildBarrierProgram", source) + self.assertIn("spec.localWaitCke", source) + self.assertIn("spec.remoteNotifyCke", source) + self.assertIn("TileXRCcuTask", runtime_header) + self.assertIn("kernelLocalCke", source) + self.assertIn("std::set channelIds", source) + self.assertIn("missing channel id for sync resource", source) + self.assertIn("duplicate channel id for sync resource", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_microcode_can_use_distinct_remote_notify_and_local_wait_cke(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_producer_plan.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 4}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 3; + resource.localWaitCke = 332; + resource.localWaitMask = 1; + resource.remoteNotifyMask = 1; + plan.syncResources.push_back(resource); + + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuProgram program; + TileXRCcuProducerPlanReport report; + if (TileXRCcuBuildMicrocode(plan, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << report.message << "\n"; + return 1; + } + if (program.sync.size() != 7) { + std::cerr << "unexpected sync size\n"; + return 2; + } + if (program.sync[5].words[1] != 0x00000001016c0002ULL) { + std::cerr << "post did not use remote notify CKE\n"; + return 3; + } + if (program.sync[6].words[0] != 0x0000000000010802ULL || + program.sync[6].words[1] != 0x000000000001014cULL) { + std::cerr << "wait/clear did not use local wait CKE\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_microcode_can_emit_sync_cke_barrier_mode_with_source_cke(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_producer_plan.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.barrierMode = TileXRCcuBarrierMode::SyncCke; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 0x220, 2}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 0x330; + resource.channelId = 2; + resource.bindingCount = 1; + resource.localWaitCke = 0x220; + resource.localWaitMask = 1; + resource.remoteNotifyMask = 1; + resource.sourceCke = 0x221; + resource.sourceCkeMask = 0xffff; + plan.syncResources.push_back(resource); + + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuProgram program; + TileXRCcuProducerPlanReport report; + if (TileXRCcuBuildMicrocode(plan, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "sync_cke microcode build failed: " << report.message << "\n"; + return 1; + } + if (program.sync.size() != 3) { + std::cerr << "unexpected sync_cke sync size\n"; + return 2; + } + if (program.sync[0].words[0] != 0xffff022100010802ULL || + program.sync[1].words[0] != 0x000102210330100bULL || + program.sync[1].words[1] != 0x0000000000000002ULL || + program.sync[2].words[0] != 0x0000000000010804ULL || + program.sync[2].words[1] != 0x0000000000010220ULL) { + std::cerr << "sync_cke barrier microcode mismatch\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_provider.py b/tests/ccu/test_tilexr_ccu_provider.py new file mode 100644 index 00000000..797bb62f --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_provider.py @@ -0,0 +1,1618 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROVIDER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_provider.h" +PROVIDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_provider.cpp" +PACKAGE_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.h" +PACKAGE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.cpp" +REPOSITORY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_repository.cpp" +DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" +PRODUCER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.cpp" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 1}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 1}; + plan.barrierMode = TileXRCcuBarrierMode::LocalCkePostOnly; + + TileXRCcuSyncResource resource; + resource.dieId = 1; + resource.localXn = 1961; + resource.remoteXn = 2361; + resource.notifyCke = 364; + resource.channelId = 2; + resource.bindingCount = 1; + resource.localWaitCke = 332; + plan.syncResources.push_back(resource); + + plan.taskWindows.push_back({1, 489, 1, 13, {}}); + return plan; + } + + TileXRCcuEvidenceSource Source( + const TileXRCcuLaunchPackage& package, + const char* label, + bool endpointRouteVerified) + { + TileXRCcuEvidenceSource source; + source.kind = TileXRCcuEvidenceKind::PublicVerified; + source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + source.packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + source.deviceId = package.installScope.deviceId; + source.rank = package.installScope.rank; + source.provider = package.installScope.provider; + source.installAttemptReceiptId = 0xabc001ULL; + source.endpointRouteVerified = endpointRouteVerified; + source.source = std::string("unit-test-public-provider:") + label; + source.detail = std::string("audited ") + label + " evidence"; + return source; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed: " << packageReport.message << "\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 2; + } + + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.notifyCkeInstalled = true; + evidence.missionSource = Source(package, "mission", false); + evidence.repositorySource = Source(package, "repository", false); + evidence.localXnSource = Source(package, "local-xn", false); + evidence.notifyCkeSource = Source(package, "notify-cke", false); + + std::vector submitTasks; + TileXRCcuProviderReport report; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "local CKE post-only submit was rejected: " << report.message << "\n"; + return 3; + } + if (!report.submitReady || submitTasks.size() != package.tasks.size()) { + std::cerr << "local CKE submit readiness mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_gate_requires_complete_hardware_install_evidence(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + + TileXRCcuProviderReport report; + if (TileXRCcuValidateHardwareInstall(package, {}, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "empty install evidence was accepted\n"; + return 2; + } + if (report.message.find("mission") == std::string::npos) { + std::cerr << "empty install diagnostic was weak: " << report.message << "\n"; + return 3; + } + + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing channel binding was accepted\n"; + return 4; + } + if (report.message.find("channel") == std::string::npos) { + std::cerr << "channel diagnostic was weak: " << report.message << "\n"; + return 5; + } + + evidence.channelBindingsInstalled = true; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != TILEXR_SUCCESS) { + std::cerr << "complete install evidence was rejected: " << report.message << "\n"; + return 6; + } + if (report.taskCount != 2 || report.installedInstructionCount != 24 || + report.repositoryCount != 170 || report.submitReady || + report.message.find("validate-compatible") == std::string::npos) { + std::cerr << "unexpected provider report\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_gate_marks_package_submit_ready_only_after_install_evidence(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(plan, &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 8; + } + + std::vector submitTasks; + TileXRCcuProviderReport report; + if (TileXRCcuPrepareSubmitTasks(package, {}, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "uninstalled package became submit-ready\n"; + return 2; + } + if (!submitTasks.empty()) { + std::cerr << "failed prepare left submit tasks\n"; + return 3; + } + + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.missionSource.source = "unit-test-public-provider:mission"; + evidence.missionSource.detail = "audited mission/key evidence"; + evidence.missionSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.repositorySource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.repositorySource.source = "unit-test-public-provider:repository"; + evidence.repositorySource.detail = "audited repository evidence"; + evidence.repositorySource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.localXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.localXnSource.source = "unit-test-public-provider:local-xn"; + evidence.localXnSource.detail = "audited local XN evidence"; + evidence.localXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.remoteXnSource.source = "ValidateRemoteXnExchangeBindingProof"; + evidence.remoteXnSource.detail = + "remote XN peer exchange proof matches syncXn operands and verified endpoint route channel contexts"; + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.remoteXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.notifyCkeSource.source = "unit-test-public-provider:notify-cke"; + evidence.notifyCkeSource.detail = "audited notify CKE evidence"; + evidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.channelBindingSource.source = "unit-test-public-provider:channel-binding"; + evidence.channelBindingSource.detail = + "channel binding contexts installed via SET_PFE, SET_JETTY_CTX, SET_CHANNEL with verified endpoint routes"; + evidence.channelBindingSource.endpointRouteVerified = true; + evidence.channelBindingSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.missionSource.packageFingerprint = packageFingerprint; + evidence.repositorySource.packageFingerprint = packageFingerprint; + evidence.localXnSource.packageFingerprint = packageFingerprint; + evidence.remoteXnSource.packageFingerprint = packageFingerprint; + evidence.notifyCkeSource.packageFingerprint = packageFingerprint; + evidence.channelBindingSource.packageFingerprint = packageFingerprint; + evidence.missionSource.deviceId = 3; + evidence.repositorySource.deviceId = 3; + evidence.localXnSource.deviceId = 3; + evidence.remoteXnSource.deviceId = 3; + evidence.notifyCkeSource.deviceId = 3; + evidence.channelBindingSource.deviceId = 3; + evidence.missionSource.rank = 1; + evidence.repositorySource.rank = 1; + evidence.localXnSource.rank = 1; + evidence.remoteXnSource.rank = 1; + evidence.notifyCkeSource.rank = 1; + evidence.channelBindingSource.rank = 1; + evidence.missionSource.provider = "unit-test-public-install-provider"; + evidence.repositorySource.provider = "unit-test-public-install-provider"; + evidence.localXnSource.provider = "unit-test-public-install-provider"; + evidence.remoteXnSource.provider = "unit-test-public-install-provider"; + evidence.notifyCkeSource.provider = "unit-test-public-install-provider"; + evidence.channelBindingSource.provider = "unit-test-public-install-provider"; + evidence.missionSource.installAttemptReceiptId = 0xabc001ULL; + evidence.repositorySource.installAttemptReceiptId = 0xabc001ULL; + evidence.localXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.remoteXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.notifyCkeSource.installAttemptReceiptId = 0xabc001ULL; + evidence.channelBindingSource.installAttemptReceiptId = 0xabc001ULL; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "complete install evidence was rejected: " << report.message << "\n"; + return 4; + } + if (submitTasks.size() != 2 || submitTasks[0].instStartId != 489 || + submitTasks[0].instCnt != 13 || submitTasks[0].argSize != 13 || + submitTasks[1].instStartId != 502 || submitTasks[1].instCnt != 143 || + submitTasks[1].argSize != 13) { + std::cerr << "unexpected submit tasks\n"; + return 5; + } + if (!report.submitReady || report.message != "ok") { + std::cerr << "unexpected submit-ready report\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_rejects_legacy_bool_only_evidence_for_submit(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(plan, &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + + TileXRCcuProviderReport report; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != TILEXR_SUCCESS) { + std::cerr << "legacy bool install evidence should remain validate-compatible: " + << report.message << "\n"; + return 2; + } + if (report.submitReady || report.legacyEvidenceCount != 6) { + std::cerr << "legacy validate report incorrectly claimed submit-ready\n"; + return 3; + } + + std::vector submitTasks; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "legacy bool-only evidence became submit-ready\n"; + return 4; + } + if (!submitTasks.empty()) { + std::cerr << "failed legacy prepare left submit tasks\n"; + return 5; + } + if (report.submitReady || report.legacyEvidenceCount != 6) { + std::cerr << "unexpected failed prepare report\n"; + return 6; + } + if (report.message.find("submit requires public verified evidence") == std::string::npos) { + std::cerr << "legacy submit diagnostic was weak: " << report.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_requires_auditable_public_verified_evidence_sources(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuHardwareInstallEvidence VerifiedEvidence(const TileXRCcuLaunchPackage& package) + { + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.repositorySource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.localXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicVerified; + return evidence; + } + + void FillAuditSources(TileXRCcuHardwareInstallEvidence& evidence, const TileXRCcuLaunchPackage& package) + { + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + evidence.missionSource.source = "unit-test-public-provider:mission"; + evidence.missionSource.detail = "audited mission/key evidence"; + evidence.missionSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.missionSource.packageFingerprint = packageFingerprint; + evidence.repositorySource.source = "unit-test-public-provider:repository"; + evidence.repositorySource.detail = "audited repository evidence"; + evidence.repositorySource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.repositorySource.packageFingerprint = packageFingerprint; + evidence.localXnSource.source = "unit-test-public-provider:local-xn"; + evidence.localXnSource.detail = "audited local XN evidence"; + evidence.localXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.localXnSource.packageFingerprint = packageFingerprint; + evidence.remoteXnSource.source = "unit-test-public-provider:remote-xn"; + evidence.remoteXnSource.detail = "audited remote XN evidence"; + evidence.remoteXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.remoteXnSource.packageFingerprint = packageFingerprint; + evidence.notifyCkeSource.source = "unit-test-public-provider:notify-cke"; + evidence.notifyCkeSource.detail = "audited notify CKE evidence"; + evidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.notifyCkeSource.packageFingerprint = packageFingerprint; + evidence.channelBindingSource.source = "unit-test-public-provider:channel-binding"; + evidence.channelBindingSource.detail = "audited channel binding evidence"; + evidence.channelBindingSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingSource.endpointRouteVerified = true; + evidence.channelBindingSource.packageFingerprint = packageFingerprint; + evidence.missionSource.deviceId = package.installScope.deviceId; + evidence.repositorySource.deviceId = package.installScope.deviceId; + evidence.localXnSource.deviceId = package.installScope.deviceId; + evidence.remoteXnSource.deviceId = package.installScope.deviceId; + evidence.notifyCkeSource.deviceId = package.installScope.deviceId; + evidence.channelBindingSource.deviceId = package.installScope.deviceId; + evidence.missionSource.rank = package.installScope.rank; + evidence.repositorySource.rank = package.installScope.rank; + evidence.localXnSource.rank = package.installScope.rank; + evidence.remoteXnSource.rank = package.installScope.rank; + evidence.notifyCkeSource.rank = package.installScope.rank; + evidence.channelBindingSource.rank = package.installScope.rank; + evidence.missionSource.provider = package.installScope.provider; + evidence.repositorySource.provider = package.installScope.provider; + evidence.localXnSource.provider = package.installScope.provider; + evidence.remoteXnSource.provider = package.installScope.provider; + evidence.notifyCkeSource.provider = package.installScope.provider; + evidence.channelBindingSource.provider = package.installScope.provider; + evidence.missionSource.installAttemptReceiptId = 0xabc001ULL; + evidence.repositorySource.installAttemptReceiptId = 0xabc001ULL; + evidence.localXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.remoteXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.notifyCkeSource.installAttemptReceiptId = 0xabc001ULL; + evidence.channelBindingSource.installAttemptReceiptId = 0xabc001ULL; + } + + void MarkEndpointRoutesVerified(TileXRCcuHardwareInstallEvidence& evidence) + { + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.channelBindingSource.endpointRouteVerified = true; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 10; + } + + TileXRCcuHardwareInstallEvidence evidence = VerifiedEvidence(package); + TileXRCcuProviderReport report; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != TILEXR_SUCCESS) { + std::cerr << "validate compatibility rejected verified kind without audit strings: " + << report.message << "\n"; + return 2; + } + + std::vector submitTasks; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "public verified evidence without source/detail became submit-ready\n"; + return 3; + } + if (!submitTasks.empty()) { + std::cerr << "failed unaudited prepare left submit tasks\n"; + return 4; + } + if (report.message.find("public verified evidence source/detail required") == std::string::npos) { + std::cerr << "unaudited diagnostic was weak: " << report.message << "\n"; + return 5; + } + + FillAuditSources(evidence, package); + evidence.missionSource.surface = TileXRCcuEvidenceSurface::Unspecified; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "public verified evidence without provider surface became submit-ready\n"; + return 6; + } + if (report.message.find("public install provider evidence") == std::string::npos) { + std::cerr << "unclassified surface diagnostic was weak: " << report.message << "\n"; + return 7; + } + + evidence.missionSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.endpointRouteVerified = false; + evidence.channelBindingSource.endpointRouteVerified = false; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "generic endpoint evidence became submit-ready\n"; + return 8; + } + if (report.message.find("verified endpoint route") == std::string::npos) { + std::cerr << "endpoint provenance diagnostic was weak: " << report.message << "\n"; + return 9; + } + + MarkEndpointRoutesVerified(evidence); + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "audited public verified evidence was rejected: " << report.message << "\n"; + return 10; + } + if (submitTasks.size() != 2 || submitTasks[0].instStartId != 489 || + submitTasks[0].instCnt != 13 || submitTasks[1].instStartId != 502 || + submitTasks[1].instCnt != 143 || !report.submitReady || + report.publicVerifiedEvidenceCount != 6) { + std::cerr << "unexpected audited submit report\n"; + return 11; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_requires_install_evidence_to_match_launch_package_fingerprint(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan(uint32_t key, uint64_t firstArg) + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, key, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {firstArg}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuEvidenceSource Source(const char* label, const TileXRCcuLaunchPackage& package) + { + TileXRCcuEvidenceSource source; + source.kind = TileXRCcuEvidenceKind::PublicVerified; + source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + source.source = label; + const uint64_t fingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + source.detail = std::string("installed package fingerprint=") + std::to_string(fingerprint); + const std::string labelText(label); + if (labelText.find("remote-xn") != std::string::npos || + labelText.find("channel-binding") != std::string::npos) { + source.endpointRouteVerified = true; + } + source.packageFingerprint = fingerprint; + source.deviceId = package.installScope.deviceId; + source.rank = package.installScope.rank; + source.provider = package.installScope.provider; + source.installAttemptReceiptId = 0xabc001ULL; + return source; + } + + TileXRCcuHardwareInstallEvidence EvidenceFor(const TileXRCcuLaunchPackage& package) + { + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource = Source("unit-test-public-provider:mission", package); + evidence.repositorySource = Source("unit-test-public-provider:repository", package); + evidence.localXnSource = Source("unit-test-public-provider:local-xn", package); + evidence.remoteXnSource = Source("unit-test-public-provider:remote-xn", package); + evidence.notifyCkeSource = Source("unit-test-public-provider:notify-cke", package); + evidence.channelBindingSource = Source("unit-test-public-provider:channel-binding", package); + return evidence; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(0x059b0f03U, 0x100051152e00ULL), &package, &packageReport) != + TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 11; + } + const uint64_t fingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + if (fingerprint == 0) { + std::cerr << "package fingerprint was zero\n"; + return 2; + } + + TileXRCcuHardwareInstallEvidence evidence = EvidenceFor(package); + std::vector submitTasks; + TileXRCcuProviderReport report; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "matching evidence rejected: " << report.message << "\n"; + return 3; + } + + TileXRCcuLaunchPackage changedKeyPackage; + if (TileXRCcuBuildLaunchPackage( + MakePlan(0xe0ac084cU, 0x100051152e00ULL), &changedKeyPackage, &packageReport) != + TILEXR_SUCCESS) { + std::cerr << "changed-key package build failed\n"; + return 4; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &changedKeyPackage, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "changed-key scope bind failed\n"; + return 12; + } + if (TileXRCcuComputeLaunchPackageFingerprint(changedKeyPackage) == fingerprint) { + std::cerr << "changed key did not affect package fingerprint\n"; + return 5; + } + if (TileXRCcuPrepareSubmitTasks(changedKeyPackage, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "evidence for old key was accepted for changed package\n"; + return 6; + } + if (report.message.find("fingerprint") == std::string::npos) { + std::cerr << "fingerprint diagnostic was weak: " << report.message << "\n"; + return 7; + } + + TileXRCcuLaunchPackage changedArgsPackage; + if (TileXRCcuBuildLaunchPackage( + MakePlan(0x059b0f03U, 0x200051152e00ULL), &changedArgsPackage, &packageReport) != + TILEXR_SUCCESS) { + std::cerr << "changed-args package build failed\n"; + return 8; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &changedArgsPackage, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "changed-args scope bind failed\n"; + return 13; + } + if (TileXRCcuComputeLaunchPackageFingerprint(changedArgsPackage) == fingerprint) { + std::cerr << "changed task args did not affect package fingerprint\n"; + return 9; + } + if (TileXRCcuPrepareSubmitTasks(changedArgsPackage, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "evidence for old task args was accepted for changed package\n"; + return 10; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_requires_install_evidence_to_match_launch_scope(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuEvidenceSource Source( + const char* label, + const TileXRCcuLaunchPackage& package, + uint32_t deviceId, + uint32_t rank, + const char* provider) + { + TileXRCcuEvidenceSource source; + source.kind = TileXRCcuEvidenceKind::PublicVerified; + source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + source.source = label; + source.detail = "installed for explicit launch scope"; + const std::string labelText(label); + if (labelText.find("remote-xn") != std::string::npos || + labelText.find("channel-binding") != std::string::npos) { + source.endpointRouteVerified = true; + } + source.packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + source.deviceId = deviceId; + source.rank = rank; + source.provider = provider; + source.installAttemptReceiptId = 0xabc001ULL; + return source; + } + + TileXRCcuHardwareInstallEvidence EvidenceFor( + const TileXRCcuLaunchPackage& package, + uint32_t deviceId, + uint32_t rank, + const char* provider) + { + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource = Source("unit-test-public-provider:mission", package, deviceId, rank, provider); + evidence.repositorySource = + Source("unit-test-public-provider:repository", package, deviceId, rank, provider); + evidence.localXnSource = + Source("unit-test-public-provider:local-xn", package, deviceId, rank, provider); + evidence.remoteXnSource = + Source("unit-test-public-provider:remote-xn", package, deviceId, rank, provider); + evidence.notifyCkeSource = + Source("unit-test-public-provider:notify-cke", package, deviceId, rank, provider); + evidence.channelBindingSource = + Source("unit-test-public-provider:channel-binding", package, deviceId, rank, provider); + return evidence; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "failed to bind launch install scope\n"; + return 2; + } + + TileXRCcuProviderReport report; + std::vector submitTasks; + + TileXRCcuHardwareInstallEvidence wrongDevice = + EvidenceFor(package, 4, 1, "unit-test-public-install-provider"); + if (TileXRCcuPrepareSubmitTasks(package, wrongDevice, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "evidence from another device became submit-ready\n"; + return 3; + } + if (!submitTasks.empty() || report.submitReady || + report.message.find("device") == std::string::npos) { + std::cerr << "device-scope diagnostic was weak: " << report.message << "\n"; + return 4; + } + + TileXRCcuHardwareInstallEvidence wrongRank = + EvidenceFor(package, 3, 0, "unit-test-public-install-provider"); + if (TileXRCcuPrepareSubmitTasks(package, wrongRank, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "evidence from another rank became submit-ready\n"; + return 5; + } + if (!submitTasks.empty() || report.submitReady || + report.message.find("rank") == std::string::npos) { + std::cerr << "rank-scope diagnostic was weak: " << report.message << "\n"; + return 6; + } + + TileXRCcuHardwareInstallEvidence wrongProvider = + EvidenceFor(package, 3, 1, "other-public-install-provider"); + if (TileXRCcuPrepareSubmitTasks(package, wrongProvider, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "evidence from another provider became submit-ready\n"; + return 7; + } + if (!submitTasks.empty() || report.submitReady || + report.message.find("provider") == std::string::npos) { + std::cerr << "provider-scope diagnostic was weak: " << report.message << "\n"; + return 8; + } + + TileXRCcuHardwareInstallEvidence matching = + EvidenceFor(package, 3, 1, "unit-test-public-install-provider"); + if (TileXRCcuPrepareSubmitTasks(package, matching, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "matching scope evidence was rejected: " << report.message << "\n"; + return 9; + } + if (submitTasks.size() != package.tasks.size() || !report.submitReady) { + std::cerr << "unexpected matching scope submit state\n"; + return 10; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_requires_all_public_verified_evidence_from_same_install_receipt(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuEvidenceSource Source( + const char* label, + const TileXRCcuLaunchPackage& package, + uint64_t receipt) + { + TileXRCcuEvidenceSource source; + source.kind = TileXRCcuEvidenceKind::PublicVerified; + source.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + source.source = label; + source.detail = "installed for a single public install attempt"; + const std::string labelText(label); + if (labelText.find("remote-xn") != std::string::npos || + labelText.find("channel-binding") != std::string::npos) { + source.endpointRouteVerified = true; + } + source.packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + source.deviceId = package.installScope.deviceId; + source.rank = package.installScope.rank; + source.provider = package.installScope.provider; + source.installAttemptReceiptId = receipt; + return source; + } + + TileXRCcuHardwareInstallEvidence EvidenceFor( + const TileXRCcuLaunchPackage& package, + uint64_t receipt) + { + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource = Source("unit-test-public-provider:mission", package, receipt); + evidence.repositorySource = Source("unit-test-public-provider:repository", package, receipt); + evidence.localXnSource = Source("unit-test-public-provider:local-xn", package, receipt); + evidence.remoteXnSource = Source("unit-test-public-provider:remote-xn", package, receipt); + evidence.notifyCkeSource = Source("unit-test-public-provider:notify-cke", package, receipt); + evidence.channelBindingSource = Source("unit-test-public-provider:channel-binding", package, receipt); + return evidence; + } + + int ExpectReceiptFailure( + const TileXRCcuLaunchPackage& package, + const TileXRCcuHardwareInstallEvidence& evidence, + const char* expectedDiagnostic) + { + std::vector submitTasks; + TileXRCcuProviderReport report; + if (TileXRCcuPrepareSubmitTasks(package, evidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "receipt mismatch became submit-ready\n"; + return 1; + } + if (!submitTasks.empty() || report.submitReady || + report.message.find(expectedDiagnostic) == std::string::npos) { + std::cerr << "weak receipt diagnostic: " << report.message << "\n"; + return 2; + } + return 0; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "failed to bind launch install scope\n"; + return 2; + } + + TileXRCcuHardwareInstallEvidence missingReceipt = EvidenceFor(package, 0); + if (ExpectReceiptFailure(package, missingReceipt, "receipt") != 0) { + return 3; + } + + TileXRCcuHardwareInstallEvidence mixedReceipt = EvidenceFor(package, 0xabc001ULL); + mixedReceipt.notifyCkeSource.installAttemptReceiptId = 0xabc002ULL; + if (ExpectReceiptFailure(package, mixedReceipt, "receipt") != 0) { + return 4; + } + + TileXRCcuHardwareInstallEvidence matchingReceipt = EvidenceFor(package, 0xabc001ULL); + std::vector submitTasks; + TileXRCcuProviderReport report; + if (TileXRCcuPrepareSubmitTasks(package, matchingReceipt, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "matching receipt evidence was rejected: " << report.message << "\n"; + return 5; + } + if (submitTasks.size() != package.tasks.size() || !report.submitReady) { + std::cerr << "unexpected matching receipt submit state\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_prepare_rejects_public_verified_evidence_from_lower_layer_or_private_sources(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuHardwareInstallEvidence AuditedEvidence(const TileXRCcuLaunchPackage& package) + { + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.missionSource.source = "unit-test-public-provider:mission"; + evidence.missionSource.detail = "audited mission/key evidence"; + evidence.missionSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.missionSource.packageFingerprint = packageFingerprint; + evidence.repositorySource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.repositorySource.source = "unit-test-public-provider:repository"; + evidence.repositorySource.detail = "audited repository evidence"; + evidence.repositorySource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.repositorySource.packageFingerprint = packageFingerprint; + evidence.localXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.localXnSource.source = "unit-test-public-provider:local-xn"; + evidence.localXnSource.detail = "audited local XN evidence"; + evidence.localXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.localXnSource.packageFingerprint = packageFingerprint; + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.remoteXnSource.source = "unit-test-public-provider:remote-xn"; + evidence.remoteXnSource.detail = "audited remote XN evidence"; + evidence.remoteXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.remoteXnSource.packageFingerprint = packageFingerprint; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.notifyCkeSource.source = "unit-test-public-provider:notify-cke"; + evidence.notifyCkeSource.detail = "audited notify CKE evidence"; + evidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.notifyCkeSource.packageFingerprint = packageFingerprint; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.channelBindingSource.source = "unit-test-public-provider:channel-binding"; + evidence.channelBindingSource.detail = "audited channel binding evidence"; + evidence.channelBindingSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingSource.endpointRouteVerified = true; + evidence.channelBindingSource.packageFingerprint = packageFingerprint; + evidence.missionSource.deviceId = package.installScope.deviceId; + evidence.repositorySource.deviceId = package.installScope.deviceId; + evidence.localXnSource.deviceId = package.installScope.deviceId; + evidence.remoteXnSource.deviceId = package.installScope.deviceId; + evidence.notifyCkeSource.deviceId = package.installScope.deviceId; + evidence.channelBindingSource.deviceId = package.installScope.deviceId; + evidence.missionSource.rank = package.installScope.rank; + evidence.repositorySource.rank = package.installScope.rank; + evidence.localXnSource.rank = package.installScope.rank; + evidence.remoteXnSource.rank = package.installScope.rank; + evidence.notifyCkeSource.rank = package.installScope.rank; + evidence.channelBindingSource.rank = package.installScope.rank; + evidence.missionSource.provider = package.installScope.provider; + evidence.repositorySource.provider = package.installScope.provider; + evidence.localXnSource.provider = package.installScope.provider; + evidence.remoteXnSource.provider = package.installScope.provider; + evidence.notifyCkeSource.provider = package.installScope.provider; + evidence.channelBindingSource.provider = package.installScope.provider; + evidence.missionSource.installAttemptReceiptId = 0xabc001ULL; + evidence.repositorySource.installAttemptReceiptId = 0xabc001ULL; + evidence.localXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.remoteXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.notifyCkeSource.installAttemptReceiptId = 0xabc001ULL; + evidence.channelBindingSource.installAttemptReceiptId = 0xabc001ULL; + return evidence; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 8; + } + + TileXRCcuProviderReport report; + std::vector submitTasks; + + TileXRCcuHardwareInstallEvidence mappingEvidence = AuditedEvidence(package); + mappingEvidence.localXnSource.source = "unit-test-public-mapper:local-xn"; + mappingEvidence.localXnSource.detail = "maps existing XN resources but does not install them"; + mappingEvidence.localXnSource.surface = TileXRCcuEvidenceSurface::LowerLayerResourceHelper; + if (TileXRCcuValidateHardwareInstall(package, mappingEvidence, &report) != TILEXR_SUCCESS) { + std::cerr << "validate compatibility rejected mapping evidence: " << report.message << "\n"; + return 2; + } + if (TileXRCcuPrepareSubmitTasks(package, mappingEvidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "lower-layer mapping helper became submit-ready\n"; + return 3; + } + if (report.message.find("public install provider evidence") == std::string::npos || + report.message.find("lower-layer resource helper") == std::string::npos) { + std::cerr << "mapping diagnostic was weak: " << report.message << "\n"; + return 4; + } + + TileXRCcuHardwareInstallEvidence notifyEvidence = AuditedEvidence(package); + notifyEvidence.notifyCkeSource.source = "unit-test-public-notify-reader:notify-cke"; + notifyEvidence.notifyCkeSource.detail = "reads an existing notify address but does not install CKE"; + notifyEvidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::LowerLayerResourceHelper; + if (TileXRCcuPrepareSubmitTasks(package, notifyEvidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "lower-layer notify helper became submit-ready\n"; + return 5; + } + + TileXRCcuHardwareInstallEvidence privateEvidence = AuditedEvidence(package); + privateEvidence.repositorySource.source = "unit-test-private-observation:repository"; + privateEvidence.repositorySource.detail = "private repository install observation"; + privateEvidence.repositorySource.surface = TileXRCcuEvidenceSurface::PrivateProducerObservation; + if (TileXRCcuPrepareSubmitTasks(package, privateEvidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "private repository installer became submit-ready\n"; + return 6; + } + + TileXRCcuHardwareInstallEvidence audited = AuditedEvidence(package); + if (TileXRCcuPrepareSubmitTasks(package, audited, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "neutral audited evidence was rejected: " << report.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_gate_rejects_private_and_unvalidated_candidate_evidence(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + TileXRCcuHardwareInstallEvidence VerifiedEvidence(const TileXRCcuLaunchPackage& package) + { + const uint64_t packageFingerprint = TileXRCcuComputeLaunchPackageFingerprint(package); + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = true; + evidence.missionSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.missionSource.source = "unit-test-public-provider:mission"; + evidence.missionSource.detail = "audited mission/key evidence"; + evidence.missionSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.missionSource.packageFingerprint = packageFingerprint; + evidence.repositorySource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.repositorySource.source = "unit-test-public-provider:repository"; + evidence.repositorySource.detail = "audited repository evidence"; + evidence.repositorySource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.repositorySource.packageFingerprint = packageFingerprint; + evidence.localXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.localXnSource.source = "unit-test-public-provider:local-xn"; + evidence.localXnSource.detail = "audited local XN evidence"; + evidence.localXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.localXnSource.packageFingerprint = packageFingerprint; + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.remoteXnSource.source = "unit-test-public-provider:remote-xn"; + evidence.remoteXnSource.detail = "audited remote XN evidence"; + evidence.remoteXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.remoteXnSource.packageFingerprint = packageFingerprint; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.notifyCkeSource.source = "unit-test-public-provider:notify-cke"; + evidence.notifyCkeSource.detail = "audited notify CKE evidence"; + evidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.notifyCkeSource.packageFingerprint = packageFingerprint; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.channelBindingSource.source = "unit-test-public-provider:channel-binding"; + evidence.channelBindingSource.detail = "audited channel binding evidence"; + evidence.channelBindingSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingSource.endpointRouteVerified = true; + evidence.channelBindingSource.packageFingerprint = packageFingerprint; + evidence.missionSource.deviceId = package.installScope.deviceId; + evidence.repositorySource.deviceId = package.installScope.deviceId; + evidence.localXnSource.deviceId = package.installScope.deviceId; + evidence.remoteXnSource.deviceId = package.installScope.deviceId; + evidence.notifyCkeSource.deviceId = package.installScope.deviceId; + evidence.channelBindingSource.deviceId = package.installScope.deviceId; + evidence.missionSource.rank = package.installScope.rank; + evidence.repositorySource.rank = package.installScope.rank; + evidence.localXnSource.rank = package.installScope.rank; + evidence.remoteXnSource.rank = package.installScope.rank; + evidence.notifyCkeSource.rank = package.installScope.rank; + evidence.channelBindingSource.rank = package.installScope.rank; + evidence.missionSource.provider = package.installScope.provider; + evidence.repositorySource.provider = package.installScope.provider; + evidence.localXnSource.provider = package.installScope.provider; + evidence.remoteXnSource.provider = package.installScope.provider; + evidence.notifyCkeSource.provider = package.installScope.provider; + evidence.channelBindingSource.provider = package.installScope.provider; + evidence.missionSource.installAttemptReceiptId = 0xabc001ULL; + evidence.repositorySource.installAttemptReceiptId = 0xabc001ULL; + evidence.localXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.remoteXnSource.installAttemptReceiptId = 0xabc001ULL; + evidence.notifyCkeSource.installAttemptReceiptId = 0xabc001ULL; + evidence.channelBindingSource.installAttemptReceiptId = 0xabc001ULL; + return evidence; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + if (TileXRCcuBindLaunchPackageInstallScope( + &package, 3, 1, "unit-test-public-install-provider") != TILEXR_SUCCESS) { + std::cerr << "scope bind failed\n"; + return 9; + } + + TileXRCcuProviderReport report; + TileXRCcuHardwareInstallEvidence privateEvidence = VerifiedEvidence(package); + privateEvidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PrivateObserved; + privateEvidence.remoteXnSource.source = "hcomm::CcuKernel::CreateVariable"; + if (TileXRCcuValidateHardwareInstall(package, privateEvidence, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "private remote XN evidence was accepted\n"; + return 2; + } + if (report.message.find("remote XN") == std::string::npos || + report.message.find("private") == std::string::npos) { + std::cerr << "private evidence diagnostic was weak: " << report.message << "\n"; + return 3; + } + + TileXRCcuHardwareInstallEvidence candidateEvidence = VerifiedEvidence(package); + candidateEvidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicCandidate; + candidateEvidence.channelBindingSource.source = "rtCcuBindChannel"; + if (TileXRCcuPrepareSubmitTasks(package, candidateEvidence, nullptr, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null submit vector was not rejected first\n"; + return 4; + } + + std::vector submitTasks; + if (TileXRCcuPrepareSubmitTasks(package, candidateEvidence, &submitTasks, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "public candidate channel binding was accepted\n"; + return 5; + } + if (!submitTasks.empty()) { + std::cerr << "candidate prepare left submit tasks\n"; + return 6; + } + if (report.message.find("channel") == std::string::npos || + report.message.find("candidate") == std::string::npos) { + std::cerr << "candidate diagnostic was weak: " << report.message << "\n"; + return 7; + } + + TileXRCcuHardwareInstallEvidence verified = VerifiedEvidence(package); + if (TileXRCcuPrepareSubmitTasks(package, verified, &submitTasks, &report) != TILEXR_SUCCESS) { + std::cerr << "public verified evidence was rejected: " << report.message << "\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_report_counts_evidence_kinds_for_auditing(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_provider.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuLaunchPackage package; + TileXRCcuLaunchPackageReport packageReport; + if (TileXRCcuBuildLaunchPackage(MakePlan(), &package, &packageReport) != TILEXR_SUCCESS) { + std::cerr << "package build failed\n"; + return 1; + } + + TileXRCcuHardwareInstallEvidence evidence; + evidence.missionInstalled = true; + evidence.repositoryInstalled = true; + evidence.localXnInstalled = true; + evidence.remoteXnBound = true; + evidence.notifyCkeInstalled = true; + evidence.channelBindingsInstalled = false; + evidence.missionSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.repositorySource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.localXnSource.kind = TileXRCcuEvidenceKind::LegacyBoolean; + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PrivateObserved; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicCandidate; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::Missing; + + TileXRCcuProviderReport report; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "mixed bad evidence was accepted\n"; + return 2; + } + if (report.evidenceBitCount != 6 || + report.publicVerifiedEvidenceCount != 2 || + report.legacyEvidenceCount != 1 || + report.privateObservedEvidenceCount != 1 || + report.publicCandidateEvidenceCount != 1 || + report.missingEvidenceCount != 1) { + std::cerr << "unexpected evidence counters: bits=" << report.evidenceBitCount + << " public=" << report.publicVerifiedEvidenceCount + << " legacy=" << report.legacyEvidenceCount + << " private=" << report.privateObservedEvidenceCount + << " candidate=" << report.publicCandidateEvidenceCount + << " missing=" << report.missingEvidenceCount << "\n"; + return 3; + } + if (report.message.find("remote XN") == std::string::npos || + report.message.find("private") == std::string::npos) { + std::cerr << "first rejection diagnostic was weak: " << report.message << "\n"; + return 4; + } + + evidence.remoteXnSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.remoteXnSource.source = "unit-test-public-provider:remote-xn"; + evidence.remoteXnSource.detail = "audited remote XN evidence"; + evidence.remoteXnSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.remoteXnSource.endpointRouteVerified = true; + evidence.notifyCkeSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.notifyCkeSource.source = "unit-test-public-provider:notify-cke"; + evidence.notifyCkeSource.detail = "audited notify CKE evidence"; + evidence.notifyCkeSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingsInstalled = true; + evidence.channelBindingSource.kind = TileXRCcuEvidenceKind::PublicVerified; + evidence.channelBindingSource.source = "unit-test-public-provider:channel-binding"; + evidence.channelBindingSource.detail = "audited channel binding evidence"; + evidence.channelBindingSource.surface = TileXRCcuEvidenceSurface::PublicInstallProvider; + evidence.channelBindingSource.endpointRouteVerified = true; + if (TileXRCcuValidateHardwareInstall(package, evidence, &report) != TILEXR_SUCCESS) { + std::cerr << "fixed evidence was rejected: " << report.message << "\n"; + return 5; + } + if (report.evidenceBitCount != 6 || + report.publicVerifiedEvidenceCount != 5 || + report.legacyEvidenceCount != 1 || + report.privateObservedEvidenceCount != 0 || + report.publicCandidateEvidenceCount != 0 || + report.missingEvidenceCount != 0 || + report.submitReady) { + std::cerr << "unexpected ready evidence counters\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_layer_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = PROVIDER_HEADER.read_text(encoding="utf-8") + source = PROVIDER_SOURCE.read_text(encoding="utf-8") + package_header = PACKAGE_HEADER.read_text(encoding="utf-8") + package_source = PACKAGE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_provider.h", cmake) + self.assertIn("ccu/tilexr_ccu_provider.cpp", cmake) + self.assertIn("struct TileXRCcuHardwareInstallEvidence", header) + self.assertIn("enum class TileXRCcuEvidenceKind", header) + self.assertIn("enum class TileXRCcuEvidenceSurface", header) + self.assertIn("struct TileXRCcuEvidenceSource", header) + self.assertIn("TileXRCcuValidateHardwareInstall", header) + self.assertIn("TileXRCcuPrepareSubmitTasks", header) + self.assertIn("tilexr_ccu_launch_package.h", header) + self.assertIn("submitReady", header) + self.assertIn("evidenceBitCount", header) + self.assertIn("publicVerifiedEvidenceCount", header) + self.assertIn("publicCandidateEvidenceCount", header) + self.assertIn("privateObservedEvidenceCount", header) + self.assertIn("missingEvidenceCount", header) + self.assertIn("uint64_t packageFingerprint", header) + self.assertIn("uint32_t deviceId", header) + self.assertIn("uint32_t rank", header) + self.assertIn("std::string provider", header) + self.assertIn("uint64_t installAttemptReceiptId", header) + self.assertIn("TileXRCcuComputeLaunchPackageFingerprint", package_header) + self.assertIn("TileXRCcuComputeLaunchPackageFingerprint", package_source) + self.assertIn("package fingerprint mismatch", source) + self.assertIn("TileXRCcuLaunchInstallScope", package_header) + self.assertIn("installScope", package_header) + self.assertIn("TileXRCcuBindLaunchPackageInstallScope", package_header) + self.assertIn("device scope mismatch", source) + self.assertIn("rank scope mismatch", source) + self.assertIn("provider scope mismatch", source) + self.assertIn("install attempt receipt", source) + self.assertIn("endpointRouteVerified", header) + self.assertIn("submit requires verified endpoint route evidence", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_prepare_path_requires_public_verified_evidence_source(self): + source = PROVIDER_SOURCE.read_text(encoding="utf-8") + prepare_start = source.index("int TileXRCcuPrepareSubmitTasks(") + prepare_body = source[prepare_start:] + + validate_pos = prepare_body.index("TileXRCcuValidateHardwareInstall(package, evidence, report)") + submit_gate_pos = prepare_body.index("ValidateSubmitEvidence(package, evidence, report)") + copy_pos = prepare_body.index("*submitTasks = package.tasks") + + self.assertIn("submit requires public verified evidence", source) + self.assertLess(validate_pos, submit_gate_pos) + self.assertLess(submit_gate_pos, copy_pos) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py b/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py new file mode 100644 index 00000000..da73b900 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROBE_C = REPO_ROOT / "tests" / "ccu" / "ccu_public_direct_api_compile_probe.c" +INCLUDE_DIR = REPO_ROOT / "src" / "include" + + +class TileXRCcuPublicApiCompileProbeTest(unittest.TestCase): + def test_probe_source_uses_only_public_api_header(self): + source = PROBE_C.read_text(encoding="utf-8") + + self.assertIn('#include "tilexr_api.h"', source) + for needle in [ + '#include "tilexr_comm.h"', + '#include "ccu/', + "#include ", source) + self.assertNotIn("TileXRCcuDirectInstallAttempt", source) + self.assertNotIn("TileXRCcuSubmitPreparedTasks", source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py new file mode 100644 index 00000000..91903270 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py @@ -0,0 +1,2110 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +LOADER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.h" +LOADER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp" +TYPES_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_types.h" +ABI_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_abi_constants.h" +DIRECT_RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.h" +DIRECT_RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp" +DRIVER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.h" +DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" + + +class TileXRCcuRaCustomChannelLoaderTest(unittest.TestCase): + def compile_and_run(self, code: str, extra_sources=None, extra_link_flags=None, extra_env=None): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + extra_sources = extra_sources or [] + extra_link_flags = extra_link_flags or [] + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "hccp_loader_test.cpp" + test_bin = temp_path / "hccp_loader_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(REPO_ROOT / "src" / "include"), + "-I", + str(REPO_ROOT / "src" / "comm"), + str(test_cpp), + *[str(source) for source in extra_sources], + "-o", + str(test_bin), + *extra_link_flags, + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + env = None if extra_env is None else {**extra_env} + return subprocess.run( + [str(test_bin)], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + env=env, + ) + + def test_ccu_hccp_loader_exposes_ra_custom_channel_for_tilexr_ccu(self): + header = LOADER_HEADER.read_text(encoding="utf-8") + source = LOADER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TileXRCcuRaCustomChannelFunc", TYPES_HEADER.read_text(encoding="utf-8")) + self.assertIn("RaCustomChannel", header) + self.assertIn("RaCustomChannel = nullptr", header) + self.assertIn('LoadSymbol(raHandle_, RaCustomChannel, "RaCustomChannel", "ra_custom_channel")', source) + self.assertIn("RaCustomChannel = nullptr", source) + + def test_ccu_hccp_loader_exposes_runtime_phy_id_mapping_for_ccu_runtime(self): + header = LOADER_HEADER.read_text(encoding="utf-8") + source = LOADER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TileXRCcuRtGetDevicePhyIdByIndexFunc", TYPES_HEADER.read_text(encoding="utf-8")) + self.assertIn("RtGetDevicePhyIdByIndex", header) + self.assertIn("RtGetDevicePhyIdByIndex = nullptr", header) + self.assertIn('dlopen("libruntime.so", RTLD_NOW)', source) + self.assertIn( + 'LoadOptionalSymbol(runtimeHandle_, RtGetDevicePhyIdByIndex, "rtGetDevicePhyIdByIndex", nullptr)', + source, + ) + self.assertIn("ResolveDevicePhyId", header) + self.assertIn("ResolveDevicePhyId", source) + + def test_ccu_hccp_loader_exposes_public_ra_ctx_resource_window_symbols(self): + header = LOADER_HEADER.read_text(encoding="utf-8") + source = LOADER_SOURCE.read_text(encoding="utf-8") + types = TYPES_HEADER.read_text(encoding="utf-8") + + for needle in [ + "TileXRCcuRaGetDevEidInfoNumFunc", + "TileXRCcuRaGetDevEidInfoListFunc", + "TileXRCcuRaCtxInitFunc", + "TileXRCcuRaCtxDeinitFunc", + "TileXRCcuRaCtxTokenIdAllocFunc", + "TileXRCcuRaCtxTokenIdFreeFunc", + "TileXRCcuRaCtxLmemRegisterFunc", + "TileXRCcuRaCtxLmemUnregisterFunc", + "TileXRCcuRaGetSecRandomFunc", + "TileXRCcuHccpMrRegInfo", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, types) + + for needle in [ + "RaGetDevEidInfoNum", + "RaGetDevEidInfoList", + "RaCtxInit", + "RaCtxDeinit", + "RaCtxTokenIdAlloc", + "RaCtxTokenIdFree", + "RaCtxLmemRegister", + "RaCtxLmemUnregister", + "RaGetSecRandom", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, header) + self.assertIn(needle, source) + + def test_ccu_hccp_loader_exposes_public_ra_ctx_endpoint_route_symbols(self): + header = LOADER_HEADER.read_text(encoding="utf-8") + source = LOADER_SOURCE.read_text(encoding="utf-8") + types = TYPES_HEADER.read_text(encoding="utf-8") + + for needle in [ + "TileXRCcuRaCtxCqCreateFunc", + "TileXRCcuRaCtxCqDestroyFunc", + "TileXRCcuRaCtxQpCreateFunc", + "TileXRCcuRaCtxQpDestroyFunc", + "TileXRCcuRaCtxQpImportFunc", + "TileXRCcuRaCtxQpUnimportFunc", + "TileXRCcuRaCtxQpBindFunc", + "TileXRCcuRaCtxQpUnbindFunc", + "TileXRCcuRaGetTpInfoListAsyncFunc", + "TileXRCcuRaGetAsyncReqResultFunc", + "TileXRCcuHccpQpCreateAttr", + "TileXRCcuHccpQpImportInfo", + "TileXRCcuHccpGetTpCfg", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, types) + + for needle in [ + "RaCtxCqCreate", + "RaCtxCqDestroy", + "RaCtxQpCreate", + "RaCtxQpDestroy", + "RaCtxQpImport", + "RaCtxQpUnimport", + "RaCtxQpBind", + "RaCtxQpUnbind", + "RaGetTpInfoListAsync", + "RaGetAsyncReqResult", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, header) + self.assertIn(needle, source) + + def test_ccu_hccp_loader_fails_closed_when_runtime_phy_id_mapping_fails(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + int FakeRtGetDevicePhyIdByIndex(uint32_t logicDevId, uint32_t* phyId) + { + if (logicDevId != 7 || phyId == nullptr) { + return -99; + } + *phyId = 0xbeefU; + return -1234; + } + + int main() + { + TileXRCcuHccpLoader loader; + loader.loaded_ = true; + loader.RtGetDevicePhyIdByIndex = FakeRtGetDevicePhyIdByIndex; + + uint32_t phyId = 0; + TileXRCcuHccpLoaderReport report; + const int ret = loader.ResolveDevicePhyId(7, &phyId, &report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "phy id mapping failure silently succeeded\n"; + return 1; + } + if (phyId == 7 || phyId == 0xbeefU) { + std::cerr << "failed mapping leaked fallback phy id: " << phyId << "\n"; + return 2; + } + if (report.message.find("rtGetDevicePhyIdByIndex failed") == std::string::npos) { + std::cerr << "missing phy id failure message: " << report.message << "\n"; + return 3; + } + if (report.logicDevId != 7 || report.runtimePhyIdRet != -1234) { + std::cerr << "phy id report mismatch\n"; + return 4; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_initializes_and_deinitializes_ra_hdc_session(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + TileXRCcuRaInitConfig g_lastInitConfig {}; + TileXRCcuRaInitConfig g_lastDeinitConfig {}; + int g_initCalls = 0; + int g_deinitCalls = 0; + int g_openCalls = 0; + int g_closeCalls = 0; + + int FakeRtOpenNetService(const TileXRCcuRtNetServiceOpenArgs*) + { + ++g_openCalls; + return 0; + } + + int FakeRtCloseNetService() + { + ++g_closeCalls; + return 0; + } + + int FakeRaInit(TileXRCcuRaInitConfig* config) + { + if (config == nullptr) { + return -9; + } + g_lastInitConfig = *config; + ++g_initCalls; + return 0; + } + + int FakeRaDeinit(TileXRCcuRaInitConfig* config) + { + if (config == nullptr) { + return -8; + } + g_lastDeinitConfig = *config; + ++g_deinitCalls; + return 0; + } + + int main() + { + TileXRCcuHccpLoader first; + first.loaded_ = true; + first.RtOpenNetService = FakeRtOpenNetService; + first.RtCloseNetService = FakeRtCloseNetService; + first.RaInit = FakeRaInit; + first.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoader second; + second.loaded_ = true; + second.RtOpenNetService = FakeRtOpenNetService; + second.RtCloseNetService = FakeRtCloseNetService; + second.RaInit = FakeRaInit; + second.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoaderReport report; + if (first.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report) != TILEXR_SUCCESS) { + std::cerr << "first init failed: " << report.message << "\n"; + return 1; + } + if (second.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report) != TILEXR_SUCCESS) { + std::cerr << "second init failed: " << report.message << "\n"; + return 2; + } + if (g_initCalls != 1 || !report.raInitialized || report.raInitRefCount != 2) { + std::cerr << "RA init refcount mismatch\n"; + return 3; + } + if (g_openCalls != 1 || report.netServiceRefCount != 2) { + std::cerr << "net service refcount mismatch\n"; + return 8; + } + if (g_lastInitConfig.phyId != 3 || + g_lastInitConfig.nicPosition != TILEXR_CCU_NETWORK_OFFLINE || + g_lastInitConfig.hdcType != TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2 || + !g_lastInitConfig.enableHdcAsync) { + std::cerr << "RA init config mismatch\n"; + return 4; + } + + second.Unload(); + if (g_deinitCalls != 0) { + std::cerr << "RA deinit happened before last reference\n"; + return 5; + } + if (g_closeCalls != 0) { + std::cerr << "runtime net service closed before last reference\n"; + return 9; + } + first.Unload(); + if (g_deinitCalls != 1) { + std::cerr << "RA deinit count mismatch: " << g_deinitCalls << "\n"; + return 6; + } + if (g_closeCalls != 1) { + std::cerr << "runtime net service close count mismatch: " << g_closeCalls << "\n"; + return 10; + } + if (g_lastDeinitConfig.phyId != 3 || + g_lastDeinitConfig.nicPosition != TILEXR_CCU_NETWORK_OFFLINE || + g_lastDeinitConfig.hdcType != TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2) { + std::cerr << "RA deinit config mismatch\n"; + return 7; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_opens_runtime_net_service_before_ra_init(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + #include + #include + + using namespace TileXR; + + std::string g_openArg; + int g_openCalls = 0; + int g_closeCalls = 0; + int g_initCalls = 0; + int g_order = 0; + int g_openOrder = 0; + int g_initOrder = 0; + + int FakeRtOpenNetService(const TileXRCcuRtNetServiceOpenArgs* args) + { + ++g_openCalls; + g_openOrder = ++g_order; + if (args == nullptr || args->extParamList == nullptr || args->extParamCnt != 1 || + args->extParamList[0].paramInfo == nullptr) { + return -11; + } + g_openArg.assign(args->extParamList[0].paramInfo, args->extParamList[0].paramLen); + return 0; + } + + int FakeRtCloseNetService() + { + ++g_closeCalls; + return 0; + } + + int FakeRaInit(TileXRCcuRaInitConfig*) + { + ++g_initCalls; + g_initOrder = ++g_order; + return 0; + } + + int FakeRaDeinit(TileXRCcuRaInitConfig*) + { + return 0; + } + + int main() + { + TileXRCcuHccpLoader loader; + loader.loaded_ = true; + loader.RtOpenNetService = FakeRtOpenNetService; + loader.RtCloseNetService = FakeRtCloseNetService; + loader.RaInit = FakeRaInit; + loader.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoaderReport report; + if (loader.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + if (g_openCalls != 1 || g_initCalls != 1 || g_openArg != "--hdcType=18") { + std::cerr << "open/init mismatch open=" << g_openCalls + << " init=" << g_initCalls + << " arg=" << g_openArg << "\n"; + return 2; + } + if (g_openOrder == 0 || g_initOrder == 0 || g_openOrder > g_initOrder) { + std::cerr << "rtOpenNetService did not precede RaInit\n"; + return 3; + } + loader.Unload(); + if (g_closeCalls != 1) { + std::cerr << "close count mismatch: " << g_closeCalls << "\n"; + return 4; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_refcounts_runtime_net_service_by_hdc_type(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + int g_openCalls = 0; + int g_closeCalls = 0; + int g_initCalls = 0; + int g_deinitCalls = 0; + + int FakeRtOpenNetService(const TileXRCcuRtNetServiceOpenArgs*) + { + ++g_openCalls; + return 0; + } + + int FakeRtCloseNetService() + { + ++g_closeCalls; + return 0; + } + + int FakeRaInit(TileXRCcuRaInitConfig*) + { + ++g_initCalls; + return 0; + } + + int FakeRaDeinit(TileXRCcuRaInitConfig*) + { + ++g_deinitCalls; + return 0; + } + + int main() + { + TileXRCcuHccpLoader first; + first.loaded_ = true; + first.RtOpenNetService = FakeRtOpenNetService; + first.RtCloseNetService = FakeRtCloseNetService; + first.RaInit = FakeRaInit; + first.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoader second; + second.loaded_ = true; + second.RtOpenNetService = FakeRtOpenNetService; + second.RtCloseNetService = FakeRtCloseNetService; + second.RaInit = FakeRaInit; + second.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoaderReport report; + if (first.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report) != TILEXR_SUCCESS || + second.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report) != TILEXR_SUCCESS) { + std::cerr << "init failed: " << report.message << "\n"; + return 1; + } + if (g_openCalls != 1 || g_initCalls != 1 || report.raInitRefCount != 2) { + std::cerr << "open/init ref mismatch open=" << g_openCalls + << " init=" << g_initCalls + << " refs=" << report.raInitRefCount << "\n"; + return 2; + } + second.Unload(); + if (g_closeCalls != 0 || g_deinitCalls != 0) { + std::cerr << "closed/deinit before final reference\n"; + return 3; + } + first.Unload(); + if (g_closeCalls != 1 || g_deinitCalls != 1) { + std::cerr << "final close/deinit mismatch close=" << g_closeCalls + << " deinit=" << g_deinitCalls << "\n"; + return 4; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_fails_before_ra_init_when_net_service_open_fails(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + int g_openCalls = 0; + int g_initCalls = 0; + + int FakeRtOpenNetService(const TileXRCcuRtNetServiceOpenArgs*) + { + ++g_openCalls; + return 128000; + } + + int FakeRtCloseNetService() + { + return 0; + } + + int FakeRaInit(TileXRCcuRaInitConfig*) + { + ++g_initCalls; + return 0; + } + + int FakeRaDeinit(TileXRCcuRaInitConfig*) + { + return 0; + } + + int main() + { + TileXRCcuHccpLoader loader; + loader.loaded_ = true; + loader.RtOpenNetService = FakeRtOpenNetService; + loader.RtCloseNetService = FakeRtCloseNetService; + loader.RaInit = FakeRaInit; + loader.RaDeinit = FakeRaDeinit; + + TileXRCcuHccpLoaderReport report; + const int ret = loader.InitRaHdc(3, TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2, true, &report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "open failure unexpectedly succeeded\n"; + return 1; + } + if (g_openCalls != 1 || g_initCalls != 0) { + std::cerr << "open failure call mismatch open=" << g_openCalls + << " init=" << g_initCalls << "\n"; + return 2; + } + if (report.message.find("rtOpenNetService failed ret=128000") == std::string::npos || + report.message.find("--hdcType=18") == std::string::npos) { + std::cerr << "missing open failure diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_initializes_and_releases_ccu_tlv_session(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + TileXRCcuTlvInitInfo g_lastInitInfo {}; + uint32_t g_lastRequestModule = 0; + uint32_t g_lastRequestType = 0; + uint32_t g_lastReleaseType = 0; + int g_initCalls = 0; + int g_requestCalls = 0; + int g_deinitCalls = 0; + int g_order = 0; + int g_initOrder = 0; + int g_requestOrder = 0; + int g_deinitOrder = 0; + + int FakeRaTlvInit(TileXRCcuTlvInitInfo* initInfo, uint32_t* bufferSize, void** tlvHandle) + { + ++g_initCalls; + g_initOrder = ++g_order; + if (initInfo == nullptr || bufferSize == nullptr || tlvHandle == nullptr) { + return -1; + } + g_lastInitInfo = *initInfo; + *bufferSize = 4096; + *tlvHandle = reinterpret_cast(0x12345678ULL); + return 0; + } + + int FakeRaTlvRequest(void* tlvHandle, uint32_t moduleType, TileXRCcuTlvMsg* sendMsg, TileXRCcuTlvMsg* recvMsg) + { + ++g_requestCalls; + g_requestOrder = ++g_order; + if (tlvHandle != reinterpret_cast(0x12345678ULL) || sendMsg == nullptr || recvMsg == nullptr) { + return -2; + } + g_lastRequestModule = moduleType; + if (sendMsg->type == TILEXR_CCU_TLV_MSG_TYPE_CCU_UNINIT) { + g_lastReleaseType = sendMsg->type; + } else { + g_lastRequestType = sendMsg->type; + } + return 0; + } + + int FakeRaTlvDeinit(void* tlvHandle) + { + ++g_deinitCalls; + g_deinitOrder = ++g_order; + return tlvHandle == reinterpret_cast(0x12345678ULL) ? 0 : -3; + } + + int main() + { + TileXRCcuHccpLoader loader; + loader.loaded_ = true; + loader.RaTlvInit = FakeRaTlvInit; + loader.RaTlvRequest = FakeRaTlvRequest; + loader.RaTlvDeinit = FakeRaTlvDeinit; + + TileXRCcuHccpLoaderReport report; + if (loader.InitCcuTlv(3, &report) != TILEXR_SUCCESS) { + std::cerr << "tlv init failed: " << report.message << "\n"; + return 1; + } + if (!report.ccuTlvInitialized || report.ccuTlvRefCount != 1 || + report.ccuTlvBufferSize != 4096) { + std::cerr << "tlv report mismatch\n"; + return 2; + } + if (g_initCalls != 1 || g_requestCalls != 1 || g_deinitCalls != 0) { + std::cerr << "tlv init call count mismatch\n"; + return 3; + } + if (g_lastInitInfo.version != TILEXR_CCU_TLV_VERSION || + g_lastInitInfo.phyId != 3 || + g_lastInitInfo.nicPosition != TILEXR_CCU_NETWORK_OFFLINE || + g_lastRequestModule != TILEXR_CCU_TLV_MODULE_TYPE_CCU || + g_lastRequestType != TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT) { + std::cerr << "tlv init/request envelope mismatch\n"; + return 4; + } + if (g_initOrder == 0 || g_requestOrder == 0 || g_initOrder > g_requestOrder) { + std::cerr << "tlv request did not follow init\n"; + return 5; + } + + loader.Unload(); + if (g_requestCalls != 2 || g_lastReleaseType != TILEXR_CCU_TLV_MSG_TYPE_CCU_UNINIT || + g_deinitCalls != 1) { + std::cerr << "tlv release mismatch requestCalls=" << g_requestCalls + << " releaseType=" << g_lastReleaseType + << " deinitCalls=" << g_deinitCalls << "\n"; + return 6; + } + if (g_deinitOrder == 0 || g_requestOrder > g_deinitOrder) { + std::cerr << "tlv deinit happened before release request\n"; + return 7; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_refcounts_ccu_tlv_by_device_phy_id(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + int g_initCalls = 0; + int g_requestCalls = 0; + int g_deinitCalls = 0; + + int FakeRaTlvInit(TileXRCcuTlvInitInfo*, uint32_t* bufferSize, void** tlvHandle) + { + ++g_initCalls; + *bufferSize = 1024; + *tlvHandle = reinterpret_cast(0x1000ULL); + return 0; + } + + int FakeRaTlvRequest(void*, uint32_t, TileXRCcuTlvMsg*, TileXRCcuTlvMsg*) + { + ++g_requestCalls; + return 0; + } + + int FakeRaTlvDeinit(void*) + { + ++g_deinitCalls; + return 0; + } + + int main() + { + TileXRCcuHccpLoader first; + first.loaded_ = true; + first.RaTlvInit = FakeRaTlvInit; + first.RaTlvRequest = FakeRaTlvRequest; + first.RaTlvDeinit = FakeRaTlvDeinit; + + TileXRCcuHccpLoader second; + second.loaded_ = true; + second.RaTlvInit = FakeRaTlvInit; + second.RaTlvRequest = FakeRaTlvRequest; + second.RaTlvDeinit = FakeRaTlvDeinit; + + TileXRCcuHccpLoaderReport report; + if (first.InitCcuTlv(3, &report) != TILEXR_SUCCESS || + second.InitCcuTlv(3, &report) != TILEXR_SUCCESS) { + std::cerr << "tlv init failed: " << report.message << "\n"; + return 1; + } + if (g_initCalls != 1 || g_requestCalls != 1 || report.ccuTlvRefCount != 2) { + std::cerr << "tlv refcount mismatch init=" << g_initCalls + << " request=" << g_requestCalls + << " refs=" << report.ccuTlvRefCount << "\n"; + return 2; + } + second.Unload(); + if (g_deinitCalls != 0 || g_requestCalls != 1) { + std::cerr << "tlv released before final reference\n"; + return 3; + } + first.Unload(); + if (g_deinitCalls != 1 || g_requestCalls != 2) { + std::cerr << "tlv final release mismatch deinit=" << g_deinitCalls + << " request=" << g_requestCalls << "\n"; + return 4; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_hccp_loader_rolls_back_tlv_handle_when_ccu_init_request_fails(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_hccp_loader.h" + #undef private + + #include + + using namespace TileXR; + + int g_initCalls = 0; + int g_requestCalls = 0; + int g_deinitCalls = 0; + + int FakeRaTlvInit(TileXRCcuTlvInitInfo*, uint32_t* bufferSize, void** tlvHandle) + { + ++g_initCalls; + *bufferSize = 1024; + *tlvHandle = reinterpret_cast(0x1000ULL); + return 0; + } + + int FakeRaTlvRequest(void*, uint32_t moduleType, TileXRCcuTlvMsg* sendMsg, TileXRCcuTlvMsg*) + { + ++g_requestCalls; + if (moduleType != TILEXR_CCU_TLV_MODULE_TYPE_CCU || + sendMsg == nullptr || sendMsg->type != TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT) { + return -77; + } + return 128308; + } + + int FakeRaTlvDeinit(void*) + { + ++g_deinitCalls; + return 0; + } + + int main() + { + TileXRCcuHccpLoader loader; + loader.loaded_ = true; + loader.RaTlvInit = FakeRaTlvInit; + loader.RaTlvRequest = FakeRaTlvRequest; + loader.RaTlvDeinit = FakeRaTlvDeinit; + + TileXRCcuHccpLoaderReport report; + const int ret = loader.InitCcuTlv(3, &report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "tlv request failure unexpectedly succeeded\n"; + return 1; + } + if (g_initCalls != 1 || g_requestCalls != 1 || g_deinitCalls != 1) { + std::cerr << "tlv rollback mismatch init=" << g_initCalls + << " request=" << g_requestCalls + << " deinit=" << g_deinitCalls << "\n"; + return 2; + } + if (report.ccuTlvInitialized || report.raTlvRequestRet != 128308 || + report.message.find("RaTlvRequest CCU_INIT failed ret=128308") == std::string::npos) { + std::cerr << "weak tlv failure diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + result = self.compile_and_run(code, extra_sources=[LOADER_SOURCE], extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_ccu_runtime_initializes_ra_hdc_after_resolving_device_phy_id(self): + header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + types = TYPES_HEADER.read_text(encoding="utf-8") + loader_header = LOADER_HEADER.read_text(encoding="utf-8") + loader_source = LOADER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TileXRCcuRaInitConfig", types) + self.assertIn("TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2", types) + self.assertIn("TileXRCcuRaInitFunc", types) + self.assertIn("TileXRCcuRaDeinitFunc", types) + self.assertIn("RaInit", loader_header) + self.assertIn("RaDeinit", loader_header) + self.assertIn("InitRaHdc", loader_header) + self.assertIn('LoadSymbol(raHandle_, RaInit, "RaInit", nullptr)', loader_source) + self.assertIn('LoadSymbol(raHandle_, RaDeinit, "RaDeinit", nullptr)', loader_source) + self.assertIn("InitCcuTlv", loader_header) + self.assertIn('LoadOptionalSymbol(raHandle_, RaTlvInit, "RaTlvInit", nullptr)', loader_source) + self.assertIn('LoadOptionalSymbol(raHandle_, RaTlvRequest, "RaTlvRequest", nullptr)', loader_source) + self.assertIn('LoadOptionalSymbol(raHandle_, RaTlvDeinit, "RaTlvDeinit", nullptr)', loader_source) + self.assertIn("loader_.InitRaHdc", source) + self.assertIn("loader_.InitRaHdc(devicePhyId_, hdcType, true, &raReport)", source) + self.assertIn("loader_.InitCcuTlv(devicePhyId_, &tlvReport)", source) + self.assertIn("SelectDirectCcuHdcType", source) + self.assertIn("TILEXR_CCU_DIRECT_HDC_TYPE", source) + self.assertIn("raInitialized", header) + self.assertIn("ccuTlvInitialized", header) + + def test_direct_runtime_ra_init_failure_report_keeps_phy_and_hdc_context(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_runtime.h" + + #include + #include + + using namespace TileXR; + + int main() + { + setenv("TILEXR_CCU_DIRECT_HDC_TYPE", "6", 1); + + TileXRCcuDirectRuntime runtime; + TileXRCcuDirectRuntimeOptions options; + options.devId = 7; + options.rank = 0; + options.rankSize = 2; + TileXRCcuDirectRuntimeReport report; + const int ret = runtime.Init(options, &report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "runtime init unexpectedly succeeded\n"; + return 1; + } + if (report.logicDevId != 7 || report.devicePhyId != 0xbeefU || + report.hdcType != TILEXR_CCU_HDC_SERVICE_TYPE_RDMA || + report.raInitialized) { + std::cerr << "report lost init context logic=" << report.logicDevId + << " phy=" << report.devicePhyId + << " hdc=" << report.hdcType + << " ra=" << report.raInitialized << "\n"; + return 2; + } + if (report.message.find("RaInit failed") == std::string::npos) { + std::cerr << "missing RA failure diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + libra_code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_hccp_types.h" + + extern "C" int RaCustomChannel( + TileXR::TileXRCcuRaInfo, + TileXR::TileXRCcuCustomChannelIn*, + TileXR::TileXRCcuCustomChannelOut*) + { + return 0; + } + + extern "C" int RaInit(TileXR::TileXRCcuRaInitConfig*) + { + return -55; + } + + extern "C" int RaDeinit(TileXR::TileXRCcuRaInitConfig*) + { + return 0; + } + ''' + ) + runtime_code = textwrap.dedent( + r''' + #include + + extern "C" int rtGetDevicePhyIdByIndex(uint32_t logicDevId, uint32_t* phyId) + { + if (logicDevId != 7 || phyId == nullptr) { + return -1; + } + *phyId = 0xbeefU; + return 0; + } + + struct TileXRCcuRtProcExtParam { + const char *paramInfo; + uint64_t paramLen; + }; + + struct TileXRCcuRtNetServiceOpenArgs { + TileXRCcuRtProcExtParam *extParamList; + uint64_t extParamCnt; + }; + + extern "C" int rtOpenNetService(const TileXRCcuRtNetServiceOpenArgs*) + { + return 0; + } + + extern "C" int rtCloseNetService() + { + return 0; + } + ''' + ) + + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + libra_cpp = temp_path / "fake_libra.cpp" + runtime_cpp = temp_path / "fake_libruntime.cpp" + libra_so = temp_path / "libra.so" + runtime_so = temp_path / "libruntime.so" + libra_cpp.write_text(libra_code, encoding="utf-8") + runtime_cpp.write_text(runtime_code, encoding="utf-8") + common_flags = [ + compiler, + "-std=c++14", + "-shared", + "-fPIC", + "-I", + str(REPO_ROOT / "src" / "include"), + "-I", + str(REPO_ROOT / "src" / "comm"), + ] + subprocess.run( + [*common_flags, str(libra_cpp), "-o", str(libra_so)], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + subprocess.run( + [*common_flags, str(runtime_cpp), "-o", str(runtime_so)], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + DRIVER_SOURCE, + ], + extra_link_flags=["-ldl"], + extra_env={"LD_LIBRARY_PATH": str(temp_path)}, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_runtime_tlv_init_failure_report_keeps_ra_phy_and_hdc_context(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_runtime.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuDirectRuntime runtime; + TileXRCcuDirectRuntimeOptions options; + options.devId = 7; + options.rank = 0; + options.rankSize = 2; + TileXRCcuDirectRuntimeReport report; + const int ret = runtime.Init(options, &report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "runtime init unexpectedly succeeded\n"; + return 1; + } + if (report.logicDevId != 7 || report.devicePhyId != 0xbeefU || + report.hdcType != TILEXR_CCU_HDC_SERVICE_TYPE_RDMA_V2 || + !report.raInitialized || report.ccuTlvInitialized) { + std::cerr << "report lost TLV failure context logic=" << report.logicDevId + << " phy=" << report.devicePhyId + << " hdc=" << report.hdcType + << " ra=" << report.raInitialized + << " tlv=" << report.ccuTlvInitialized << "\n"; + return 2; + } + if (report.message.find("RaTlvRequest CCU_INIT failed ret=128308") == std::string::npos) { + std::cerr << "missing TLV failure diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + libra_code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_hccp_types.h" + + extern "C" int RaCustomChannel( + TileXR::TileXRCcuRaInfo, + TileXR::TileXRCcuCustomChannelIn*, + TileXR::TileXRCcuCustomChannelOut*) + { + return 0; + } + + extern "C" int RaInit(TileXR::TileXRCcuRaInitConfig*) + { + return 0; + } + + extern "C" int RaDeinit(TileXR::TileXRCcuRaInitConfig*) + { + return 0; + } + + extern "C" int RaTlvInit(TileXR::TileXRCcuTlvInitInfo*, uint32_t* bufferSize, void** tlvHandle) + { + *bufferSize = 1024; + *tlvHandle = reinterpret_cast(0x1000ULL); + return 0; + } + + extern "C" int RaTlvRequest( + void*, + uint32_t, + TileXR::TileXRCcuTlvMsg* sendMsg, + TileXR::TileXRCcuTlvMsg*) + { + return sendMsg != nullptr && sendMsg->type == TileXR::TILEXR_CCU_TLV_MSG_TYPE_CCU_INIT ? + 128308 : 0; + } + + extern "C" int RaTlvDeinit(void*) + { + return 0; + } + ''' + ) + runtime_code = textwrap.dedent( + r''' + #include + + extern "C" int rtGetDevicePhyIdByIndex(uint32_t logicDevId, uint32_t* phyId) + { + if (logicDevId != 7 || phyId == nullptr) { + return -1; + } + *phyId = 0xbeefU; + return 0; + } + + struct TileXRCcuRtProcExtParam { + const char *paramInfo; + uint64_t paramLen; + }; + + struct TileXRCcuRtNetServiceOpenArgs { + TileXRCcuRtProcExtParam *extParamList; + uint64_t extParamCnt; + }; + + extern "C" int rtOpenNetService(const TileXRCcuRtNetServiceOpenArgs*) + { + return 0; + } + + extern "C" int rtCloseNetService() + { + return 0; + } + ''' + ) + + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + libra_cpp = temp_path / "fake_libra.cpp" + runtime_cpp = temp_path / "fake_libruntime.cpp" + libra_so = temp_path / "libra.so" + runtime_so = temp_path / "libruntime.so" + libra_cpp.write_text(libra_code, encoding="utf-8") + runtime_cpp.write_text(runtime_code, encoding="utf-8") + common_flags = [ + compiler, + "-std=c++14", + "-shared", + "-fPIC", + "-I", + str(REPO_ROOT / "src" / "include"), + "-I", + str(REPO_ROOT / "src" / "comm"), + ] + subprocess.run( + [*common_flags, str(libra_cpp), "-o", str(libra_so)], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + subprocess.run( + [*common_flags, str(runtime_cpp), "-o", str(runtime_so)], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + DRIVER_SOURCE, + ], + extra_link_flags=["-ldl"], + extra_env={"LD_LIBRARY_PATH": str(temp_path)}, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_ccu_driver_adapter_keeps_dynamic_loading_in_ccu_hccp_loader(self): + combined = DRIVER_HEADER.read_text(encoding="utf-8") + "\n" + DRIVER_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "dlopen", + "dlsym", + "libra.so", + "libhcomm", + "libhccl_v2", + "#include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuHccpLoader loader; + TileXRCcuHccpLoaderReport report; + if (loader.LoadEndpointRouteProviderFromEnv(&report) != TILEXR_SUCCESS) { + std::cerr << "load provider failed: " << report.message << "\n"; + return 1; + } + if (loader.CollectLocalEndpointRoute == nullptr) { + std::cerr << "endpoint route provider function missing\n"; + return 2; + } + + TileXRCcuEndpointRouteProviderResourceWindow window; + window.addr = 0x10000000ULL; + window.bytes = 0x2000; + window.tokenId = 0x1234; + window.rawTokenId = 0x2234; + window.tokenValue = 0; + + TileXRCcuEndpointRouteProviderRoute route; + if (loader.CollectLocalEndpointRoute(0x55, &window, &route) != TILEXR_SUCCESS) { + std::cerr << "provider call failed\n"; + return 3; + } + if (!route.endpointRouteVerified || route.remoteEid[0] != 0xaa || + route.tpn != 0x10203 || route.doorbellVa != 0x1122334455667788ULL || + route.doorbellTokenId != 0x3456 || route.doorbellTokenValue != 0 || + route.sqDepth != 64) { + std::cerr << "route mismatch\n"; + return 4; + } + if (!report.endpointRouteProviderLoaded) { + std::cerr << "provider report did not record loaded state\n"; + return 5; + } + return 0; + } + ''' + ) + plugin_code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_hccp_loader.h" + #include "tilexr_types.h" + + #include + + using namespace TileXR; + + extern "C" int TileXRCcuCollectLocalEndpointRoute( + uint32_t devicePhyId, + const TileXRCcuEndpointRouteProviderResourceWindow* window, + TileXRCcuEndpointRouteProviderRoute* route) + { + if (devicePhyId != 0x55 || window == nullptr || route == nullptr || + window->addr != 0x10000000ULL || window->tokenId != 0x1234) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::memset(route, 0, sizeof(*route)); + route->remoteEid[0] = 0xaa; + route->tpn = 0x10203; + route->doorbellVa = 0x1122334455667788ULL; + route->doorbellTokenId = 0x3456; + route->doorbellTokenValue = 0; + route->sqDepth = 64; + route->endpointRouteVerified = true; + return TILEXR_SUCCESS; + } + ''' + ) + + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + plugin_cpp = temp_path / "tilexr_ccu_endpoint_route_provider.cpp" + plugin_so = temp_path / "tilexr_ccu_endpoint_route_provider.so" + plugin_cpp.write_text(plugin_code, encoding="utf-8") + plugin_build = subprocess.run( + [ + compiler, + "-std=c++14", + "-shared", + "-fPIC", + "-I", + str(REPO_ROOT / "src" / "include"), + "-I", + str(REPO_ROOT / "src" / "comm"), + str(plugin_cpp), + "-o", + str(plugin_so), + ], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + self.assertEqual(0, plugin_build.returncode, plugin_build.stdout + plugin_build.stderr) + result = self.compile_and_run( + code, + extra_sources=[LOADER_SOURCE], + extra_link_flags=["-ldl"], + extra_env={"TILEXR_CCU_ENDPOINT_ROUTE_PROVIDER": str(plugin_so)}, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_runtime_uses_optional_tilexr_endpoint_route_provider_before_env(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + #include + #include + + using namespace TileXR; + + int EchoLocalExchangeAsPeer(const void* sendBuf, size_t sendBytes, void* recvBuf, void*) + { + if (sendBuf == nullptr || sendBytes != sizeof(TileXRCcuResourceWindowExchange)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const auto* local = static_cast(sendBuf); + auto* out = static_cast(recvBuf); + std::memset(out, 0, sizeof(TileXRCcuResourceWindowExchange) * 2); + out[0] = *local; + return TILEXR_SUCCESS; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 1; + runtime.options_.rankSize = 2; + runtime.options_.allGather = EchoLocalExchangeAsPeer; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_SUCCESS) { + std::cerr << "provider route was not collected: " << report.message << "\n"; + return 1; + } + + std::vector buffers; + if (runtime.ExportRemoteCcuRmaBuffers(&buffers) != TILEXR_SUCCESS) { + std::cerr << "remote export failed\n"; + return 2; + } + if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || + buffers[0].remoteEid[0] != 0xaa || + buffers[0].tpn != 0x10203 || + buffers[0].doorbellVa != 0x1122334455667788ULL || + buffers[0].doorbellTokenId != 0x3456 || + buffers[0].doorbellTokenValue != 0 || + buffers[0].sqDepth != 64) { + std::cerr << "provider route was not exported\n"; + return 3; + } + return 0; + } + ''' + ) + plugin_code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_hccp_loader.h" + #include "tilexr_types.h" + + #include + + using namespace TileXR; + + extern "C" int TileXRCcuCollectLocalEndpointRoute( + uint32_t devicePhyId, + const TileXRCcuEndpointRouteProviderResourceWindow* window, + TileXRCcuEndpointRouteProviderRoute* route) + { + if (devicePhyId != 0x55 || window == nullptr || route == nullptr || + window->addr != 0x10000000ULL || window->tokenId != 0x1234) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::memset(route, 0, sizeof(*route)); + route->remoteEid[0] = 0xaa; + route->tpn = 0x10203; + route->doorbellVa = 0x1122334455667788ULL; + route->doorbellTokenId = 0x3456; + route->doorbellTokenValue = 0; + route->sqDepth = 64; + route->endpointRouteVerified = true; + return TILEXR_SUCCESS; + } + ''' + ) + + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + plugin_cpp = temp_path / "tilexr_ccu_endpoint_route_provider.cpp" + plugin_so = temp_path / "tilexr_ccu_endpoint_route_provider.so" + plugin_cpp.write_text(plugin_code, encoding="utf-8") + plugin_build = subprocess.run( + [ + compiler, + "-std=c++14", + "-shared", + "-fPIC", + "-I", + str(REPO_ROOT / "src" / "include"), + "-I", + str(REPO_ROOT / "src" / "comm"), + str(plugin_cpp), + "-o", + str(plugin_so), + ], + cwd=REPO_ROOT, + check=False, + text=True, + capture_output=True, + ) + self.assertEqual(0, plugin_build.returncode, plugin_build.stdout + plugin_build.stderr) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp", + ], + extra_link_flags=["-ldl"], + extra_env={"TILEXR_CCU_ENDPOINT_ROUTE_PROVIDER": str(plugin_so)}, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_runtime_collects_ra_ctx_loop_endpoint_route_before_env_fallback(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE", source) + self.assertIn("ra_ctx_loop", source) + self.assertIn("CollectLocalEndpointRouteWithRaCtx(&route)", source) + + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + #include + #include + + using namespace TileXR; + + constexpr uint32_t kRawDbTokenId = 0x345600U; + constexpr uint32_t kTokenValue = 0x7799U; + constexpr uint64_t kTpHandle = 0xabcddcbaULL; + constexpr uint64_t kCcuResourceBase = 0x10000000ULL; + constexpr uint64_t kHcommWqeBasicBlockOffset = 0x1000000ULL; + constexpr uint64_t kHcommSqBufferSize = 256ULL * 1024ULL; + constexpr uint32_t kDirectLoopJettyCtxId = 0; + constexpr uint64_t kExpectedSqVa = + kCcuResourceBase + kHcommWqeBasicBlockOffset + + static_cast(kDirectLoopJettyCtxId) * kHcommSqBufferSize; + + int g_cqCreateCalls = 0; + int g_qpCreateCalls = 0; + int g_tpInfoCalls = 0; + int g_reqResultCalls = 0; + int g_qpImportCalls = 0; + int g_qpBindCalls = 0; + int g_qpUnbindCalls = 0; + int g_qpUnimportCalls = 0; + int g_qpDestroyCalls = 0; + int g_cqDestroyCalls = 0; + TileXRCcuHccpQpCreateAttr g_lastQpAttr {}; + TileXRCcuHccpGetTpCfg g_lastTpCfg {}; + TileXRCcuHccpQpImportInfo g_lastImportInfo {}; + + int FakeCqCreate(void* ctx, TileXRCcuHccpCqInfo* info, void** cqHandle) + { + if (ctx != reinterpret_cast(0x1000) || info == nullptr || cqHandle == nullptr) { + return -1; + } + if (info->in.depth != 64 || info->in.ub.mode != 2) { + return -2; + } + ++g_cqCreateCalls; + *cqHandle = reinterpret_cast(0x2000); + return 0; + } + + int FakeCqDestroy(void*, void* cqHandle) + { + if (cqHandle == reinterpret_cast(0x2000)) { + ++g_cqDestroyCalls; + } + return 0; + } + + int FakeQpCreate( + void* ctx, + TileXRCcuHccpQpCreateAttr* attr, + TileXRCcuHccpQpCreateInfo* info, + void** qpHandle) + { + if (ctx != reinterpret_cast(0x1000) || attr == nullptr || info == nullptr || + qpHandle == nullptr) { + return -3; + } + g_lastQpAttr = *attr; + if (attr->scqHandle != reinterpret_cast(0x2000) || + attr->rcqHandle != reinterpret_cast(0x2000) || + attr->sqDepth != 8 || + attr->rqDepth != TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT || + attr->transportMode != TILEXR_CCU_HCCP_TRANSPORT_MODE_RM || + attr->ub.mode != static_cast(TILEXR_CCU_HCCP_JETTY_MODE_CCU) || + attr->ub.jettyId != 1024 || + attr->ub.tokenIdHandle != reinterpret_cast(0x3000) || + attr->ub.tokenValue != kTokenValue || + attr->ub.flag.bs.shareJfr != 1 || + attr->ub.jfsFlag.bs.errorSuspend != 1 || + attr->ub.extMode.cstmFlag.bs.sqCstm != 1 || + attr->ub.extMode.sqebbNum != 8 || + attr->ub.extMode.sq.buffVa != kExpectedSqVa || + attr->ub.extMode.sq.buffSize != 8 * 4 * 64) { + return -4; + } + for (uint32_t i = 0; i < TILEXR_CCU_HCCP_QP_KEY_BYTES; ++i) { + info->key.value[i] = static_cast(0x80U + i); + } + info->key.size = TILEXR_CCU_HCCP_QP_KEY_BYTES; + info->ub.dbAddr = 0x1122334455667788ULL; + info->ub.dbTokenId = kRawDbTokenId; + ++g_qpCreateCalls; + *qpHandle = reinterpret_cast(0x4000); + return 0; + } + + int FakeQpDestroy(void* qpHandle) + { + if (qpHandle == reinterpret_cast(0x4000)) { + ++g_qpDestroyCalls; + } + return 0; + } + + int FakeGetTpInfoListAsync( + void* ctx, + TileXRCcuHccpGetTpCfg* cfg, + TileXRCcuHccpTpInfo infoList[], + uint32_t* num, + void** reqHandle) + { + if (ctx != reinterpret_cast(0x1000) || cfg == nullptr || infoList == nullptr || + num == nullptr || reqHandle == nullptr || *num != 1) { + return -5; + } + g_lastTpCfg = *cfg; + if (cfg->flag.bs.rtp != 1 || cfg->transMode != TILEXR_CCU_HCCP_TRANSPORT_MODE_RM || + std::memcmp(cfg->localEid.raw, cfg->peerEid.raw, TILEXR_CCU_EID_BYTES) != 0) { + return -6; + } + infoList[0].tpHandle = kTpHandle; + ++g_tpInfoCalls; + *reqHandle = reinterpret_cast(0x5000); + return 0; + } + + int FakeGetAsyncReqResult(void* reqHandle, int* reqResult) + { + if (reqHandle != reinterpret_cast(0x5000) || reqResult == nullptr) { + return -7; + } + *reqResult = 0; + ++g_reqResultCalls; + return 0; + } + + int FakeQpImport(void* ctx, TileXRCcuHccpQpImportInfo* info, void** remoteQpHandle) + { + if (ctx != reinterpret_cast(0x1000) || info == nullptr || remoteQpHandle == nullptr) { + return -8; + } + g_lastImportInfo = *info; + if (info->in.key.size != TILEXR_CCU_HCCP_QP_KEY_BYTES || + info->in.key.value[0] != 0x80 || + info->in.ub.mode != TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP || + info->in.ub.tokenValue != kTokenValue || + info->in.ub.policy != TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR || + info->in.ub.type != TILEXR_CCU_HCCP_TARGET_TYPE_JETTY || + info->in.ub.flag.bs.tokenPolicy != TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT || + info->in.ub.expImportCfg.tpHandle != kTpHandle || + info->in.ub.expImportCfg.peerTpHandle != kTpHandle || + info->in.ub.expImportCfg.txPsn == 0 || + info->in.ub.expImportCfg.rxPsn == 0 || + info->in.ub.tpType != TILEXR_CCU_HCCP_TP_TYPE_RTP) { + return -9; + } + info->out.ub.tpn = 0; + ++g_qpImportCalls; + *remoteQpHandle = reinterpret_cast(0x6000); + return 0; + } + + int FakeQpUnimport(void*, void* remoteQpHandle) + { + if (remoteQpHandle == reinterpret_cast(0x6000)) { + ++g_qpUnimportCalls; + } + return 0; + } + + int FakeQpBind(void* qpHandle, void* remoteQpHandle) + { + if (qpHandle != reinterpret_cast(0x4000) || + remoteQpHandle != reinterpret_cast(0x6000)) { + return -10; + } + ++g_qpBindCalls; + return -11; + } + + int FakeQpUnbind(void* qpHandle) + { + if (qpHandle == reinterpret_cast(0x4000)) { + ++g_qpUnbindCalls; + } + return 0; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 0; + runtime.options_.rankSize = 1; + runtime.localResourceWindow_.addr = kCcuResourceBase; + runtime.localResourceWindow_.bytes = TILEXR_CCU_RESOURCE_WINDOW_BYTES; + runtime.localResourceWindow_.tokenId = 0x3456; + runtime.localResourceWindow_.rawTokenId = 0x345600; + runtime.localResourceWindow_.tokenValue = kTokenValue; + runtime.localResourceWindow_.raCtxHandle = reinterpret_cast(0x1000); + runtime.localResourceWindow_.tokenIdHandle = reinterpret_cast(0x3000); + runtime.localResourceWindow_.raCtxRegistered = true; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + runtime.localResourceWindow_.eid[i] = static_cast(0xa0U + i); + } + + runtime.loader_.RaCtxCqCreate = FakeCqCreate; + runtime.loader_.RaCtxCqDestroy = FakeCqDestroy; + runtime.loader_.RaCtxQpCreate = FakeQpCreate; + runtime.loader_.RaCtxQpDestroy = FakeQpDestroy; + runtime.loader_.RaGetTpInfoListAsync = FakeGetTpInfoListAsync; + runtime.loader_.RaGetAsyncReqResult = FakeGetAsyncReqResult; + runtime.loader_.RaCtxQpImport = FakeQpImport; + runtime.loader_.RaCtxQpUnimport = FakeQpUnimport; + runtime.loader_.RaCtxQpBind = FakeQpBind; + runtime.loader_.RaCtxQpUnbind = FakeQpUnbind; + + TileXRCcuDirectRuntimeReport report; + if (runtime.RefreshLocalVerifiedEndpointRoute(&report) != TILEXR_SUCCESS) { + std::cerr << "ra ctx loop route collection failed: " << report.message << "\n"; + return 1; + } + if (!runtime.localVerifiedEndpointRouteValid_) { + std::cerr << "verified route flag not set\n"; + return 2; + } + const auto& route = runtime.localVerifiedEndpointRoute_; + if (!route.endpointRouteVerified || + route.remoteEid[0] != 0xa0 || + route.tpn != 0 || + route.doorbellVa != 0x1122334455667788ULL || + route.doorbellTokenId != (kRawDbTokenId >> 8) || + route.doorbellTokenValue != kTokenValue || + route.sqDepth != 8) { + std::cerr << "verified route mismatch\n"; + return 3; + } + if (g_cqCreateCalls != 1 || g_qpCreateCalls != 1 || g_tpInfoCalls != 1 || + g_reqResultCalls != 1 || g_qpImportCalls != 1 || g_qpBindCalls != 0) { + std::cerr << "unexpected RA call counts\n"; + return 4; + } + + runtime.Shutdown(); + if (g_qpUnbindCalls != 0 || g_qpUnimportCalls != 1 || + g_qpDestroyCalls != 1 || g_cqDestroyCalls != 1) { + std::cerr << "endpoint route handles were not released\n"; + return 5; + } + return 0; + } + ''' + ) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp", + ], + extra_link_flags=["-ldl"], + extra_env={ + "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE": "ra_ctx_loop", + }, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_runtime_retries_ra_ctx_endpoint_route_after_async_result_failure(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + #include + #include + + using namespace TileXR; + + constexpr uint32_t kRawDbTokenId = 0x456700U; + constexpr uint32_t kTokenValue = 0x8811U; + constexpr uint64_t kCcuResourceBase = 0x11000000ULL; + + int g_cqCreateCalls = 0; + int g_qpCreateCalls = 0; + int g_tpInfoCalls = 0; + int g_reqResultCalls = 0; + int g_qpImportCalls = 0; + int g_qpDestroyCalls = 0; + int g_cqDestroyCalls = 0; + + int FakeCqCreate(void*, TileXRCcuHccpCqInfo*, void** cqHandle) + { + if (cqHandle == nullptr) { + return -1; + } + ++g_cqCreateCalls; + *cqHandle = reinterpret_cast(0x2000 + g_cqCreateCalls); + return 0; + } + + int FakeCqDestroy(void*, void* cqHandle) + { + if (cqHandle != nullptr) { + ++g_cqDestroyCalls; + } + return 0; + } + + int FakeQpCreate( + void*, + TileXRCcuHccpQpCreateAttr*, + TileXRCcuHccpQpCreateInfo* info, + void** qpHandle) + { + if (info == nullptr || qpHandle == nullptr) { + return -2; + } + ++g_qpCreateCalls; + for (uint32_t i = 0; i < TILEXR_CCU_HCCP_QP_KEY_BYTES; ++i) { + info->key.value[i] = static_cast(0x40U + i); + } + info->key.size = TILEXR_CCU_HCCP_QP_KEY_BYTES; + info->ub.dbAddr = 0x1222333444555666ULL; + info->ub.dbTokenId = kRawDbTokenId; + *qpHandle = reinterpret_cast(0x4000 + g_qpCreateCalls); + return 0; + } + + int FakeQpDestroy(void* qpHandle) + { + if (qpHandle != nullptr) { + ++g_qpDestroyCalls; + } + return 0; + } + + int FakeGetTpInfoListAsync( + void*, + TileXRCcuHccpGetTpCfg*, + TileXRCcuHccpTpInfo infoList[], + uint32_t* num, + void** reqHandle) + { + if (infoList == nullptr || num == nullptr || reqHandle == nullptr || *num != 1) { + return -3; + } + ++g_tpInfoCalls; + infoList[0].tpHandle = 0xabcd0000ULL + g_tpInfoCalls; + *reqHandle = reinterpret_cast(0x5000 + g_tpInfoCalls); + return 0; + } + + int FakeGetAsyncReqResult(void*, int* reqResult) + { + if (reqResult == nullptr) { + return -4; + } + ++g_reqResultCalls; + *reqResult = (g_reqResultCalls <= 3) ? 528101 : 0; + return 0; + } + + int FakeQpImport(void*, TileXRCcuHccpQpImportInfo* info, void** remoteQpHandle) + { + if (info == nullptr || remoteQpHandle == nullptr) { + return -5; + } + ++g_qpImportCalls; + info->out.ub.tpn = 0x7000 + g_qpImportCalls; + *remoteQpHandle = reinterpret_cast(0x6000 + g_qpImportCalls); + return 0; + } + + int FakeQpUnimport(void*, void*) + { + return 0; + } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 0; + runtime.options_.rankSize = 1; + runtime.localResourceWindow_.addr = kCcuResourceBase; + runtime.localResourceWindow_.bytes = TILEXR_CCU_RESOURCE_WINDOW_BYTES; + runtime.localResourceWindow_.tokenId = 0x4567; + runtime.localResourceWindow_.rawTokenId = kRawDbTokenId; + runtime.localResourceWindow_.tokenValue = kTokenValue; + runtime.localResourceWindow_.raCtxHandle = reinterpret_cast(0x1000); + runtime.localResourceWindow_.tokenIdHandle = reinterpret_cast(0x3000); + runtime.localResourceWindow_.raCtxRegistered = true; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + runtime.localResourceWindow_.eid[i] = static_cast(0xb0U + i); + } + + runtime.loader_.RaCtxCqCreate = FakeCqCreate; + runtime.loader_.RaCtxCqDestroy = FakeCqDestroy; + runtime.loader_.RaCtxQpCreate = FakeQpCreate; + runtime.loader_.RaCtxQpDestroy = FakeQpDestroy; + runtime.loader_.RaGetTpInfoListAsync = FakeGetTpInfoListAsync; + runtime.loader_.RaGetAsyncReqResult = FakeGetAsyncReqResult; + runtime.loader_.RaCtxQpImport = FakeQpImport; + runtime.loader_.RaCtxQpUnimport = FakeQpUnimport; + + TileXRCcuDirectRuntimeReport report; + const int ret = runtime.RefreshLocalVerifiedEndpointRoute(&report); + if (ret != TILEXR_SUCCESS) { + std::cerr << "ra ctx endpoint route retry failed: " << report.message + << " ret=" << ret << "\n"; + return 1; + } + if (!runtime.localVerifiedEndpointRouteValid_) { + std::cerr << "verified endpoint route flag not set after retry\n"; + return 2; + } + const auto& route = runtime.localVerifiedEndpointRoute_; + if (!route.endpointRouteVerified || + route.remoteEid[0] != 0xb0 || + route.doorbellVa != 0x1222333444555666ULL || + route.doorbellTokenId != (kRawDbTokenId >> 8) || + route.doorbellTokenValue != kTokenValue || + route.sqDepth != 8) { + std::cerr << "verified endpoint route mismatch after retry\n"; + return 3; + } + if (g_cqCreateCalls != 4 || g_qpCreateCalls != 4 || g_tpInfoCalls != 4 || + g_reqResultCalls != 4 || g_qpImportCalls != 1) { + std::cerr << "unexpected retry call counts" + << " cqCreate=" << g_cqCreateCalls + << " qpCreate=" << g_qpCreateCalls + << " tpInfo=" << g_tpInfoCalls + << " reqResult=" << g_reqResultCalls + << " qpImport=" << g_qpImportCalls << "\n"; + return 4; + } + if (g_qpDestroyCalls != 3 || g_cqDestroyCalls != 3) { + std::cerr << "failed attempt was not cleaned up before retry" + << " qpDestroy=" << g_qpDestroyCalls + << " cqDestroy=" << g_cqDestroyCalls << "\n"; + return 5; + } + + runtime.Shutdown(); + if (g_qpDestroyCalls != 4 || g_cqDestroyCalls != 4) { + std::cerr << "successful retry handles were not released on shutdown" + << " qpDestroy=" << g_qpDestroyCalls + << " cqDestroy=" << g_cqDestroyCalls << "\n"; + return 6; + } + return 0; + } + ''' + ) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp", + ], + extra_link_flags=["-ldl"], + extra_env={ + "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE": "ra_ctx_loop", + }, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_direct_runtime_fails_closed_when_configured_endpoint_route_provider_cannot_load(self): + code = textwrap.dedent( + r''' + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.resourceWindowRegistered_ = true; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 1; + runtime.options_.rankSize = 2; + runtime.localResourceWindow_.addr = 0x10000000ULL; + runtime.localResourceWindow_.bytes = 0x2000; + runtime.localResourceWindow_.tokenId = 0x1234; + runtime.localResourceWindow_.rawTokenId = 0x2234; + runtime.localResourceWindow_.tokenValue = 0; + + TileXRCcuDirectRuntimeReport report; + const int ret = runtime.RefreshLocalVerifiedEndpointRoute(&report); + if (ret == TILEXR_SUCCESS) { + std::cerr << "configured bad provider fell back to env route\n"; + return 1; + } + if (runtime.localVerifiedEndpointRouteValid_) { + std::cerr << "bad provider left a verified route\n"; + return 2; + } + if (report.message.find("failed to load direct CCU endpoint route provider") == std::string::npos) { + std::cerr << "unexpected failure message: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + result = self.compile_and_run( + code, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + LOADER_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp", + ], + extra_link_flags=["-ldl"], + extra_env={ + "TILEXR_CCU_ENDPOINT_ROUTE_PROVIDER": "/tmp/tilexr_missing_endpoint_route_provider.so", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID_RANK1": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_TPN_RANK1": "0x10203", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA_RANK1": "0x1122334455667788", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_ID_RANK1": "0x3456", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_TOKEN_VALUE_RANK1": "0", + "TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_SQ_DEPTH_RANK1": "64", + }, + ) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py new file mode 100644 index 00000000..6052edf4 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PROVIDER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.h" +PROVIDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp" +DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" +SPECS_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuRaCustomChannelProviderTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "ra_provider_test.cpp" + test_bin = temp_path / "ra_provider_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(PROVIDER_SOURCE), + str(DRIVER_SOURCE), + str(SPECS_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_provider_adapts_ra_custom_channel_to_driver_adapter(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_ra_custom_channel_provider.h" + + #include + + using namespace TileXR; + + void* g_tilexrCcuRaProviderTestState = nullptr; + + struct FakeRaState { + uint32_t phyId = 0; + uint32_t mode = 0; + uint32_t op = 0; + uint32_t die = 0; + }; + + int FakeRaCustomChannel( + TileXRCcuRaInfo info, + TileXRCcuCustomChannelIn* in, + TileXRCcuCustomChannelOut* out) + { + auto* state = static_cast(g_tilexrCcuRaProviderTestState); + state->phyId = info.phyId; + state->mode = info.mode; + state->op = in->op; + state->die = in->data.dataInfo.udieIdx; + out->opRet = 0; + out->data.dataInfo.dataArray[0].baseinfo.msId = 0x66; + out->data.dataInfo.dataArray[0].baseinfo.missionKey = 0x0badcafeU; + out->data.dataInfo.dataArray[0].baseinfo.resourceAddr = 0x300000000ULL; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap0 = (1U << 24) | (2U << 16) | 31U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap1 = (15U << 16) | 7U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap2 = (3U << 16) | 5U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap3 = (9U << 16) | 1U; + return 0; + } + + int main() + { + FakeRaState state; + g_tilexrCcuRaProviderTestState = &state; + + TileXRCcuRaCustomChannelProvider provider; + TileXRCcuRaCustomChannelProviderReport providerReport; + if (provider.Init(9, FakeRaCustomChannel, &providerReport) != TILEXR_SUCCESS) { + std::cerr << "provider init failed: " << providerReport.message << "\n"; + return 1; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + if (provider.CreateAdapter(&adapter, &adapterReport) != TILEXR_SUCCESS) { + std::cerr << "create adapter failed: " << adapterReport.message << "\n"; + return 2; + } + + TileXRCcuBasicInfo basic; + if (adapter.GetBasicInfo(1, &basic, &adapterReport) != TILEXR_SUCCESS) { + std::cerr << "get basic info failed: " << adapterReport.message << "\n"; + return 3; + } + if (state.phyId != 9 || state.mode != TILEXR_CCU_NETWORK_OFFLINE || + state.op != TILEXR_CCU_U_OP_GET_BASIC_INFO || state.die != 1) { + std::cerr << "RA call mismatch\n"; + return 4; + } + if (basic.missionKey != 0x0badcafeU || basic.resourceAddr != 0x300000000ULL || + basic.msId != 0x66) { + std::cerr << "basic info mismatch\n"; + return 5; + } + if (providerReport.message != "ok" || providerReport.devicePhyId != 9) { + std::cerr << "provider report mismatch\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_accepts_opaque_ra_custom_channel_c_abi_shape(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_ra_custom_channel_provider.h" + + #include + + using namespace TileXR; + + void* g_tilexrCcuRaProviderTestState = nullptr; + + struct LegacyRaInfo { + int mode = 0; + uint32_t phyId = 0; + }; + + struct FakeRaState { + uint32_t phyId = 0; + uint32_t mode = 0; + uint32_t op = 0; + uint32_t die = 0; + }; + + int FakeOpaqueRaCustomChannel(LegacyRaInfo info, void* rawIn, void* rawOut) + { + auto* state = static_cast(g_tilexrCcuRaProviderTestState); + auto* in = static_cast(rawIn); + auto* out = static_cast(rawOut); + state->phyId = info.phyId; + state->mode = info.mode; + state->op = in->op; + state->die = in->data.dataInfo.udieIdx; + out->opRet = 0; + out->data.dataInfo.dataArray[0].baseinfo.msId = 0x77; + out->data.dataInfo.dataArray[0].baseinfo.missionKey = 0x12345678U; + out->data.dataInfo.dataArray[0].baseinfo.resourceAddr = 0x400000000ULL; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap0 = (1U << 24) | (2U << 16) | 31U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap1 = (15U << 16) | 7U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap2 = (3U << 16) | 5U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap3 = (9U << 16) | 1U; + return 0; + } + + int main() + { + FakeRaState state; + g_tilexrCcuRaProviderTestState = &state; + + TileXRCcuRaCustomChannelProvider provider; + TileXRCcuRaCustomChannelProviderReport providerReport; + if (provider.Init(13, FakeOpaqueRaCustomChannel, &providerReport) != TILEXR_SUCCESS) { + std::cerr << "provider init failed: " << providerReport.message << "\n"; + return 1; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + if (provider.CreateAdapter(&adapter, &adapterReport) != TILEXR_SUCCESS) { + std::cerr << "create adapter failed: " << adapterReport.message << "\n"; + return 2; + } + + TileXRCcuBasicInfo basic; + if (adapter.GetBasicInfo(2, &basic, &adapterReport) != TILEXR_SUCCESS) { + std::cerr << "get basic info failed: " << adapterReport.message << "\n"; + return 3; + } + if (state.phyId != 13 || state.mode != TILEXR_CCU_NETWORK_OFFLINE || + state.op != TILEXR_CCU_U_OP_GET_BASIC_INFO || state.die != 2) { + std::cerr << "RA call mismatch\n"; + return 4; + } + if (basic.missionKey != 0x12345678U || basic.resourceAddr != 0x400000000ULL || + basic.msId != 0x77) { + std::cerr << "basic info mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_provider_is_wired_and_keeps_hcomm_runtime_out_of_ccu_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = PROVIDER_HEADER.read_text(encoding="utf-8") + source = PROVIDER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_ra_custom_channel_provider.h", cmake) + self.assertIn("ccu/tilexr_ccu_ra_custom_channel_provider.cpp", cmake) + self.assertIn("TileXRCcuRaCustomChannelProvider", header) + self.assertIn("CreateAdapter", header) + self.assertIn("TileXRCcuRaCustomChannelFunc", header) + self.assertIn("std::function", header) + self.assertIn("TILEXR_CCU_NETWORK_OFFLINE", source) + self.assertIn("TileXRCcuDriverAdapter", header) + self.assertNotIn("udma/", header) + + combined = header + "\n" + source + for needle in [ + "#include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << planReport.message << "\n"; + return 1; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + if (TileXRCcuBuildRepositoryImage(plan, program, &image, &report) != TILEXR_SUCCESS) { + std::cerr << "repository image build failed: " << report.message << "\n"; + return 2; + } + + if (image.repositoryStartId != 475 || image.repositoryCount != 170 || + image.missionStartId != 489 || image.missionCount != 156 || + image.missionOffset != 14 || image.sqeLoadOffset != 14 || + image.sqeLoadCount != 13 || image.syncOffset != 27 || image.syncCount != 11) { + std::cerr << "unexpected repository metadata\n"; + return 3; + } + if (image.instructions.size() != 170) { + std::cerr << "unexpected repository image size\n"; + return 4; + } + if (image.instructions[14].words[0] != 0x0000000007a90001ULL || + image.instructions[26].words[0] != 0x0000000c07b50001ULL || + image.instructions[27].words[0] != 0x0000000007a90001ULL || + image.instructions[28].words[0] != 0x0000000107aa0001ULL || + image.instructions[29].words[0] != 0x0000000007b60003ULL || + image.instructions[30].words[0] != 0x0000000007b60003ULL || + image.instructions[31].words[0] != 0x0001016c00000802ULL || + image.instructions[31].words[1] != 0 || + image.instructions[32].words[0] != 0x000007a90939100dULL || + image.instructions[32].words[1] != 0x00000001016c0002ULL || + image.instructions[32].words[2] != 0x0001000000000000ULL || + image.instructions[35].words[0] != 0x0000000000010802ULL || + image.instructions[35].words[1] != 0x000000000001016cULL) { + std::cerr << "unexpected installed instructions\n"; + return 5; + } + TileXRCcuInstr expectedNop; + if (TileXRCcuEncodeLoadImdToXn(plan.kernelLocalXn.startId, 0, 0, &expectedNop) != + TILEXR_SUCCESS) { + std::cerr << "failed to encode expected repository padding nop\n"; + return 6; + } + if (image.instructions[13].words[0] != expectedNop.words[0] || + image.instructions[13].words[1] != expectedNop.words[1] || + image.instructions[13].words[2] != expectedNop.words[2] || + image.instructions[13].words[3] != expectedNop.words[3] || + image.instructions[38].words[0] != expectedNop.words[0] || + image.instructions[38].words[1] != expectedNop.words[1] || + image.instructions[38].words[2] != expectedNop.words[2] || + image.instructions[38].words[3] != expectedNop.words[3]) { + std::cerr << "unused repository slots should contain valid nop padding\n"; + return 6; + } + if (report.repositoryCount != 170 || report.installedInstructionCount != 24 || + report.sqeLoadOffset != 14 || report.syncOffset != 27 || report.message != "ok") { + std::cerr << "unexpected repository report\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_image_uploads_mission_window_and_installs_via_driver_adapter(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_repository.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + uint64_t allocBytes = 0; + uint64_t copiedBytes = 0; + bool freed = false; + uint32_t observedOp = 0; + uint32_t observedOffset = 0; + uint32_t observedDataLen = 0; + uint64_t observedResourceAddr = 0; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->allocBytes = bytes; + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + state->copiedBytes = bytes; + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->observedOp = in.op; + state->observedOffset = in.offsetStartIdx; + state->observedDataLen = in.data.dataInfo.dataLen; + state->observedResourceAddr = in.data.dataInfo.dataArray[0].insinfo.resourceAddr; + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuProducerPlan plan = MakePlan(); + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << planReport.message << "\n"; + return 1; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + if (TileXRCcuBuildRepositoryImage(plan, program, &image, &report) != TILEXR_SUCCESS) { + std::cerr << "repository build failed: " << report.message << "\n"; + return 2; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 3; + } + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + + TileXRCcuRepositoryInstallReceipt receipt; + if (TileXRCcuInstallRepositoryImage( + image, memoryOps, &state, adapter, &receipt, &report) != TILEXR_SUCCESS) { + std::cerr << "repository install failed: " << report.message << "\n"; + return 4; + } + + const uint64_t expectedBytes = 156ULL * sizeof(TileXRCcuInstr); + if (state.allocBytes != expectedBytes || state.copiedBytes != expectedBytes || + receipt.instructionStartId != 489 || receipt.instructionCount != 156 || + receipt.instructionBytes != expectedBytes || !receipt.uploaded || !receipt.installed) { + std::cerr << "upload receipt mismatch\n"; + return 5; + } + if (state.observedOp != TILEXR_CCU_U_OP_SET_INSTRUCTION || + state.observedOffset != 489 || state.observedDataLen != expectedBytes || + state.observedResourceAddr != receipt.deviceInstructionAddr) { + std::cerr << "install adapter request mismatch\n"; + return 6; + } + + const auto* installed = reinterpret_cast(state.deviceBytes.data()); + if (installed[0].words[0] != image.instructions[image.missionOffset].words[0] || + installed[12].words[0] != image.instructions[image.missionOffset + 12].words[0] || + installed[13].words[0] != image.instructions[image.syncOffset].words[0]) { + std::cerr << "mission-window upload content mismatch\n"; + return 7; + } + if (report.installedInstructionCount != 156 || !report.repositoryInstalled || + report.message != "ok") { + std::cerr << "install report mismatch\n"; + return 8; + } + + if (TileXRCcuReleaseRepositoryInstallReceipt(receipt, memoryOps, &state, &report) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "release failed: " << report.message << "\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_install_options_can_upload_full_repository_with_descriptor_len(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_repository.h" + + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + uint64_t allocBytes = 0; + uint64_t copiedBytes = 0; + std::vector deviceBytes; + uint32_t observedOffset = 0; + uint32_t observedDataLen = 0; + uint64_t observedResourceAddr = 0; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->allocBytes = bytes; + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size()) { + return -1; + } + state->copiedBytes = bytes; + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->observedOffset = in.offsetStartIdx; + state->observedDataLen = in.data.dataInfo.dataLen; + state->observedResourceAddr = in.data.dataInfo.dataArray[0].insinfo.resourceAddr; + out->opRet = 0; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuProducerPlan plan = MakePlan(); + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << planReport.message << "\n"; + return 1; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + if (TileXRCcuBuildRepositoryImage(plan, program, &image, &report) != TILEXR_SUCCESS) { + std::cerr << "repository build failed: " << report.message << "\n"; + return 2; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 3; + } + + TileXRCcuRepositoryInstallOptions options; + options.window = TileXRCcuRepositoryInstallWindow::FullRepository; + options.dataLenMode = TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes; + + TileXRCcuDeviceMemoryOps memoryOps = {FakeAlloc, FakeCopy, FakeFree}; + TileXRCcuRepositoryInstallReceipt receipt; + if (TileXRCcuInstallRepositoryImageWithOptions( + image, options, memoryOps, &state, adapter, &receipt, &report) != TILEXR_SUCCESS) { + std::cerr << "repository install failed: " << report.message << "\n"; + return 4; + } + + const uint64_t expectedBytes = 170ULL * sizeof(TileXRCcuInstr); + if (state.allocBytes != expectedBytes || state.copiedBytes != expectedBytes || + receipt.instructionStartId != 475 || receipt.instructionCount != 170 || + receipt.instructionBytes != expectedBytes || !receipt.uploaded || !receipt.installed) { + std::cerr << "full repository receipt mismatch\n"; + return 5; + } + if (state.observedOffset != 475 || + state.observedDataLen != sizeof(TileXRCcuInstrInfo) || + state.observedResourceAddr != receipt.deviceInstructionAddr) { + std::cerr << "full repository SET_INSTRUCTION envelope mismatch\n"; + return 6; + } + const auto* installed = reinterpret_cast(state.deviceBytes.data()); + if (installed[0].words[0] != image.instructions[0].words[0] || + installed[14].words[0] != image.instructions[image.missionOffset].words[0]) { + std::cerr << "full repository upload content mismatch\n"; + return 7; + } + if (report.installedInstructionCount != 170 || + report.message.find("window=full_repository") == std::string::npos || + report.message.find("dataLenMode=descriptor_bytes") == std::string::npos) { + std::cerr << "install report missing option detail: " << report.message << "\n"; + return 8; + } + if (TileXRCcuReleaseRepositoryInstallReceipt(receipt, memoryOps, &state, &report) != + TILEXR_SUCCESS || !state.freed) { + std::cerr << "release failed: " << report.message << "\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_install_failure_reports_full_set_instruction_context(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_repository.h" + + #include + #include + #include + #include + + using namespace TileXR; + + struct FakeState { + std::vector deviceBytes; + uint64_t readbackBytes = 0; + uint64_t driverReadbackBytes = 0; + bool freed = false; + }; + + int FakeAlloc(uint64_t bytes, void** ptr, void* userData) + { + auto* state = static_cast(userData); + state->deviceBytes.assign(static_cast(bytes), 0); + *ptr = state->deviceBytes.data(); + return 0; + } + + int FakeCopy(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (dst != state->deviceBytes.data() || dstBytes != state->deviceBytes.size() || + bytes != dstBytes) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + return 0; + } + + int FakeCopyDeviceToHost(void* dst, uint64_t dstBytes, const void* src, uint64_t bytes, void* userData) + { + auto* state = static_cast(userData); + if (src != state->deviceBytes.data() || dstBytes < bytes || + bytes != state->deviceBytes.size()) { + return -1; + } + std::memcpy(dst, src, static_cast(bytes)); + state->readbackBytes = bytes; + return 0; + } + + int FakeFree(void* ptr, void* userData) + { + auto* state = static_cast(userData); + if (ptr != state->deviceBytes.data()) { + return -1; + } + state->freed = true; + state->deviceBytes.clear(); + return 0; + } + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + out->opRet = 0; + if (in.op == TILEXR_CCU_U_OP_SET_INSTRUCTION) { + return 328107; + } + if (in.op != TILEXR_CCU_U_OP_GET_INSTRUCTION) { + return -1; + } + auto* state = static_cast(userData); + constexpr uint32_t kBaseInstruction = 475; + const uint32_t count = in.data.dataInfo.dataArraySize; + if (count == 0 || count > TILEXR_CCU_MAX_DATA_ARRAY_SIZE || + in.offsetStartIdx < kBaseInstruction || + in.data.dataInfo.dataLen != count * TILEXR_CCU_INSTRUCTION_BYTES) { + return -2; + } + const uint64_t byteOffset = + static_cast(in.offsetStartIdx - kBaseInstruction) * + TILEXR_CCU_INSTRUCTION_BYTES; + const uint64_t bytes = static_cast(count) * TILEXR_CCU_INSTRUCTION_BYTES; + if (byteOffset + bytes > state->deviceBytes.size()) { + return -3; + } + for (uint32_t i = 0; i < count; ++i) { + std::memcpy( + out->data.dataInfo.dataArray[i].byte32.raw, + state->deviceBytes.data() + byteOffset + + static_cast(i) * TILEXR_CCU_INSTRUCTION_BYTES, + TILEXR_CCU_INSTRUCTION_BYTES); + } + out->data.dataInfo.dataArraySize = count; + out->data.dataInfo.dataLen = static_cast(bytes); + out->offsetNextIdx = in.offsetStartIdx + count; + state->driverReadbackBytes += bytes; + return 0; + } + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {0x100051152e00ULL}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + bool Has(const std::string& haystack, const std::string& needle) + { + return haystack.find(needle) != std::string::npos; + } + + int main() + { + TileXRCcuProducerPlan plan = MakePlan(); + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << planReport.message << "\n"; + return 1; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + if (TileXRCcuBuildRepositoryImage(plan, program, &image, &report) != TILEXR_SUCCESS) { + std::cerr << "repository build failed: " << report.message << "\n"; + return 2; + } + + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport driverReport; + if (adapter.Init(5, FakeCustomChannel, &state, &driverReport) != TILEXR_SUCCESS) { + std::cerr << "adapter init failed\n"; + return 3; + } + + TileXRCcuRepositoryInstallOptions options; + options.window = TileXRCcuRepositoryInstallWindow::FullRepository; + options.dataLenMode = TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; + + TileXRCcuDeviceMemoryOps memoryOps; + memoryOps.alloc = FakeAlloc; + memoryOps.copyHostToDevice = FakeCopy; + memoryOps.free = FakeFree; + memoryOps.copyDeviceToHost = FakeCopyDeviceToHost; + TileXRCcuRepositoryInstallReceipt receipt; + const int ret = TileXRCcuInstallRepositoryImageWithOptions( + image, options, memoryOps, &state, adapter, &receipt, &report); + if (ret != TILEXR_ERROR_MKIRT || !state.freed || receipt.deviceInstructionPtr != nullptr || + state.readbackBytes != 5440ULL || state.driverReadbackBytes != 5440ULL) { + std::cerr << "install failure handling mismatch ret=" << ret + << " freed=" << state.freed + << " readbackBytes=" << state.readbackBytes + << " driverReadbackBytes=" << state.driverReadbackBytes << "\n"; + return 4; + } + + const std::string msg = report.message; + for (const char* needle : { + "failed to install CCU repository instruction image", + "CCU custom channel call failed op=251 driverRet=328107 opRet=0", + "dieId=1", + "installStartId=475", + "installCount=170", + "instructionBytes=5440", + "customChannelDataLen=5440", + "deviceInstructionAddr=0x", + "window=full_repository", + "dataLenMode=instruction_bytes", + "firstInstructionWords=", + "lastInstructionWords=", + "instructionFnv1a64=0x", + "uploadReadback=ok", + "uploadReadbackBytes=5440", + "uploadReadbackFnv1a64=0x", + "uploadReadbackFirstInstructionWords=", + "uploadReadbackLastInstructionWords=", + "uploadReadbackMismatchCount=0", + "driverReadback=ok", + "driverReadbackRet=0", + "driverReadbackBytes=5440", + "driverReadbackFnv1a64=0x", + "driverReadbackFirstInstructionWords=", + "driverReadbackLastInstructionWords=", + "driverReadbackMismatchCount=0"}) { + if (!Has(msg, needle)) { + std::cerr << "missing diagnostic field '" << needle << "' in: " << msg << "\n"; + return 5; + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_image_places_pure_barrier_sync_microcode_without_sqe_load(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_repository.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuProducerPlan plan; + plan.mission = {0, 1, 0x12345678U, true}; + plan.kernelLocalXn = {0, 1, 1}; + plan.kernelLocalCke = {0, 1, 1}; + plan.kernelLocalMission = {0, 1, 1}; + plan.instructionWindow = {0, 1, 2, 1, 2}; + plan.syncResources.push_back({0, 1, 2, 1, 1, 1, 1, 1, 1}); + plan.taskWindows.push_back({0, 1, 2, 13, {}}); + + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed: " << planReport.message << "\n"; + return 1; + } + if (!program.sqeLoad.empty() || program.sync.size() != 2) { + std::cerr << "pure barrier microcode size mismatch\n"; + return 2; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + if (TileXRCcuBuildRepositoryImage(plan, program, &image, &report) != TILEXR_SUCCESS) { + std::cerr << "repository image build failed: " << report.message << "\n"; + return 3; + } + if (image.repositoryStartId != 1 || image.repositoryCount != 2 || + image.missionStartId != 1 || image.missionCount != 2 || + image.missionOffset != 0 || image.sqeLoadCount != 0 || + image.syncOffset != 0 || image.syncCount != 2 || + image.instructions.size() != 2) { + std::cerr << "pure barrier repository metadata mismatch\n"; + return 4; + } + if (image.instructions[0].words[0] == 0 || + image.instructions[1].words[0] == 0 || + report.installedInstructionCount != 2 || + report.sqeLoadOffset != 0 || report.syncOffset != 0) { + std::cerr << "pure barrier repository content/report mismatch\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_image_rejects_inconsistent_windows(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_repository.h" + + #include + + using namespace TileXR; + + TileXRCcuProducerPlan MakePlan() + { + TileXRCcuProducerPlan plan; + plan.mission = {1, 6, 0x059b0f03U, true}; + plan.kernelLocalXn = {1, 1961, 62}; + plan.kernelLocalCke = {1, 332, 1}; + plan.kernelLocalMission = {1, 6, 1}; + plan.instructionWindow = {1, 475, 170, 489, 156}; + plan.syncResources.push_back({1, 1961, 2361, 364, 2, 3}); + plan.syncResources.push_back({1, 1962, 2362, 364, 3, 3}); + plan.syncResources.push_back({1, 1963, 2364, 364, 4, 3}); + plan.taskWindows.push_back({1, 489, 13, 13, {}}); + plan.taskWindows.push_back({1, 502, 143, 13, {}}); + return plan; + } + + int main() + { + TileXRCcuProducerPlan plan = MakePlan(); + TileXRCcuProgram program; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildMicrocode(plan, &program, &planReport) != TILEXR_SUCCESS) { + std::cerr << "microcode build failed\n"; + return 1; + } + + TileXRCcuRepositoryImage image; + TileXRCcuRepositoryReport report; + + TileXRCcuProducerPlan smallRepo = plan; + smallRepo.instructionWindow.repositoryCount = 15; + if (TileXRCcuBuildRepositoryImage(smallRepo, program, &image, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "small repository was accepted\n"; + return 2; + } + if (report.message.find("repository") == std::string::npos) { + std::cerr << "small repository diagnostic was weak: " << report.message << "\n"; + return 3; + } + + TileXRCcuProducerPlan shortSqeTask = plan; + shortSqeTask.taskWindows[0].instCnt = 12; + if (TileXRCcuBuildRepositoryImage(shortSqeTask, program, &image, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "short SQE task window was accepted\n"; + return 4; + } + if (report.message.find("SQE") == std::string::npos) { + std::cerr << "short SQE task diagnostic was weak: " << report.message << "\n"; + return 5; + } + + TileXRCcuProducerPlan shiftedSyncTask = plan; + shiftedSyncTask.taskWindows[1].instStartId = 501; + if (TileXRCcuBuildRepositoryImage(shiftedSyncTask, program, &image, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "misaligned sync task window was accepted\n"; + return 6; + } + if (report.message.find("sync") == std::string::npos) { + std::cerr << "misaligned sync diagnostic was weak: " << report.message << "\n"; + return 7; + } + + if (TileXRCcuBuildRepositoryImage(plan, program, nullptr, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "null output image was accepted\n"; + return 8; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_repository_layer_is_wired_and_has_no_private_hcomm_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = REPOSITORY_HEADER.read_text(encoding="utf-8") + source = REPOSITORY_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_repository.h", cmake) + self.assertIn("ccu/tilexr_ccu_repository.cpp", cmake) + self.assertIn("struct TileXRCcuRepositoryImage", header) + self.assertIn("struct TileXRCcuRepositoryReport", header) + self.assertIn("struct TileXRCcuDeviceMemoryOps", header) + self.assertIn("TileXRCcuCopyDeviceToHostFn", header) + self.assertIn("copyDeviceToHost", header) + self.assertIn("struct TileXRCcuRepositoryInstallReceipt", header) + self.assertIn("enum class TileXRCcuRepositoryMemoryAllocMode", header) + self.assertIn("TileXRCcuMakeAclModule3DeviceMemoryOps", header) + self.assertIn("RtHbm", header) + self.assertIn("TileXRCcuMakeRtHbmDeviceMemoryOps", header) + self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps", header) + self.assertIn("TileXRCcuRepositoryInstallOptions", header) + self.assertIn("TileXRCcuBuildRepositoryImage", header) + self.assertIn("TileXRCcuInstallRepositoryImageWithOptions", header) + self.assertIn("TileXRCcuInstallRepositoryImage", header) + self.assertIn("TileXRCcuReleaseRepositoryInstallReceipt", header) + self.assertIn("TileXRCcuMakeAclDeviceMemoryOps", header) + self.assertIn("tilexr_ccu_producer_plan.h", header) + self.assertIn("missionOffset", header) + self.assertIn("sqeLoadOffset", header) + self.assertIn("syncOffset", header) + self.assertIn("#include ", source) + self.assertIn("#include ", source) + self.assertIn("aclrtMalloc", source) + self.assertIn("ACL_MEM_MALLOC_HUGE_FIRST", source) + self.assertIn("aclrtMallocWithCfg", source) + self.assertIn("ACL_RT_MEM_ATTR_MODULE_ID", source) + self.assertIn("ACL_MEM_TYPE_HIGH_BAND_WIDTH", source) + self.assertIn("TILEXR_CCU_ACL_MODULE3_ID", source) + self.assertIn("aclrtMemcpy", source) + self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) + self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) + self.assertIn("aclrtFree", source) + self.assertIn("rtMalloc", source) + self.assertIn("RT_MEMORY_HBM", source) + self.assertIn("rtMemcpy", source) + self.assertIn("RT_MEMCPY_HOST_TO_DEVICE", source) + self.assertIn("RT_MEMCPY_DEVICE_TO_HOST", source) + self.assertIn("rtFree", source) + self.assertIn("TileXRCcuMakeAclDeviceMemoryOps", source) + self.assertIn("TileXRCcuMakeAclModule3DeviceMemoryOps", source) + self.assertIn("TileXRCcuMakeRtHbmDeviceMemoryOps", source) + self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_repository_install_failure_diagnostic_source_contract(self): + source = REPOSITORY_SOURCE.read_text(encoding="utf-8") + + for needle in [ + "BuildInstallFailureDiagnostic", + "dieId=", + "installStartId=", + "installCount=", + "instructionBytes=", + "customChannelDataLen=", + "deviceInstructionAddr=0x", + "window=", + "dataLenMode=", + "firstInstructionWords=", + "lastInstructionWords=", + "instructionFnv1a64=0x", + "uploadReadback=", + "uploadReadbackBytes=", + "uploadReadbackFnv1a64=0x", + "uploadReadbackFirstInstructionWords=", + "uploadReadbackLastInstructionWords=", + "uploadReadbackMismatchCount=", + "driverReadback=", + "driverReadbackRet=", + "driverReadbackBytes=", + "driverReadbackFnv1a64=0x", + "driverReadbackFirstInstructionWords=", + "driverReadbackLastInstructionWords=", + "driverReadbackMismatchCount=", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_resource_allocator.py b/tests/ccu/test_tilexr_ccu_resource_allocator.py new file mode 100644 index 00000000..17c105f4 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_resource_allocator.py @@ -0,0 +1,917 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +ALLOCATOR_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_resource_allocator.h" +ALLOCATOR_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_resource_allocator.cpp" +PRODUCER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_producer_plan.cpp" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuResourceAllocatorTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "allocator_test.cpp" + test_bin = temp_path / "allocator_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(ALLOCATOR_SOURCE), + str(PRODUCER_SOURCE), + str(BARRIER_SOURCE), + str(MICROCODE_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_allocator_declares_and_uses_separate_mission_instruction_start(self): + header = ALLOCATOR_HEADER.read_text(encoding="utf-8") + source = ALLOCATOR_SOURCE.read_text(encoding="utf-8") + + self.assertIn("uint16_t missionInstructionStartId = 0;", header) + self.assertIn("missionInstructionStartId", source) + self.assertIn("missionInstructionStart", source) + self.assertIn("repositoryPrefixCount", source) + self.assertIn("result.repository.num", source) + self.assertIn("result.repository.startId", source) + self.assertIn("generated.instructionWindow = {", source) + + def test_allocator_builds_complete_tilexr_owned_producer_plan(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.gsaStartId = 510; + spec.gsaCount = 20; + spec.ckeStartId = 332; + spec.ckeCount = 32; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 3; + request.syncInstructionCount = 143; + request.bindingsPerSyncResource = 3; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (!plan.mission.installed || plan.mission.dieId != 1 || plan.mission.missionId != 6 || + plan.mission.key != 0x059b0f03U) { + std::cerr << "mission/key mismatch\n"; + return 3; + } + if (plan.kernelLocalMission.startId != 6 || plan.kernelLocalMission.num != 1 || + plan.kernelLocalXn.startId != 1961 || plan.kernelLocalXn.num != 14 || + plan.kernelLocalGsa.startId != 510 || plan.kernelLocalGsa.num != 1 || + plan.kernelLocalCke.startId != 332 || plan.kernelLocalCke.num != 3) { + std::cerr << "kernel local ranges mismatch\n"; + return 4; + } + if (plan.instructionWindow.repositoryStartId != 475 || + plan.instructionWindow.repositoryCount != 156 || + plan.instructionWindow.missionStartId != 475 || + plan.instructionWindow.missionCount != 156) { + std::cerr << "instruction window mismatch\n"; + return 5; + } + if (plan.syncResources.size() != 3 || plan.taskWindows.size() != 2) { + std::cerr << "resource/task count mismatch\n"; + return 6; + } + if (plan.syncResources[0].localXn != 1961 || plan.syncResources[0].remoteXn != 1975 || + plan.syncResources[0].notifyCke != 332 || plan.syncResources[0].channelId != 2 || + plan.syncResources[0].bindingCount != 3) { + std::cerr << "first sync resource mismatch\n"; + return 7; + } + if (plan.syncResources[2].localXn != 1963 || plan.syncResources[2].remoteXn != 1977 || + plan.syncResources[2].notifyCke != 334 || plan.syncResources[2].channelId != 4) { + std::cerr << "last sync resource mismatch\n"; + return 8; + } + if (plan.taskWindows[0].instStartId != 475 || plan.taskWindows[0].instCnt != 13 || + plan.taskWindows[0].argSize != 13 || + plan.taskWindows[1].instStartId != 488 || plan.taskWindows[1].instCnt != 143 || + plan.taskWindows[1].argSize != 13) { + std::cerr << "task window mismatch\n"; + return 9; + } + + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "generated plan invalid: " << planReport.message << "\n"; + return 10; + } + if (allocation.receiptId == 0 || allocation.packageProvider != "tilexr-hcomm-derived-resource-allocator" || + allocation.localXn.startId != 1961 || allocation.remoteXn.startId != 1975 || + allocation.localGsa.startId != 510 || allocation.localGsa.num != 1 || + allocation.notifyCke.startId != 332 || allocation.channels.startId != 2 || + allocation.channels.num != 3) { + std::cerr << "allocation receipt mismatch\n"; + return 11; + } + if (report.missionAllocated != 1 || report.localXnAllocated != 14 || + report.localGsaAllocated != 1 || + report.remoteXnAllocated != 3 || report.notifyCkeAllocated != 3 || + report.channelBindingsAllocated != 9 || report.repositoryAllocated != 156 || + report.message != "ok") { + std::cerr << "report mismatch\n"; + return 12; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_can_assign_distinct_local_wait_and_remote_notify_cke_ranges(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 332; + spec.ckeCount = 32; + spec.localWaitCkeStartId = 332; + spec.localWaitCkeCount = 8; + spec.remoteNotifyCkeStartId = 364; + spec.remoteNotifyCkeCount = 8; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 3; + request.syncInstructionCount = 143; + request.bindingsPerSyncResource = 3; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (allocation.localWaitCke.startId != 332 || allocation.localWaitCke.num != 3 || + allocation.remoteNotifyCke.startId != 364 || allocation.remoteNotifyCke.num != 3 || + allocation.notifyCke.startId != 364 || allocation.notifyCke.num != 3) { + std::cerr << "split CKE allocation ranges mismatch\n"; + return 3; + } + if (plan.kernelLocalCke.startId != 332 || plan.kernelLocalCke.num != 3) { + std::cerr << "kernel-local CKE should describe the local wait CKE range\n"; + return 4; + } + if (plan.syncResources.size() != 3 || + plan.syncResources[0].localWaitCke != 332 || + plan.syncResources[0].notifyCke != 364 || + plan.syncResources[1].localWaitCke != 333 || + plan.syncResources[1].notifyCke != 365 || + plan.syncResources[2].localWaitCke != 334 || + plan.syncResources[2].notifyCke != 366) { + std::cerr << "split CKE sync resources mismatch\n"; + return 5; + } + if (report.localWaitCkeAllocated != 3 || + report.remoteNotifyCkeAllocated != 3 || + report.notifyCkeAllocated != 3) { + std::cerr << "split CKE report mismatch\n"; + return 6; + } + + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "split CKE generated plan invalid: " << planReport.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_can_reserve_repository_prefix_before_mission_instruction_window(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.missionInstructionStartId = 489; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 332; + spec.ckeCount = 32; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 3; + request.syncInstructionCount = 143; + request.bindingsPerSyncResource = 3; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (plan.instructionWindow.repositoryStartId != 475 || + plan.instructionWindow.repositoryCount != 170 || + plan.instructionWindow.missionStartId != 489 || + plan.instructionWindow.missionCount != 156) { + std::cerr << "repository/mission instruction window mismatch\n"; + return 3; + } + if (plan.taskWindows.size() != 2 || + plan.taskWindows[0].instStartId != 489 || + plan.taskWindows[0].instCnt != 13 || + plan.taskWindows[1].instStartId != 502 || + plan.taskWindows[1].instCnt != 143) { + std::cerr << "mission task windows did not start at mission instruction window\n"; + return 4; + } + if (allocation.repository.startId != 475 || + allocation.repository.num != 170 || + report.repositoryAllocated != 170) { + std::cerr << "repository allocation/report did not include prefix\n"; + return 5; + } + + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "prefixed repository plan invalid: " << planReport.message << "\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_builds_pure_barrier_plan_without_sqe_load_task(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 0; + spec.missionKey = 0x12345678U; + spec.missionStartId = 1; + spec.missionCount = 1; + spec.instructionStartId = 1; + spec.instructionCount = 16; + spec.xnStartId = 1; + spec.xnCount = 8; + spec.ckeStartId = 1; + spec.ckeCount = 8; + spec.channelStartId = 1; + spec.channelCount = 2; + + TileXRCcuResourceRequest request; + request.sqeArgCount = 0; + request.syncResourceCount = 1; + request.syncInstructionCount = 2; + request.bindingsPerSyncResource = 1; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (plan.taskWindows.size() != 1) { + std::cerr << "pure barrier should have one sync task\n"; + return 3; + } + if (plan.taskWindows[0].instStartId != 1 || + plan.taskWindows[0].instCnt != 2 || + plan.taskWindows[0].argSize != TILEXR_CCU_SQE_ARGS_LEN) { + std::cerr << "pure barrier sync task mismatch\n"; + return 4; + } + if (plan.kernelLocalXn.startId != 1 || plan.kernelLocalXn.num != 1 || + allocation.localXn.startId != 1 || allocation.localXn.num != 1 || + allocation.remoteXn.startId != 2 || allocation.remoteXn.num != 1 || + allocation.repository.startId != 1 || allocation.repository.num != 2) { + std::cerr << "pure barrier allocation mismatch\n"; + return 5; + } + if (report.localXnAllocated != 1 || + report.remoteXnAllocated != 1 || + report.repositoryAllocated != 2) { + std::cerr << "pure barrier report mismatch\n"; + return 6; + } + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "pure barrier plan invalid: " << planReport.message << "\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_assigns_independent_source_cke_for_sync_cke_barrier_mode(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 0x220; + spec.ckeCount = 16; + spec.localWaitCkeStartId = 0x220; + spec.localWaitCkeCount = 8; + spec.remoteNotifyCkeStartId = 0x330; + spec.remoteNotifyCkeCount = 8; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 1; + request.syncInstructionCount = 3; + request.bindingsPerSyncResource = 1; + request.barrierMode = TileXRCcuBarrierMode::SyncCke; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (plan.barrierMode != TileXRCcuBarrierMode::SyncCke || + allocation.localWaitCke.startId != 0x220 || allocation.localWaitCke.num != 1 || + allocation.sourceCke.startId != 0x221 || allocation.sourceCke.num != 1 || + plan.kernelLocalCke.startId != 0x220 || plan.kernelLocalCke.num != 2) { + std::cerr << "sync_cke CKE allocation ranges mismatch\n"; + return 3; + } + if (plan.syncResources.size() != 1 || + plan.syncResources[0].notifyCke != 0x330 || + plan.syncResources[0].localWaitCke != 0x220 || + plan.syncResources[0].sourceCke != 0x221 || + plan.syncResources[0].sourceCkeMask != 0xffff) { + std::cerr << "sync_cke sync resource CKE fields mismatch\n"; + return 4; + } + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "sync_cke generated plan invalid: " << planReport.message << "\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_accepts_synccke_post_only_with_source_cke_and_two_instructions(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 0x220; + spec.ckeCount = 16; + spec.localWaitCkeStartId = 0x220; + spec.localWaitCkeCount = 8; + spec.remoteNotifyCkeStartId = 0x330; + spec.remoteNotifyCkeCount = 8; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 1; + request.syncInstructionCount = 2; + request.bindingsPerSyncResource = 1; + request.barrierMode = TileXRCcuBarrierMode::SyncCkePostOnly; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (plan.barrierMode != TileXRCcuBarrierMode::SyncCkePostOnly || + allocation.sourceCke.startId != 0x221 || allocation.sourceCke.num != 1 || + plan.instructionWindow.repositoryCount != TILEXR_CCU_SQE_ARGS_LEN + 2) { + std::cerr << "sync_cke_post_only allocation mismatch\n"; + return 3; + } + if (plan.syncResources.size() != 1 || + plan.syncResources[0].sourceCke != 0x221 || + plan.syncResources[0].notifyCke != 0x330 || + plan.syncResources[0].localWaitCke != 0x220) { + std::cerr << "sync_cke_post_only resource fields mismatch\n"; + return 4; + } + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "sync_cke_post_only generated plan invalid: " << planReport.message << "\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_accepts_local_cke_post_only_with_one_instruction(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 0x220; + spec.ckeCount = 16; + spec.localWaitCkeStartId = 0x220; + spec.localWaitCkeCount = 8; + spec.remoteNotifyCkeStartId = 0x330; + spec.remoteNotifyCkeCount = 8; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = 0; + request.syncResourceCount = 1; + request.syncInstructionCount = 1; + request.bindingsPerSyncResource = 1; + request.barrierMode = TileXRCcuBarrierMode::LocalCkePostOnly; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "allocate failed: " << report.message << "\n"; + return 2; + } + + if (plan.barrierMode != TileXRCcuBarrierMode::LocalCkePostOnly || + plan.taskWindows.size() != 1 || + plan.taskWindows[0].instCnt != 1 || + plan.instructionWindow.repositoryCount != 1) { + std::cerr << "local_cke_post_only allocation mismatch\n"; + return 3; + } + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuValidateProducerPlan(plan, &planReport) != TILEXR_SUCCESS) { + std::cerr << "local_cke_post_only generated plan invalid: " << planReport.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_rejects_resource_exhaustion_and_double_release(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 0; + spec.missionKey = 0x12345678U; + spec.missionStartId = 1; + spec.missionCount = 1; + spec.instructionStartId = 100; + spec.instructionCount = 32; + spec.xnStartId = 200; + spec.xnCount = 17; + spec.ckeStartId = 300; + spec.ckeCount = 2; + spec.channelStartId = 4; + spec.channelCount = 1; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 3; + request.syncInstructionCount = 11; + request.bindingsPerSyncResource = 1; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "resource exhaustion was accepted\n"; + return 2; + } + if (report.message.find("insufficient CKE resources") == std::string::npos) { + std::cerr << "weak exhaustion diagnostic: " << report.message << "\n"; + return 3; + } + + spec.ckeCount = 8; + spec.channelCount = 3; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "reinit failed\n"; + return 4; + } + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_SUCCESS) { + std::cerr << "second allocate failed: " << report.message << "\n"; + return 5; + } + if (allocator.Release(allocation.receiptId) != TILEXR_SUCCESS) { + std::cerr << "release failed\n"; + return 6; + } + if (allocator.Release(allocation.receiptId) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "double release was accepted\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_rejects_sync_instruction_window_too_small_for_barrier_program(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 332; + spec.ckeCount = 32; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 2; + request.syncInstructionCount = 3; + request.bindingsPerSyncResource = 1; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "undersized barrier sync instruction window was accepted\n"; + return 2; + } + if (report.message.find("barrier") == std::string::npos || + report.message.find("sync instruction") == std::string::npos) { + std::cerr << "weak barrier instruction diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_counts_hcomm_style_task1_prelude_for_two_task_sync_xn_programs(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 1; + spec.missionKey = 0x059b0f03U; + spec.missionStartId = 6; + spec.missionCount = 2; + spec.instructionStartId = 475; + spec.instructionCount = 170; + spec.xnStartId = 1961; + spec.xnCount = 62; + spec.ckeStartId = 332; + spec.ckeCount = 32; + spec.channelStartId = 2; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = TILEXR_CCU_SQE_ARGS_LEN; + request.syncResourceCount = 2; + request.syncInstructionCount = 2; + request.bindingsPerSyncResource = 1; + request.barrierMode = TileXRCcuBarrierMode::SyncXnPostOnly; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "allocator init failed\n"; + return 1; + } + + TileXRCcuProducerPlan plan; + TileXRCcuResourceAllocation allocation; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &plan, &allocation, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "undersized hcomm-style task1 prelude window was accepted\n"; + return 2; + } + if (report.message.find("prelude") == std::string::npos || + report.message.find("sync instruction") == std::string::npos) { + std::cerr << "weak prelude instruction diagnostic: " << report.message << "\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_allocator_is_wired_and_does_not_reference_hcomm_runtime_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = ALLOCATOR_HEADER.read_text(encoding="utf-8") + source = ALLOCATOR_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_resource_allocator.h", cmake) + self.assertIn("ccu/tilexr_ccu_resource_allocator.cpp", cmake) + self.assertIn("TileXRCcuResourceAllocator", header) + self.assertIn("TileXRCcuResourceSpec", header) + self.assertIn("TileXRCcuResourceRequest", header) + self.assertIn("TileXRCcuResourceAllocation", header) + self.assertIn("TileXRCcuProducerPlan", header) + self.assertIn("tilexr-hcomm-derived-resource-allocator", source) + self.assertIn("requiredBarrierInstructionCount", source) + self.assertIn("syncResourceCount * 2U", source) + self.assertIn("TileXRCcuBarrierMode::LocalCkePostOnly", source) + self.assertIn("request.barrierMode == TileXRCcuBarrierMode::LocalCkePostOnly", source) + self.assertIn("barrier sync instruction window is too small", source) + self.assertIn("const uint32_t channelCount = request.syncResourceCount", source) + self.assertNotIn("TILEXR_CCU_DIRECT_SYNC_RESOURCE_MAP", source) + self.assertNotIn("UseHcommTraceSyncResourceMap", source) + self.assertNotIn("TILEXR_CCU_DIRECT_SQE_LOAD_XN_MAP", source) + self.assertNotIn("TILEXR_CCU_DIRECT_TASK1_PRELUDE_CKE", source) + + combined = header + "\n" + source + for needle in [ + "#include ", source) + self.assertIn("static_assert(RT_CCU_SQE_ARGS_LEN == TILEXR_CCU_SQE_ARGS_LEN", source) + self.assertIn("rtCcuTaskInfo_t runtimeTask", source) + self.assertIn("rtCCULaunch(&runtimeTask, stream)", source) + self.assertIn("stream == nullptr", source) + self.assertIn("RT_CCU_INST_CNT_INVALID", source) + self.assertIn("RT_CCU_INST_START_MAX", source) + self.assertIn("task.argSize != 1 && task.argSize != TILEXR_CCU_SQE_ARGS_LEN", source) + self.assertNotIn("TILEXR_CCU_DIRECT_TASK_DIE_ID", source) + self.assertNotIn("TILEXR_CCU_DIRECT_TASK_TIMEOUT", source) + self.assertNotIn("TILEXR_CCU_DIRECT_TASK_ARG_SIZE", source) + self.assertNotIn("TILEXR_CCU_DIRECT_TASK_ARG", source) + self.assertNotIn("ApplyRuntimeTaskOverrides", source) + self.assertIn("TILEXR_ERROR_MKIRT", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_rt_ccu_launch_reference_is_confined_to_runtime_submit_wrapper(self): + offenders = [] + for root in [REPO_ROOT / "src" / "comm", REPO_ROOT / "src" / "include"]: + for path in root.rglob("*"): + if path.is_file() and path.suffix in {".h", ".hpp", ".cpp", ".cc", ".c"}: + text = path.read_text(encoding="utf-8", errors="replace") + if "rtCCULaunch" in text and path != CCU_SOURCE: + offenders.append(path.relative_to(REPO_ROOT).as_posix()) + self.assertEqual([], offenders) + + def test_runtime_kernel_header_is_confined_to_runtime_submit_wrapper(self): + offenders = [] + for root in [REPO_ROOT / "src" / "comm", REPO_ROOT / "src" / "include"]: + for path in root.rglob("*"): + if path.is_file() and path.suffix in {".h", ".hpp", ".cpp", ".cc", ".c"}: + text = path.read_text(encoding="utf-8", errors="replace") + if "runtime/kernel.h" in text and path != CCU_SOURCE: + offenders.append(path.relative_to(REPO_ROOT).as_posix()) + self.assertEqual([], offenders) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_runtime_fake_launch.py b/tests/ccu/test_tilexr_ccu_runtime_fake_launch.py new file mode 100644 index 00000000..d9d4c43e --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_runtime_fake_launch.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMM_DIR = REPO_ROOT / "src" / "comm" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +RUNTIME_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_runtime.cpp" +ORCHESTRATOR_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_direct_orchestrator.cpp" +REPOSITORY_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_repository.cpp" +COMM_WRAP_SOURCE = COMM_DIR / "comm_wrap.cpp" + + +class TileXRCcuRuntimeFakeLaunchTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + runtime_dir = temp_path / "runtime" + runtime_dir.mkdir(parents=True) + acl_dir = temp_path / "acl" + acl_dir.mkdir(parents=True) + (acl_dir / "acl_rt.h").write_text( + textwrap.dedent( + r""" + #ifndef TILEXR_TEST_FAKE_ACL_RT_H + #define TILEXR_TEST_FAKE_ACL_RT_H + + #include + + #define ACL_SUCCESS 0 + #define ACL_MEM_MALLOC_HUGE_FIRST 0 + #define ACL_MEMCPY_HOST_TO_DEVICE 0 + #define ACL_MEMCPY_DEVICE_TO_HOST 1 + #define ACL_MEM_TYPE_HIGH_BAND_WIDTH 0 + #define ACL_RT_MEM_ATTR_MODULE_ID 0 + + typedef int aclError; + typedef int aclrtMemMallocPolicy; + typedef union aclrtMallocAttrValue { + unsigned int moduleId; + } aclrtMallocAttrValue; + typedef struct aclrtMallocAttribute { + int attr; + aclrtMallocAttrValue value; + } aclrtMallocAttribute; + typedef struct aclrtMallocConfig { + aclrtMallocAttribute* attrs; + size_t attrCount; + } aclrtMallocConfig; + + extern "C" aclError aclrtSetDevice(int deviceId); + extern "C" aclError aclrtMalloc(void** devPtr, size_t size, int policy); + static inline aclError aclrtMallocWithCfg( + void** devPtr, size_t size, int memoryType, aclrtMallocConfig* cfg) + { + (void)memoryType; + (void)cfg; + return aclrtMalloc(devPtr, size, ACL_MEM_MALLOC_HUGE_FIRST); + } + extern "C" aclError aclrtMemcpy( + void* dst, size_t destMax, const void* src, size_t count, int kind); + extern "C" aclError aclrtFree(void* devPtr); + + #endif + """ + ), + encoding="utf-8", + ) + (runtime_dir / "kernel.h").write_text( + textwrap.dedent( + r""" + #ifndef TILEXR_TEST_FAKE_RUNTIME_KERNEL_H + #define TILEXR_TEST_FAKE_RUNTIME_KERNEL_H + + #include + + #define RT_CCU_SQE_ARGS_LEN 13U + #define RT_CCU_INST_CNT_INVALID 0U + #define RT_CCU_INST_START_MAX 65535U + #define RT_ERROR_NONE 0 + + typedef int32_t rtError_t; + typedef void* rtStream_t; + + typedef struct rtCcuTaskInfo { + uint8_t dieId; + uint8_t missionId; + uint16_t timeout; + uint16_t instStartId; + uint16_t instCnt; + uint32_t key; + uint32_t argSize; + uint64_t args[RT_CCU_SQE_ARGS_LEN]; + } rtCcuTaskInfo_t; + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t stream); + + #endif + """ + ), + encoding="utf-8", + ) + + test_cpp = temp_path / "test.cpp" + test_cpp.write_text(code, encoding="utf-8") + test_bin = temp_path / "test_fake_launch" + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(temp_path), + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(RUNTIME_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def compile_and_run_public_prepared_handle(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + runtime_dir = temp_path / "runtime" + runtime_dir.mkdir(parents=True) + (runtime_dir / "kernel.h").write_text( + textwrap.dedent( + r""" + #ifndef TILEXR_TEST_FAKE_RUNTIME_KERNEL_H + #define TILEXR_TEST_FAKE_RUNTIME_KERNEL_H + + #include + + #define RT_CCU_SQE_ARGS_LEN 13U + #define RT_CCU_INST_CNT_INVALID 0U + #define RT_CCU_INST_START_MAX 65535U + #define RT_ERROR_NONE 0 + + typedef int32_t rtError_t; + typedef void* rtStream_t; + + typedef struct rtCcuTaskInfo { + uint8_t dieId; + uint8_t missionId; + uint16_t timeout; + uint16_t instStartId; + uint16_t instCnt; + uint32_t key; + uint32_t argSize; + uint64_t args[RT_CCU_SQE_ARGS_LEN]; + } rtCcuTaskInfo_t; + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t stream); + + #endif + """ + ), + encoding="utf-8", + ) + acl_dir = temp_path / "acl" + acl_dir.mkdir(parents=True) + (acl_dir / "acl_rt.h").write_text( + textwrap.dedent( + r""" + #ifndef TILEXR_TEST_FAKE_ACL_RT_H + #define TILEXR_TEST_FAKE_ACL_RT_H + + #include + + #define ACL_SUCCESS 0 + #define ACL_MEM_MALLOC_HUGE_FIRST 0 + #define ACL_MEMCPY_HOST_TO_DEVICE 0 + #define ACL_MEMCPY_DEVICE_TO_HOST 1 + #define ACL_MEM_TYPE_HIGH_BAND_WIDTH 0 + #define ACL_RT_MEM_ATTR_MODULE_ID 0 + + typedef int aclError; + typedef int aclrtMemMallocPolicy; + typedef union aclrtMallocAttrValue { + unsigned int moduleId; + } aclrtMallocAttrValue; + typedef struct aclrtMallocAttribute { + int attr; + aclrtMallocAttrValue value; + } aclrtMallocAttribute; + typedef struct aclrtMallocConfig { + aclrtMallocAttribute* attrs; + size_t attrCount; + } aclrtMallocConfig; + + extern "C" aclError aclrtSetDevice(int deviceId); + extern "C" aclError aclrtMalloc(void** devPtr, size_t size, int policy); + static inline aclError aclrtMallocWithCfg( + void** devPtr, size_t size, int memoryType, aclrtMallocConfig* cfg) + { + (void)memoryType; + (void)cfg; + return aclrtMalloc(devPtr, size, ACL_MEM_MALLOC_HUGE_FIRST); + } + extern "C" aclError aclrtMemcpy( + void* dst, size_t destMax, const void* src, size_t count, int kind); + extern "C" aclError aclrtFree(void* devPtr); + + #endif + """ + ), + encoding="utf-8", + ) + + test_cpp = temp_path / "test_public_prepared.cpp" + test_cpp.write_text(code, encoding="utf-8") + test_bin = temp_path / "test_public_prepared" + subprocess.run( + [ + compiler, + "-std=c++14", + "-DTILEXR_CCU_TESTING=1", + "-DTILEXR_LOG_DISABLE_SPDLOG=1", + "-ffunction-sections", + "-fdata-sections", + "-I", + str(temp_path), + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(COMM_WRAP_SOURCE), + str(ORCHESTRATOR_SOURCE), + str(REPOSITORY_SOURCE), + str(RUNTIME_SOURCE), + "-Wl,--gc-sections", + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_submit_task_copies_tilexr_fields_to_runtime_task_info(self): + code = textwrap.dedent( + r""" + #include "ccu/tilexr_ccu_runtime.h" + #include "tilexr_types.h" + + #include + #include + #include + + #include + + namespace { + rtCcuTaskInfo_t g_capturedTask {}; + rtStream_t g_capturedStream = nullptr; + int g_launchCount = 0; + } + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t stream) + { + ++g_launchCount; + g_capturedStream = stream; + std::memcpy(&g_capturedTask, taskInfo, sizeof(g_capturedTask)); + return RT_ERROR_NONE; + } + + int main() + { + TileXR::TileXRCcuTask task {}; + task.dieId = 3; + task.missionId = 7; + task.timeout = 68; + task.instStartId = 1024; + task.instCnt = 13; + task.key = 0x05ab1234U; + task.argSize = TileXR::TILEXR_CCU_SQE_ARGS_LEN; + for (uint32_t i = 0; i < TileXR::TILEXR_CCU_SQE_ARGS_LEN; ++i) { + task.args[i] = 0x1000000000000000ULL + i; + } + + void* stream = reinterpret_cast(0x12345678ULL); + const int ret = TileXR::TileXRCcuSubmitTask(task, stream); + if (ret != TileXR::TILEXR_SUCCESS) { + std::printf("unexpected submit ret=%d\n", ret); + return 1; + } + if (g_launchCount != 1 || g_capturedStream != stream) { + std::printf("launchCount=%d capturedStream=%p\n", g_launchCount, g_capturedStream); + return 2; + } + if (g_capturedTask.dieId != task.dieId || g_capturedTask.missionId != task.missionId || + g_capturedTask.timeout != task.timeout || g_capturedTask.instStartId != task.instStartId || + g_capturedTask.instCnt != task.instCnt || g_capturedTask.key != task.key || + g_capturedTask.argSize != task.argSize) { + std::printf("runtime scalar field mismatch\n"); + return 3; + } + for (uint32_t i = 0; i < TileXR::TILEXR_CCU_SQE_ARGS_LEN; ++i) { + if (g_capturedTask.args[i] != task.args[i]) { + std::printf("arg[%u] mismatch\n", i); + return 4; + } + } + return 0; + } + """ + ) + + result = self.compile_and_run(code) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_submit_report_records_final_runtime_task(self): + code = textwrap.dedent( + r""" + #include "ccu/tilexr_ccu_runtime.h" + #include "tilexr_types.h" + + #include + #include + #include + + #include + + namespace { + rtCcuTaskInfo_t g_capturedTask {}; + } + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t) + { + std::memcpy(&g_capturedTask, taskInfo, sizeof(g_capturedTask)); + return RT_ERROR_NONE; + } + + int main() + { + TileXR::TileXRCcuTask task {}; + task.dieId = 3; + task.missionId = 6; + task.timeout = 68; + task.instStartId = 489; + task.instCnt = 2; + task.key = 0x059b0f03U; + task.argSize = TileXR::TILEXR_CCU_SQE_ARGS_LEN; + task.args[0] = 0x1111ULL; + task.args[1] = 0x2222ULL; + + TileXR::TileXRCcuRuntimeSubmitReport report {}; + const int ret = TileXR::TileXRCcuSubmitTaskWithReport( + task, reinterpret_cast(0x1ULL), &report); + if (ret != TileXR::TILEXR_SUCCESS) { + std::printf("submit ret=%d\n", ret); + return 1; + } + if (!report.finalTaskCaptured) { + std::printf("final task was not captured\n"); + return 2; + } + if (report.finalTask.dieId != task.dieId || report.finalTask.timeout != task.timeout || + report.finalTask.argSize != task.argSize || report.finalTask.args[0] != 0x1111ULL || + report.finalTask.args[1] != 0x2222ULL) { + std::printf("final task mismatch die=%u timeout=%u argSize=%u arg0=0x%llx arg1=0x%llx\n", + static_cast(report.finalTask.dieId), + static_cast(report.finalTask.timeout), + static_cast(report.finalTask.argSize), + static_cast(report.finalTask.args[0]), + static_cast(report.finalTask.args[1])); + return 3; + } + if (g_capturedTask.dieId != report.finalTask.dieId || + g_capturedTask.timeout != report.finalTask.timeout || + g_capturedTask.argSize != report.finalTask.argSize || + g_capturedTask.args[0] != report.finalTask.args[0]) { + std::printf("captured runtime task differs from report\n"); + return 4; + } + return 0; + } + """ + ) + + result = self.compile_and_run(code) + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_submit_task_maps_runtime_launch_failure_to_mkirt_error(self): + code = textwrap.dedent( + r""" + #include "ccu/tilexr_ccu_runtime.h" + #include "tilexr_types.h" + + #include + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t*, rtStream_t) + { + return 507000; + } + + int main() + { + TileXR::TileXRCcuTask task {}; + task.dieId = 1; + task.missionId = 2; + task.instStartId = 8; + task.instCnt = 1; + task.key = 0x1234U; + task.argSize = 1; + task.args[0] = 0xfeedULL; + + const int ret = TileXR::TileXRCcuSubmitTask(task, reinterpret_cast(0x1ULL)); + return ret == TileXR::TILEXR_ERROR_MKIRT ? 0 : 1; + } + """ + ) + + result = self.compile_and_run(code) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_public_prepared_handle_submits_batch_through_runtime_launch(self): + code = textwrap.dedent( + r""" + #include "tilexr_api.h" + #include "tilexr_types.h" + + #include + #include + #include + + #include + #include + + extern "C" TileXRDirectCcuPreparedTasksPtr TileXRDirectCcuCreatePreparedForTest( + const TileXRDirectCcuTaskInfo* tasks, uint32_t taskCount); + + namespace { + rtCcuTaskInfo_t g_tasks[4] {}; + rtStream_t g_streams[4] {}; + int g_launchCount = 0; + int g_failOnCall = 0; + } + + extern "C" aclError aclrtSetDevice(int) { return ACL_SUCCESS; } + extern "C" aclError aclrtMalloc(void**, size_t, int) { return ACL_SUCCESS; } + extern "C" aclError aclrtMemcpy(void*, size_t, const void*, size_t, int) { return ACL_SUCCESS; } + extern "C" aclError aclrtFree(void*) { return ACL_SUCCESS; } + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t* taskInfo, rtStream_t stream) + { + const int index = g_launchCount++; + g_streams[index] = stream; + std::memcpy(&g_tasks[index], taskInfo, sizeof(g_tasks[index])); + return g_failOnCall == g_launchCount ? 507000 : RT_ERROR_NONE; + } + + TileXRDirectCcuTaskInfo MakeTask(uint8_t mission, uint16_t instStart) + { + TileXRDirectCcuTaskInfo task {}; + task.dieId = 2; + task.missionId = mission; + task.timeout = static_cast(30 + mission); + task.instStartId = instStart; + task.instCnt = 2; + task.key = 0xabc00000U + mission; + task.argSize = 13; + for (uint32_t i = 0; i < 13; ++i) { + task.args[i] = 0x8000000000000000ULL + (static_cast(mission) << 8U) + i; + } + return task; + } + + bool SameTask(const TileXRDirectCcuTaskInfo& expected, const rtCcuTaskInfo_t& actual) + { + if (expected.dieId != actual.dieId || expected.missionId != actual.missionId || + expected.timeout != actual.timeout || expected.instStartId != actual.instStartId || + expected.instCnt != actual.instCnt || expected.key != actual.key || + expected.argSize != actual.argSize) { + return false; + } + for (uint32_t i = 0; i < 13; ++i) { + if (expected.args[i] != actual.args[i]) { + return false; + } + } + return true; + } + + int main() + { + TileXRDirectCcuTaskInfo tasks[2] = {MakeTask(7, 101), MakeTask(8, 103)}; + TileXRDirectCcuPreparedTasksPtr prepared = TileXRDirectCcuCreatePreparedForTest(tasks, 2); + if (prepared == nullptr) { + std::printf("missing prepared handle\n"); + return 1; + } + + TileXRDirectCcuTaskInfo preview {}; + if (TileXRDirectCcuGetPreparedTask(prepared, 1, &preview) != TileXR::TILEXR_SUCCESS || + preview.missionId != tasks[1].missionId || preview.args[12] != tasks[1].args[12]) { + std::printf("prepared task preview mismatch\n"); + return 2; + } + + void* stream = reinterpret_cast(0x12345678ULL); + TileXRDirectCcuSubmitReport report {}; + const int ret = TileXRDirectCcuSubmitPrepared(prepared, stream, &report); + if (ret != TileXR::TILEXR_SUCCESS || !report.submitted || + report.taskCount != 2 || report.submittedTaskCount != 2) { + std::printf("submit report mismatch ret=%d submitted=%d taskCount=%u submittedTaskCount=%u\n", + ret, report.submitted ? 1 : 0, report.taskCount, report.submittedTaskCount); + return 3; + } + if (g_launchCount != 2 || g_streams[0] != stream || g_streams[1] != stream) { + std::printf("launch count or stream mismatch count=%d\n", g_launchCount); + return 4; + } + if (!SameTask(tasks[0], g_tasks[0]) || !SameTask(tasks[1], g_tasks[1])) { + std::printf("runtime task payload mismatch\n"); + return 5; + } + + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet == TileXR::TILEXR_SUCCESS ? 0 : 6; + } + """ + ) + + result = self.compile_and_run_public_prepared_handle(code) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_public_prepared_handle_reports_null_stream_and_mid_batch_failure(self): + code = textwrap.dedent( + r""" + #include "tilexr_api.h" + #include "tilexr_types.h" + + #include + #include + #include + + #include + #include + + extern "C" TileXRDirectCcuPreparedTasksPtr TileXRDirectCcuCreatePreparedForTest( + const TileXRDirectCcuTaskInfo* tasks, uint32_t taskCount); + + namespace { + int g_launchCount = 0; + int g_failOnCall = 0; + } + + extern "C" aclError aclrtSetDevice(int) { return ACL_SUCCESS; } + extern "C" aclError aclrtMalloc(void**, size_t, int) { return ACL_SUCCESS; } + extern "C" aclError aclrtMemcpy(void*, size_t, const void*, size_t, int) { return ACL_SUCCESS; } + extern "C" aclError aclrtFree(void*) { return ACL_SUCCESS; } + + extern "C" rtError_t rtCCULaunch(rtCcuTaskInfo_t*, rtStream_t) + { + ++g_launchCount; + return g_failOnCall == g_launchCount ? 507000 : RT_ERROR_NONE; + } + + TileXRDirectCcuTaskInfo MakeTask(uint8_t mission, uint16_t instStart) + { + TileXRDirectCcuTaskInfo task {}; + task.dieId = 1; + task.missionId = mission; + task.timeout = static_cast(60 + mission); + task.instStartId = instStart; + task.instCnt = static_cast(4 + mission); + task.key = 0x12340000U + mission; + task.argSize = 13; + for (uint32_t i = 0; i < 13; ++i) { + task.args[i] = 0xfeed000000000000ULL + (static_cast(mission) << 8U) + i; + } + return task; + } + + bool Contains(const char* text, const char* needle) + { + return text != nullptr && std::strstr(text, needle) != nullptr; + } + + int main() + { + TileXRDirectCcuTaskInfo tasks[2] = {MakeTask(2, 11), MakeTask(3, 12)}; + TileXRDirectCcuPreparedTasksPtr prepared = TileXRDirectCcuCreatePreparedForTest(tasks, 2); + if (prepared == nullptr) { + return 1; + } + + TileXRDirectCcuSubmitReport nullStreamReport {}; + int ret = TileXRDirectCcuSubmitPrepared(prepared, nullptr, &nullStreamReport); + if (ret != TileXR::TILEXR_ERROR_PARA_CHECK_FAIL || + nullStreamReport.submitted || nullStreamReport.taskCount != 2 || + nullStreamReport.submittedTaskCount != 0 || + !Contains(nullStreamReport.message, "missing runtime stream") || + g_launchCount != 0) { + std::printf("bad null-stream report ret=%d launchCount=%d message=%s\n", + ret, g_launchCount, nullStreamReport.message); + return 2; + } + + g_failOnCall = 2; + TileXRDirectCcuSubmitReport failReport {}; + ret = TileXRDirectCcuSubmitPrepared(prepared, reinterpret_cast(0x1ULL), &failReport); + if (ret != TileXR::TILEXR_ERROR_MKIRT || failReport.submitted || + failReport.taskCount != 2 || failReport.submittedTaskCount != 1 || + !Contains(failReport.message, "task=1") || + !Contains(failReport.message, "rtRet=507000") || + !Contains(failReport.message, "dieId=1") || + !Contains(failReport.message, "missionId=3") || + !Contains(failReport.message, "timeout=63") || + !Contains(failReport.message, "instStartId=12") || + !Contains(failReport.message, "instCnt=7") || + !Contains(failReport.message, "key=0x12340003") || + !Contains(failReport.message, "argSize=13") || + !Contains(failReport.message, "args[0]=0xfeed000000000300") || + !Contains(failReport.message, "args[12]=0xfeed00000000030c") || + g_launchCount != 2) { + std::printf("bad mid-batch report ret=%d launchCount=%d taskCount=%u submitted=%u message=%s\n", + ret, g_launchCount, failReport.taskCount, failReport.submittedTaskCount, + failReport.message); + return 3; + } + + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet == TileXR::TILEXR_SUCCESS ? 0 : 4; + } + """ + ) + + result = self.compile_and_run_public_prepared_handle(code) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_public_prepared_handle_test_seam_is_test_only_and_keeps_private_ccu_out(self): + source = COMM_WRAP_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_TESTING", source) + self.assertIn("TileXRDirectCcuCreatePreparedForTest", source) + for needle in [ + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_source_guard_coverage.py b/tests/ccu/test_tilexr_ccu_source_guard_coverage.py new file mode 100644 index 00000000..2f45fd5f --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_source_guard_coverage.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SOURCE_GUARD = REPO_ROOT / "tests" / "comm" / "unit" / "test_tilexr_source_guards.cpp" +BUILD_ARTIFACT_GUARD = REPO_ROOT / "tests" / "ccu" / "check_tile_comm_no_hcomm_deps.sh" +DIRECT_SMOKE_PROBE = REPO_ROOT / "tests" / "ccu" / "ccu_tilexr_direct_smoke_probe.cpp" +DIRECT_SMOKE_RUNNER = REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh" +ROOT_CMAKE = REPO_ROOT / "CMakeLists.txt" +CCU_CMAKE = REPO_ROOT / "tests" / "ccu" / "CMakeLists.txt" + + +class TileXRCcuSourceGuardCoverageTest(unittest.TestCase): + def test_source_guard_covers_hccl_include_and_path_variants(self): + source = SOURCE_GUARD.read_text(encoding="utf-8").replace('\\"', '"') + + for needle in [ + "TestRootCMakeHcclIncludesAreNotTileCommSurface", + "#include ", + "#include \"hccl.h\"", + "pkg_inc/hccl", + "include/hccl", + "${ARCH}-linux/include/hccl", + "${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + + def test_root_cmake_does_not_expose_hccl_include_globally(self): + cmake = ROOT_CMAKE.read_text(encoding="utf-8") + + self.assertNotIn("${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl/", cmake) + self.assertNotIn("${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl", cmake) + + def test_ccu_cmake_does_not_define_private_probe_targets(self): + cmake = CCU_CMAKE.read_text(encoding="utf-8") + + for needle in [ + "TILEXR_BUILD_PRIVATE_CCU_PROBES", + "ccu_context_probe", + "ccu_barrier_kernel_probe", + "ccu_taskinfo_probe", + "include/hccl", + "pkg_inc/hcomm", + "hcomm", + "hccl", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, cmake) + + def test_build_artifact_guard_scans_private_hcomm_hccl_and_ccu_surfaces(self): + script = BUILD_ARTIFACT_GUARD.read_text(encoding="utf-8") + + for needle in [ + "readelf -d", + "ldd", + "nm -D", + "strings -a", + "libhcomm\\.so", + "libhccl_v2\\.so", + "libhccl_fwk\\.so", + "libmc2_client\\.so", + "HcclGetCcuTaskInfo", + "HcclChannelAcquire", + "HcommChannelNotify", + "RT_RES_TYPE_CCU_XN", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, script) + + def test_direct_smoke_probe_keeps_private_producer_dependencies_out(self): + source = DIRECT_SMOKE_PROBE.read_text(encoding="utf-8") + + for needle in [ + "#include ", + "#include \"hccl.h\"", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "CcuResBatchAllocator", + "CcuResRepository", + "runtime/kernel.h", + "rtCCULaunch", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, source) + + def test_direct_smoke_runner_keeps_private_link_dependencies_out(self): + source = DIRECT_SMOKE_RUNNER.read_text(encoding="utf-8") + + for needle in [ + "-lhcomm", + "-lhccl", + "-lhccl_v2", + "-lhccl_fwk", + "-lmc2_client", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "pkg_inc/hcomm", + "pkg_inc/hccl", + "include/hccl", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, source) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_specs.py b/tests/ccu/test_tilexr_ccu_specs.py new file mode 100644 index 00000000..4760eb98 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_specs.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SPECS_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.h" +SPECS_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp" +ALLOCATOR_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_resource_allocator.h" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuSpecsTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "specs_test.cpp" + test_bin = temp_path / "specs_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(SPECS_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_decodes_hcomm_basic_info_caps_into_tilexr_spec(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_specs.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x25; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 169U; + basic.caps.cap1 = (61U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 35U; + basic.caps.cap3 = (127U << 16) | 3U; + basic.caps.cap4 = 15U; + + TileXRCcuSpecInfo info; + TileXRCcuSpecsReport report; + if (TileXRCcuDecodeBasicInfo(basic, &info, &report) != TILEXR_SUCCESS) { + std::cerr << "decode failed: " << report.message << "\n"; + return 1; + } + + if (info.dieId != 1 || info.msId != 0x25 || info.missionKey != 0x059b0f03U || + info.resourceAddr != 0x100000000ULL) { + std::cerr << "base fields mismatch\n"; + return 2; + } + if (info.instructionNum != 170 || info.xnNum != 62 || info.gsaNum != 32 || + info.msNum != 64 || info.ckeNum != 36 || info.jettyNum != 128 || + info.channelNum != 4 || info.pfeNum != 16 || + info.missionNum != 12 || info.loopEngineNum != 8) { + std::cerr << "caps decode mismatch\n"; + return 3; + } + if (info.xnBaseAddr != 0x100000000ULL + TILEXR_CCU_V1_XN_RESOURCE_OFFSET) { + std::cerr << "xn base mismatch\n"; + return 4; + } + + TileXRCcuResourceSpec spec; + if (TileXRCcuBuildResourceSpec(info, 6, 475, 1961, 332, 2, &spec, &report, 510) != TILEXR_SUCCESS) { + std::cerr << "build spec failed: " << report.message << "\n"; + return 5; + } + if (spec.dieId != 1 || spec.missionKey != 0x059b0f03U || + spec.missionStartId != 6 || spec.missionCount != 12 || + spec.instructionStartId != 475 || spec.instructionCount != 170 || + spec.gsaStartId != 510 || spec.gsaCount != 32 || + spec.xnStartId != 1961 || spec.xnCount != 62 || + spec.ckeStartId != 332 || spec.ckeCount != 36 || + spec.channelStartId != 2 || spec.channelCount != 4) { + std::cerr << "resource spec mismatch\n"; + return 6; + } + if (report.message != "ok" || report.instructionNum != 170 || report.xnNum != 62) { + std::cerr << "report mismatch\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_rejects_invalid_basic_info_and_overflowing_windows(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_specs.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 0; + basic.missionKey = 0; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = 15U; + basic.caps.cap1 = 15U; + basic.caps.cap2 = 15U; + basic.caps.cap3 = 15U; + + TileXRCcuSpecInfo info; + TileXRCcuSpecsReport report; + if (TileXRCcuDecodeBasicInfo(basic, &info, &report) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "zero mission key was accepted\n"; + return 1; + } + if (report.message.find("missing CCU mission key") == std::string::npos) { + std::cerr << "weak mission diagnostic: " << report.message << "\n"; + return 2; + } + + basic.missionKey = 0x12345678U; + basic.caps.cap0 = (31U << 16) | 15U; + if (TileXRCcuDecodeBasicInfo(basic, &info, &report) != TILEXR_SUCCESS) { + std::cerr << "decode failed after fixing key: " << report.message << "\n"; + return 3; + } + + TileXRCcuResourceSpec spec; + if (TileXRCcuBuildResourceSpec(info, 65520, 10, 20, 30, 40, &spec, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "overflowing mission window was accepted\n"; + return 4; + } + if (report.message.find("mission resource window overflows") == std::string::npos) { + std::cerr << "weak overflow diagnostic: " << report.message << "\n"; + return 5; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_specs_are_wired_and_do_not_reference_hcomm_runtime_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = SPECS_HEADER.read_text(encoding="utf-8") + source = SPECS_SOURCE.read_text(encoding="utf-8") + allocator_header = ALLOCATOR_HEADER.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_specs.h", cmake) + self.assertIn("ccu/tilexr_ccu_specs.cpp", cmake) + self.assertIn("tilexr_ccu_resource_allocator.h", header) + self.assertIn("TileXRCcuBasicInfo", header) + self.assertIn("TileXRCcuSpecInfo", header) + self.assertIn("TileXRCcuDecodeBasicInfo", header) + self.assertIn("TileXRCcuBuildResourceSpec", header) + self.assertIn("gsaStartId", header) + self.assertIn("gsaStartId", source) + self.assertIn("TileXRCcuResourceSpec", allocator_header) + + combined = header + "\n" + source + for needle in [ + "#include #include #include +#include namespace { @@ -70,6 +71,53 @@ std::string RunCommand(const std::string& command) return output; } +std::string ShellQuote(const std::string& value) +{ + std::string quoted = "'"; + for (const char ch : value) { + if (ch == '\'') { + quoted += "'\\''"; + } else { + quoted += ch; + } + } + quoted += "'"; + return quoted; +} + +std::vector SplitPathspec(const std::string& pathspec) +{ + std::vector parts; + std::istringstream input(pathspec); + std::string part; + while (input >> part) { + parts.push_back(part); + } + return parts; +} + +std::string StripTrailingDot(const std::string& path) +{ + if (path.size() >= 2U && path.substr(path.size() - 2U) == "/.") { + return path.substr(0, path.size() - 2U); + } + return path; +} + +std::string ListFilesCommand(const std::string& pathspec) +{ + const std::string root = StripTrailingDot(RepoPath(".")); + std::string command = "if git -C " + ShellQuote(root) + + " rev-parse --is-inside-work-tree >/dev/null 2>&1; then git -C " + + ShellQuote(root) + " ls-files --cached --others --exclude-standard -- " + pathspec + "; else "; + for (const auto& part : SplitPathspec(pathspec)) { + command += "find " + ShellQuote(root + "/" + part) + + " -type f 2>/dev/null | sed " + ShellQuote("s#^" + root + "/##") + "; "; + } + command += "true; fi"; + return command; +} + void CheckNoLineContains(const std::string& description, const std::string& text, const std::string& needle) { std::istringstream lines(text); @@ -82,10 +130,72 @@ void CheckNoLineContains(const std::string& description, const std::string& text } } +void CheckTrackedFilesDoNotContain( + const std::string& description, + const std::string& gitPathspec, + const std::vector& forbiddenNeedles) +{ + const auto trackedFiles = RunCommand(ListFilesCommand(gitPathspec)); + + std::istringstream paths(trackedFiles); + std::string path; + while (std::getline(paths, path)) { + if (path.empty()) { + continue; + } + const auto text = ReadFile(path); + for (const auto& needle : forbiddenNeedles) { + const auto pos = text.find(needle); + if (pos != std::string::npos) { + std::cerr << "unexpected " << description << " in " << path + << ": " << needle << " at byte " << pos << std::endl; + ++g_failures; + } + } + } +} + +bool ContainsPath(const std::vector& paths, const std::string& path) +{ + for (const auto& item : paths) { + if (item == path) { + return true; + } + } + return false; +} + +void CheckTrackedFilesContainNeedleOnly( + const std::string& description, + const std::string& gitPathspec, + const std::string& needle, + const std::vector& allowedPaths) +{ + const auto trackedFiles = RunCommand(ListFilesCommand(gitPathspec)); + + std::istringstream paths(trackedFiles); + std::string path; + while (std::getline(paths, path)) { + if (path.empty()) { + continue; + } + const auto text = ReadFile(path); + const auto pos = text.find(needle); + if (pos != std::string::npos && !ContainsPath(allowedPaths, path)) { + std::cerr << "unexpected " << description << " in " << path + << ": " << needle << " at byte " << pos << std::endl; + ++g_failures; + } + } + + for (const auto& allowedPath : allowedPaths) { + CheckContains(allowedPath, ReadFile(allowedPath), needle); + } +} + void TestOpenSourceTarballsAreNotTracked() { - const std::string command = "git -C " + RepoPath(".") + " ls-files 3rdparty/open_source"; - const auto trackedFiles = RunCommand(command); + const auto trackedFiles = RunCommand(ListFilesCommand("3rdparty/open_source")); CheckNoLineContains("tracked open-source dependency archive", trackedFiles, ".tar.gz"); CheckNoLineContains("tracked open-source dependency archive", trackedFiles, ".tar.xz"); @@ -179,6 +289,651 @@ void TestCommBuildIncludesProfilingHeaders() CheckContains(commPath, commText, "${ASCEND_HOME_PATH}/${ARCH}-linux/pkg_inc/profiling/"); } +void TestChipNameResolverRecognizesAscend950PrVariants() +{ + const std::string path = "src/comm/tilexr_internal.cpp"; + const auto text = ReadFile(path); + + CheckContains(path, text, "ChipName ResolveChipNameFromSocVersion"); + CheckContains(path, text, "{\"Ascend950PR\", ChipName::CHIP_950PR}"); + CheckContains(path, text, "{\"Ascend950PR_\", ChipName::CHIP_950PR}"); + CheckContains(path, text, "{\"Ascend950DT_\", ChipName::CHIP_950}"); + CheckContains(path, text, "ResolveChipNameFromSocVersion(chipName)"); +} + +void TestCommRuntimeDoesNotUseHcommOrHcclV2() +{ + const std::string path = "src/comm/CMakeLists.txt"; + const auto text = ReadFile(path); + + CheckNotContains(path, text, "hcomm"); + CheckNotContains(path, text, "hccl"); + CheckNotContains(path, text, "libhccl_v2"); + CheckNotContains(path, text, "libhccl_fwk"); + CheckNotContains(path, text, "libmc2_client"); + + CheckTrackedFilesDoNotContain("hcomm or HCCL CCU runtime dependency", "src/comm src/include", { + "#include ", + "#include \"hccl.h\"", + "pkg_inc/hcomm", + "pkg_inc/hccl", + "include/hccl", + "${ARCH}-linux/include/hccl", + "${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclAllocComResourceByTiling", + "HcclCreateOpResCtx", + "HcclEngineCtx", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "HcclChannelAcquire", + "HcclGetChannelForCcu", + "HcclAllocAlgResourceCcu", + "HcclCcuKernel", + "HcommChannelNotify", + "HcommChannelFence", + "rtGetNotifyAddress", + "HrtCcuLaunch", + "HrtGetDevResAddress", + "HrtReleaseDevResAddress", + "HrtNotifyGetAddr", + "HrtRaCustomChannel", + "HrtCntNotify", + "CcuResBatchAllocator", + "CcuResRepository", + "CcuResReq", + "CcuDeviceManager", + "CcuDevMgrImp", + "CcuRepContext", + "CcuKernelMgr", + "CtxMgrImp", + "CcuTaskParam", + "CcuTaskArg", + "GeneTaskParam", + "GetMissionKey", + "SetMissionId", + "SetMissionKey", + "SetInstrId", + "SetCcuInstrInfo", + "LoadInstruction", + "AllocIns", + "AllocCke", + "AllocXn", + "COMM_ENGINE_CCU", + "COMM_PROTOCOL_UBC_CTP", + "RT_RES_TYPE_CCU_CKE", + "RT_RES_TYPE_CCU_XN", + "HCCL_SERVER_TYPE_CCU", + }); +} + +void TestRootCMakeHcclIncludesAreNotTileCommSurface() +{ + const std::string rootPath = "CMakeLists.txt"; + const std::string commPath = "src/comm/CMakeLists.txt"; + const auto rootText = ReadFile(rootPath); + const auto commText = ReadFile(commPath); + + for (const auto& privateInclude : { + "${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl", + "${ARCH}-linux/include/hccl", + "pkg_inc/hccl", + "include/hccl", + "hccl/", + }) { + CheckNotContains(rootPath, rootText, privateInclude); + CheckNotContains(commPath, commText, privateInclude); + } +} + +void TestCommDirectCcuInstallAttemptDoesNotSubmit() +{ + const std::string commHeaderPath = "src/comm/tilexr_comm.h"; + const std::string commSourcePath = "src/comm/tilexr_comm.cpp"; + const std::string ccuRuntimeHeaderPath = "src/comm/ccu/tilexr_ccu_direct_runtime.h"; + const std::string ccuRuntimeSourcePath = "src/comm/ccu/tilexr_ccu_direct_runtime.cpp"; + const auto commHeaderText = ReadFile(commHeaderPath); + const auto commSourceText = ReadFile(commSourcePath); + const auto ccuRuntimeHeaderText = ReadFile(ccuRuntimeHeaderPath); + const auto ccuRuntimeSourceText = ReadFile(ccuRuntimeSourcePath); + + CheckContains(commHeaderPath, commHeaderText, "PrepareDirectCcuInstallAttempt"); + CheckContains(commHeaderPath, commHeaderText, "FillDirectCcuLowerLayerPlanFromAllocation"); + CheckContains(commHeaderPath, commHeaderText, "PrepareDirectCcuLowerLayerPlanCallback"); + CheckContains(commHeaderPath, commHeaderText, "std::unique_ptr ccuDirectRuntime_"); + CheckContains(commSourcePath, commSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckContains(commSourcePath, commSourceText, "int TileXRComm::InitDirectCcuRuntime"); + CheckContains(commSourcePath, commSourceText, "int TileXRComm::PrepareDirectCcuInstallAttempt"); + CheckContains(commSourcePath, commSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); + CheckContains(commSourcePath, commSourceText, "TileXRCcuMakeAclDeviceMemoryOps()"); + CheckContains(commSourcePath, commSourceText, "next.lowerLayerPlan = nullptr"); + CheckContains( + commSourcePath, + commSourceText, + "next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback"); + CheckContains(commSourcePath, commSourceText, "next.lowerLayerPlanUserData = this"); + CheckContains(commSourcePath, commSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); + CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "int CreateDriverAdapter("); + CheckContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, "int TileXRCcuDirectRuntime::CreateDriverAdapter"); + CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "TileXRCcuHccpLoader"); + for (const auto& forbiddenUdmaCcuCall : { + std::string("udmaTransport_->") + "CreateCcuDriverAdapter", + std::string("udmaTransport_->") + "QueryCcuBasicInfo", + std::string("udmaTransport_->") + "RegisterCcuResourceRmaBuffer", + std::string("udmaTransport_->") + "ExportLocalCcuRmaBuffer", + std::string("udmaTransport_->") + "ExportRemoteCcuRmaBuffers", + std::string("udmaTransport_->") + "ExportLowerLayerTransportSnapshot", + }) { + CheckNotContains(commSourcePath, commSourceText, forbiddenUdmaCcuCall); + } + + const auto initUdmaBegin = commSourceText.find("int TileXRComm::InitUDMA"); + const auto initDirectCcuBegin = commSourceText.find("int TileXRComm::InitDirectCcuRuntime"); + if (initUdmaBegin == std::string::npos || initDirectCcuBegin == std::string::npos || + initUdmaBegin >= initDirectCcuBegin) { + std::cerr << commSourcePath << ": cannot isolate InitUDMA body" << std::endl; + ++g_failures; + } else { + const auto initUdmaBody = commSourceText.substr(initUdmaBegin, initDirectCcuBegin - initUdmaBegin); + CheckNotContains(commSourcePath, initUdmaBody, "RefreshDirectCcuBasicInfo"); + CheckNotContains(commSourcePath, initUdmaBody, "ResetDirectCcuBasicInfo"); + } + + const auto registerUdmaBegin = commSourceText.find("int TileXRComm::RegisterUDMAMemory"); + const auto unregisterUdmaBegin = commSourceText.find("int TileXRComm::UnregisterUDMAMemory"); + const auto getUdmaRegistryBegin = commSourceText.find("GM_ADDR TileXRComm::GetUDMARegistryPtr"); + if (registerUdmaBegin == std::string::npos || unregisterUdmaBegin == std::string::npos || + getUdmaRegistryBegin == std::string::npos || registerUdmaBegin >= unregisterUdmaBegin || + unregisterUdmaBegin >= getUdmaRegistryBegin) { + std::cerr << commSourcePath << ": cannot isolate UDMA memory registration bodies" << std::endl; + ++g_failures; + } else { + const auto registerUdmaBody = commSourceText.substr( + registerUdmaBegin, unregisterUdmaBegin - registerUdmaBegin); + const auto unregisterUdmaBody = commSourceText.substr( + unregisterUdmaBegin, getUdmaRegistryBegin - unregisterUdmaBegin); + CheckNotContains(commSourcePath, registerUdmaBody, "ResetDirectCcuLowerLayerPlan"); + CheckNotContains(commSourcePath, unregisterUdmaBody, "ResetDirectCcuLowerLayerPlan"); + } + + for (const auto& forbidden : { + "TileXRCcuPrepareSubmitTasks", + "TileXRCcuSubmitTask", + "rtCCULaunch", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "libhcomm", + "libhccl_v2", + }) { + CheckNotContains(commHeaderPath, commHeaderText, forbidden); + CheckNotContains(commSourcePath, commSourceText, forbidden); + CheckNotContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, forbidden); + CheckNotContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, forbidden); + } +} + +void TestCcuRuntimeSubmitBoundaryUsesPublicRuntimeOnly() +{ + const std::string cmakePath = "src/comm/CMakeLists.txt"; + const std::string abiConstantsHeaderPath = "src/comm/ccu/tilexr_ccu_abi_constants.h"; + const std::string barrierHeaderPath = "src/comm/ccu/tilexr_ccu_barrier_program.h"; + const std::string barrierSourcePath = "src/comm/ccu/tilexr_ccu_barrier_program.cpp"; + const std::string directHeaderPath = "src/comm/ccu/tilexr_ccu_direct_orchestrator.h"; + const std::string directSourcePath = "src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp"; + const std::string directRuntimeHeaderPath = "src/comm/ccu/tilexr_ccu_direct_runtime.h"; + const std::string directRuntimeSourcePath = "src/comm/ccu/tilexr_ccu_direct_runtime.cpp"; + const std::string driverHeaderPath = "src/comm/ccu/tilexr_ccu_driver_adapter.h"; + const std::string driverSourcePath = "src/comm/ccu/tilexr_ccu_driver_adapter.cpp"; + const std::string hccpTypesHeaderPath = "src/comm/ccu/tilexr_ccu_hccp_types.h"; + const std::string hccpLoaderHeaderPath = "src/comm/ccu/tilexr_ccu_hccp_loader.h"; + const std::string hccpLoaderSourcePath = "src/comm/ccu/tilexr_ccu_hccp_loader.cpp"; + const std::string installHeaderPath = "src/comm/ccu/tilexr_ccu_install_provider.h"; + const std::string installSourcePath = "src/comm/ccu/tilexr_ccu_install_provider.cpp"; + const std::string lowerLayerPlanHeaderPath = "src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h"; + const std::string lowerLayerPlanSourcePath = "src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp"; + const std::string packageHeaderPath = "src/comm/ccu/tilexr_ccu_launch_package.h"; + const std::string packageSourcePath = "src/comm/ccu/tilexr_ccu_launch_package.cpp"; + const std::string microcodeHeaderPath = "src/comm/ccu/tilexr_ccu_microcode.h"; + const std::string microcodeSourcePath = "src/comm/ccu/tilexr_ccu_microcode.cpp"; + const std::string planHeaderPath = "src/comm/ccu/tilexr_ccu_producer_plan.h"; + const std::string planSourcePath = "src/comm/ccu/tilexr_ccu_producer_plan.cpp"; + const std::string providerHeaderPath = "src/comm/ccu/tilexr_ccu_provider.h"; + const std::string providerSourcePath = "src/comm/ccu/tilexr_ccu_provider.cpp"; + const std::string raProviderHeaderPath = "src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.h"; + const std::string raProviderSourcePath = "src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp"; + const std::string repositoryHeaderPath = "src/comm/ccu/tilexr_ccu_repository.h"; + const std::string repositorySourcePath = "src/comm/ccu/tilexr_ccu_repository.cpp"; + const std::string allocatorHeaderPath = "src/comm/ccu/tilexr_ccu_resource_allocator.h"; + const std::string allocatorSourcePath = "src/comm/ccu/tilexr_ccu_resource_allocator.cpp"; + const std::string specsHeaderPath = "src/comm/ccu/tilexr_ccu_specs.h"; + const std::string specsSourcePath = "src/comm/ccu/tilexr_ccu_specs.cpp"; + const std::string headerPath = "src/comm/ccu/tilexr_ccu_runtime.h"; + const std::string sourcePath = "src/comm/ccu/tilexr_ccu_runtime.cpp"; + const auto cmakeText = ReadFile(cmakePath); + const auto abiConstantsHeaderText = ReadFile(abiConstantsHeaderPath); + const auto barrierHeaderText = ReadFile(barrierHeaderPath); + const auto barrierSourceText = ReadFile(barrierSourcePath); + const auto directHeaderText = ReadFile(directHeaderPath); + const auto directSourceText = ReadFile(directSourcePath); + const auto directRuntimeHeaderText = ReadFile(directRuntimeHeaderPath); + const auto directRuntimeSourceText = ReadFile(directRuntimeSourcePath); + const auto driverHeaderText = ReadFile(driverHeaderPath); + const auto driverSourceText = ReadFile(driverSourcePath); + const auto hccpTypesHeaderText = ReadFile(hccpTypesHeaderPath); + const auto hccpLoaderHeaderText = ReadFile(hccpLoaderHeaderPath); + const auto hccpLoaderSourceText = ReadFile(hccpLoaderSourcePath); + const auto installHeaderText = ReadFile(installHeaderPath); + const auto installSourceText = ReadFile(installSourcePath); + const auto lowerLayerPlanHeaderText = ReadFile(lowerLayerPlanHeaderPath); + const auto lowerLayerPlanSourceText = ReadFile(lowerLayerPlanSourcePath); + const auto packageHeaderText = ReadFile(packageHeaderPath); + const auto packageSourceText = ReadFile(packageSourcePath); + const auto microcodeHeaderText = ReadFile(microcodeHeaderPath); + const auto microcodeSourceText = ReadFile(microcodeSourcePath); + const auto planHeaderText = ReadFile(planHeaderPath); + const auto planSourceText = ReadFile(planSourcePath); + const auto providerHeaderText = ReadFile(providerHeaderPath); + const auto providerSourceText = ReadFile(providerSourcePath); + const auto raProviderHeaderText = ReadFile(raProviderHeaderPath); + const auto raProviderSourceText = ReadFile(raProviderSourcePath); + const auto repositoryHeaderText = ReadFile(repositoryHeaderPath); + const auto repositorySourceText = ReadFile(repositorySourcePath); + const auto allocatorHeaderText = ReadFile(allocatorHeaderPath); + const auto allocatorSourceText = ReadFile(allocatorSourcePath); + const auto specsHeaderText = ReadFile(specsHeaderPath); + const auto specsSourceText = ReadFile(specsSourcePath); + const auto headerText = ReadFile(headerPath); + const auto sourceText = ReadFile(sourcePath); + + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_abi_constants.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_barrier_program.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_barrier_program.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_driver_adapter.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_direct_orchestrator.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_direct_orchestrator.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_direct_runtime.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_direct_runtime.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_driver_adapter.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_hccp_types.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_hccp_loader.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_hccp_loader.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_launch_package.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_launch_package.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_install_provider.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_install_provider.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_lower_layer_plan_builder.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_lower_layer_plan_builder.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_microcode.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_microcode.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_producer_plan.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_producer_plan.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_provider.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_provider.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_ra_custom_channel_provider.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_ra_custom_channel_provider.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_repository.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_repository.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_resource_allocator.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_resource_allocator.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_runtime.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_runtime.cpp"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_specs.h"); + CheckContains(cmakePath, cmakeText, "ccu/tilexr_ccu_specs.cpp"); + + CheckContains(abiConstantsHeaderPath, abiConstantsHeaderText, "TILEXR_CCU_EID_BYTES"); + CheckContains(abiConstantsHeaderPath, abiConstantsHeaderText, "TILEXR_CCU_REMOTE_CCU_VA_SHIFT"); + CheckNotContains(abiConstantsHeaderPath, abiConstantsHeaderText, "runtime/kernel.h"); + CheckNotContains(abiConstantsHeaderPath, abiConstantsHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(directHeaderPath, directHeaderText, "TileXRCcuDirectInstallOptions"); + CheckContains(directHeaderPath, directHeaderText, "TileXRCcuDirectInstallAttempt"); + CheckContains(directHeaderPath, directHeaderText, "TileXRCcuDirectInstallReport"); + CheckContains(directHeaderPath, directHeaderText, "TileXRCcuRunDirectInstallAttempt"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuDecodeBasicInfo"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuBuildResourceSpec"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuResourceAllocator"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuBuildLaunchPackage"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuBindLaunchPackageInstallScope"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuBuildInstallManifest"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuInstallHardware"); + CheckContains(directSourcePath, directSourceText, "TileXRCcuPrepareSubmitTasks"); + CheckNotContains(directHeaderPath, directHeaderText, "runtime/kernel.h"); + CheckNotContains(directHeaderPath, directHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(directRuntimeHeaderPath, directRuntimeHeaderText, "TileXRCcuDirectRuntime"); + CheckContains(directRuntimeHeaderPath, directRuntimeHeaderText, "QueryBasicInfo"); + CheckContains(directRuntimeHeaderPath, directRuntimeHeaderText, "CreateDriverAdapter"); + CheckContains(directRuntimeHeaderPath, directRuntimeHeaderText, "RegisterCcuResourceRmaBuffer"); + CheckContains(directRuntimeSourcePath, directRuntimeSourceText, "TileXRCcuRaCustomChannelProvider"); + CheckContains(directRuntimeSourcePath, directRuntimeSourceText, "loader_.Load"); + CheckContains(directRuntimeSourcePath, directRuntimeSourceText, "loader_.ResolveDevicePhyId"); + CheckContains(directRuntimeSourcePath, directRuntimeSourceText, "loader_.RaCustomChannel"); + CheckNotContains(directRuntimeHeaderPath, directRuntimeHeaderText, "udma/"); + CheckNotContains(directRuntimeSourcePath, directRuntimeSourceText, "udma/"); + + CheckContains(driverHeaderPath, driverHeaderText, "TileXRCcuDriverAdapter"); + CheckContains(driverHeaderPath, driverHeaderText, "TileXRCcuCustomChannelIn"); + CheckContains(driverHeaderPath, driverHeaderText, "TILEXR_CCU_U_OP_GET_BASIC_INFO"); + CheckContains(driverHeaderPath, driverHeaderText, "TILEXR_CCU_U_OP_GET_DIE_WORKING"); + CheckContains(driverHeaderPath, driverHeaderText, "GetBasicInfo"); + CheckContains(driverHeaderPath, driverHeaderText, "GetDieEnabled"); + CheckNotContains(driverHeaderPath, driverHeaderText, "runtime/kernel.h"); + CheckNotContains(driverHeaderPath, driverHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallRequest"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallManifest"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallManifestReport"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallRequirement"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallRequirementKind"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuBuildInstallManifest"); + CheckContains(installHeaderPath, installHeaderText, "provider"); + CheckContains(installHeaderPath, installHeaderText, "manifest"); + CheckContains(installHeaderPath, installHeaderText, "installAttemptReceiptRequired"); + CheckContains(installHeaderPath, installHeaderText, "requiredEvidenceKind"); + CheckContains(installHeaderPath, installHeaderText, "requiredEvidenceSurface"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallStepEvidence"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallProviderReport"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuBuildInstallEvidence"); + CheckContains(installHeaderPath, installHeaderText, "TileXRCcuInstallHardware"); + CheckContains(installHeaderPath, installHeaderText, "installAttemptReceiptId"); + CheckContains(installSourcePath, installSourceText, "ValidateInstallRequestScope"); + CheckContains(installSourcePath, installSourceText, "ValidateInstallManifestScope"); + CheckContains(installSourcePath, installSourceText, "ValidateInstallRequestManifest"); + CheckContains(installSourcePath, installSourceText, "ValidatePublicVerifiedStepScope"); + CheckContains(installHeaderPath, installHeaderText, "requiredInstallSurfaceCount"); + CheckContains(installHeaderPath, installHeaderText, "publicVerifiedInstallSurfaceCount"); + CheckContains(installHeaderPath, installHeaderText, "missingInstallSurfaceCount"); + CheckContains(installSourcePath, installSourceText, "FillManifestInstallSurfaceCounts"); + CheckContains(installSourcePath, installSourceText, "launch install scope is stale"); + CheckContains(installSourcePath, installSourceText, "install manifest requirement kind mismatch"); + CheckContains(installSourcePath, installSourceText, "install manifest hardware requirement mismatch"); + CheckContains(installSourcePath, installSourceText, "install manifest mission requirement mismatch"); + CheckContains(installSourcePath, installSourceText, "install manifest channel requirement mismatch"); + CheckContains(installSourcePath, installSourceText, "public verified evidence scope is incomplete"); + CheckContains(installSourcePath, installSourceText, "missing CCU repository image for install manifest"); + CheckContains(installSourcePath, installSourceText, "install generated CCU repository image"); + CheckContains(installSourcePath, installSourceText, "bind CCU channel routes for sync resources"); + CheckContains(installHeaderPath, installHeaderText, "localWaitCke"); + CheckContains(installSourcePath, installSourceText, "local wait CKE"); + CheckContains(installSourcePath, installSourceText, "remote notify CKE"); + CheckContains(installSourcePath, installSourceText, "ValidateInstallReceipt"); + CheckContains(installSourcePath, installSourceText, "install attempt did not succeed"); + CheckContains(installSourcePath, installSourceText, "install attempt receipt mismatch"); + CheckContains(installSourcePath, installSourceText, "device scope mismatch"); + CheckContains(installSourcePath, installSourceText, "rank scope mismatch"); + CheckContains(installSourcePath, installSourceText, "provider scope mismatch"); + CheckContains(installHeaderPath, installHeaderText, "tilexr_ccu_provider.h"); + CheckNotContains(installHeaderPath, installHeaderText, "runtime/kernel.h"); + CheckNotContains(installHeaderPath, installHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(lowerLayerPlanHeaderPath, lowerLayerPlanHeaderText, "TileXRCcuLowerLayerPlanSpec"); + CheckContains(lowerLayerPlanHeaderPath, lowerLayerPlanHeaderText, "TileXRCcuBuildLowerLayerInstallPlan"); + CheckContains(lowerLayerPlanHeaderPath, lowerLayerPlanHeaderText, "remoteNotifyCke"); + CheckContains(lowerLayerPlanSourcePath, lowerLayerPlanSourceText, "TileXRCcuBuildPfeCtx"); + CheckContains(lowerLayerPlanSourcePath, lowerLayerPlanSourceText, "TileXRCcuBuildLocalJettyCtx"); + CheckContains(lowerLayerPlanSourcePath, lowerLayerPlanSourceText, "TileXRCcuBuildChannelCtxV1"); + CheckContains(lowerLayerPlanSourcePath, lowerLayerPlanSourceText, "localWaitCke"); + CheckNotContains(lowerLayerPlanHeaderPath, lowerLayerPlanHeaderText, "runtime/kernel.h"); + CheckNotContains(lowerLayerPlanHeaderPath, lowerLayerPlanHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuLaunchPackage"); + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuLaunchPackageReport"); + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuBuildLaunchPackage"); + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuComputeLaunchPackageFingerprint"); + CheckContains(packageSourcePath, packageSourceText, "TileXRCcuComputeLaunchPackageFingerprint"); + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuLaunchInstallScope"); + CheckContains(packageHeaderPath, packageHeaderText, "installScope"); + CheckContains(packageHeaderPath, packageHeaderText, "TileXRCcuBindLaunchPackageInstallScope"); + CheckContains(packageSourcePath, packageSourceText, "TileXRCcuBindLaunchPackageInstallScope"); + CheckContains(packageHeaderPath, packageHeaderText, "requiresHardwareInstall"); + CheckContains(packageHeaderPath, packageHeaderText, "tilexr_ccu_repository.h"); + CheckNotContains(packageHeaderPath, packageHeaderText, "runtime/kernel.h"); + CheckNotContains(packageHeaderPath, packageHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(microcodeHeaderPath, microcodeHeaderText, "struct TileXRCcuInstr"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "struct TileXRCcuSyncXnSpec"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "struct TileXRCcuCkeSpec"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "TileXRCcuEncodeLoadSqeArgsToX"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "TileXRCcuEncodeSyncXn"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "TileXRCcuEncodeSetCke"); + CheckContains(microcodeHeaderPath, microcodeHeaderText, "TileXRCcuEncodeClearCke"); + CheckContains(microcodeSourcePath, microcodeSourceText, "0x0001U"); + CheckContains(microcodeSourcePath, microcodeSourceText, "0x0802U"); + CheckContains(microcodeSourcePath, microcodeSourceText, "0x0804U"); + CheckContains(microcodeSourcePath, microcodeSourceText, "0x100dU"); + CheckNotContains(microcodeHeaderPath, microcodeHeaderText, "runtime/kernel.h"); + CheckNotContains(microcodeHeaderPath, microcodeHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(barrierHeaderPath, barrierHeaderText, "TileXRCcuBarrierSyncSpec"); + CheckContains(barrierHeaderPath, barrierHeaderText, "TileXRCcuBarrierProgramReport"); + CheckContains(barrierHeaderPath, barrierHeaderText, "TileXRCcuBuildBarrierProgram"); + CheckContains(barrierSourcePath, barrierSourceText, "TileXRCcuEncodeSyncXn"); + CheckContains(barrierSourcePath, barrierSourceText, "TileXRCcuEncodeClearCke"); + CheckNotContains(barrierHeaderPath, barrierHeaderText, "runtime/kernel.h"); + CheckNotContains(barrierHeaderPath, barrierHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuProducerPlan"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuValidateProducerPlan"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuBuildTasks"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuBuildMicrocode"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuSyncResource"); + CheckContains(planHeaderPath, planHeaderText, "localWaitCke"); + CheckContains(planHeaderPath, planHeaderText, "localWaitMask"); + CheckContains(planHeaderPath, planHeaderText, "remoteNotifyMask"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuInstructionWindow"); + CheckContains(planHeaderPath, planHeaderText, "TileXRCcuProgram"); + CheckContains(planHeaderPath, planHeaderText, "tilexr_ccu_barrier_program.h"); + CheckContains(planSourcePath, planSourceText, "TileXRCcuBuildBarrierProgram"); + CheckContains(planSourcePath, planSourceText, "spec.localWaitCke"); + CheckNotContains(planHeaderPath, planHeaderText, "runtime/kernel.h"); + CheckNotContains(planHeaderPath, planHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(providerHeaderPath, providerHeaderText, "TileXRCcuHardwareInstallEvidence"); + CheckContains(providerHeaderPath, providerHeaderText, "TileXRCcuValidateHardwareInstall"); + CheckContains(providerHeaderPath, providerHeaderText, "TileXRCcuPrepareSubmitTasks"); + CheckContains(providerHeaderPath, providerHeaderText, "submitReady"); + CheckContains(providerHeaderPath, providerHeaderText, "packageFingerprint"); + CheckContains(providerHeaderPath, providerHeaderText, "deviceId"); + CheckContains(providerHeaderPath, providerHeaderText, "rank"); + CheckContains(providerHeaderPath, providerHeaderText, "provider"); + CheckContains(providerHeaderPath, providerHeaderText, "installAttemptReceiptId"); + CheckContains(providerSourcePath, providerSourceText, "package fingerprint mismatch"); + CheckContains(providerSourcePath, providerSourceText, "launch install scope is not bound"); + CheckContains(providerSourcePath, providerSourceText, "device scope mismatch"); + CheckContains(providerSourcePath, providerSourceText, "rank scope mismatch"); + CheckContains(providerSourcePath, providerSourceText, "provider scope mismatch"); + CheckContains(providerSourcePath, providerSourceText, "install attempt receipt is missing"); + CheckContains(providerSourcePath, providerSourceText, "install attempt receipt mismatch"); + CheckContains(providerHeaderPath, providerHeaderText, "tilexr_ccu_launch_package.h"); + CheckNotContains(providerHeaderPath, providerHeaderText, "runtime/kernel.h"); + CheckNotContains(providerHeaderPath, providerHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(raProviderHeaderPath, raProviderHeaderText, "TileXRCcuRaCustomChannelProvider"); + CheckContains(raProviderHeaderPath, raProviderHeaderText, "CreateAdapter"); + CheckContains(raProviderHeaderPath, raProviderHeaderText, "TileXRCcuRaCustomChannelFunc"); + CheckContains(raProviderSourcePath, raProviderSourceText, "TILEXR_CCU_NETWORK_OFFLINE"); + CheckNotContains(raProviderHeaderPath, raProviderHeaderText, "udma/"); + CheckNotContains(raProviderHeaderPath, raProviderHeaderText, "runtime/kernel.h"); + CheckNotContains(raProviderHeaderPath, raProviderHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(hccpTypesHeaderPath, hccpTypesHeaderText, "TileXRCcuRaInfo"); + CheckContains(hccpTypesHeaderPath, hccpTypesHeaderText, "TileXRCcuRaCustomChannelFunc"); + CheckContains(hccpLoaderHeaderPath, hccpLoaderHeaderText, "TileXRCcuHccpLoader"); + CheckContains(hccpLoaderSourcePath, hccpLoaderSourceText, "dlopen(\"libra.so\", RTLD_NOW)"); + CheckContains(hccpLoaderSourcePath, hccpLoaderSourceText, "RaCustomChannel"); + CheckContains(hccpLoaderSourcePath, hccpLoaderSourceText, "rtGetDevicePhyIdByIndex"); + + CheckContains(repositoryHeaderPath, repositoryHeaderText, "TileXRCcuRepositoryImage"); + CheckContains(repositoryHeaderPath, repositoryHeaderText, "TileXRCcuRepositoryReport"); + CheckContains(repositoryHeaderPath, repositoryHeaderText, "TileXRCcuBuildRepositoryImage"); + CheckContains(repositoryHeaderPath, repositoryHeaderText, "missionOffset"); + CheckContains(repositoryHeaderPath, repositoryHeaderText, "sqeLoadOffset"); + CheckContains(repositoryHeaderPath, repositoryHeaderText, "syncOffset"); + CheckNotContains(repositoryHeaderPath, repositoryHeaderText, "runtime/kernel.h"); + CheckNotContains(repositoryHeaderPath, repositoryHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(allocatorHeaderPath, allocatorHeaderText, "TileXRCcuResourceAllocator"); + CheckContains(allocatorHeaderPath, allocatorHeaderText, "TileXRCcuResourceSpec"); + CheckContains(allocatorHeaderPath, allocatorHeaderText, "TileXRCcuResourceRequest"); + CheckContains(allocatorSourcePath, allocatorSourceText, "TileXRCcuValidateProducerPlan"); + CheckNotContains(allocatorHeaderPath, allocatorHeaderText, "runtime/kernel.h"); + CheckNotContains(allocatorHeaderPath, allocatorHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(specsHeaderPath, specsHeaderText, "TileXRCcuBasicInfo"); + CheckContains(specsHeaderPath, specsHeaderText, "TileXRCcuDecodeBasicInfo"); + CheckContains(specsHeaderPath, specsHeaderText, "TileXRCcuBuildResourceSpec"); + CheckContains(specsSourcePath, specsSourceText, "TILEXR_CCU_V1_XN_RESOURCE_OFFSET"); + CheckNotContains(specsHeaderPath, specsHeaderText, "runtime/kernel.h"); + CheckNotContains(specsHeaderPath, specsHeaderText, "rtCcuTaskInfo_t"); + + CheckContains(headerPath, headerText, "TILEXR_CCU_SQE_ARGS_LEN"); + CheckContains(headerPath, headerText, "struct TileXRCcuTask"); + CheckContains(headerPath, headerText, "TileXRCcuValidateTask"); + CheckContains(headerPath, headerText, "TileXRCcuSubmitTask"); + CheckNotContains(headerPath, headerText, "runtime/kernel.h"); + CheckNotContains(headerPath, headerText, "rtCcuTaskInfo_t"); + + CheckContains(sourcePath, sourceText, "#include "); + CheckContains(sourcePath, sourceText, "rtCcuTaskInfo_t runtimeTask"); + CheckContains(sourcePath, sourceText, "rtCCULaunch(&runtimeTask, stream)"); + CheckContains(sourcePath, sourceText, "RT_CCU_INST_CNT_INVALID"); + CheckContains(sourcePath, sourceText, "RT_CCU_INST_START_MAX"); + CheckContains(sourcePath, sourceText, "task.argSize != 1 && task.argSize != TILEXR_CCU_SQE_ARGS_LEN"); + CheckContains(sourcePath, sourceText, "TILEXR_ERROR_MKIRT"); + + for (const auto& privateNeedle : { + "#include ", + "#include \"hccl.h\"", + "pkg_inc/hcomm", + "pkg_inc/hccl", + "include/hccl", + "${ARCH}-linux/include/hccl", + "${ASCEND_HOME_PATH}/${ARCH}-linux/include/hccl", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclCcuKernel", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + "HcclChannelAcquire", + "HcclGetChannelForCcu", + "CcuResBatchAllocator", + "CcuResRepository", + "GetMissionKey", + "SetMissionId", + "SetMissionKey", + "SetInstrId", + "SetCcuInstrInfo", + "LoadInstruction", + "AllocIns", + "AllocXn", + "AllocCke", + "RT_RES_TYPE_CCU_CKE", + "RT_RES_TYPE_CCU_XN", + "dlopen", + "dlsym", + }) { + CheckNotContains(directSourcePath, directSourceText, privateNeedle); + CheckNotContains(abiConstantsHeaderPath, abiConstantsHeaderText, privateNeedle); + CheckNotContains(directHeaderPath, directHeaderText, privateNeedle); + CheckNotContains(directRuntimeSourcePath, directRuntimeSourceText, privateNeedle); + CheckNotContains(directRuntimeHeaderPath, directRuntimeHeaderText, privateNeedle); + CheckNotContains(driverSourcePath, driverSourceText, privateNeedle); + CheckNotContains(driverHeaderPath, driverHeaderText, privateNeedle); + CheckNotContains(barrierSourcePath, barrierSourceText, privateNeedle); + CheckNotContains(barrierHeaderPath, barrierHeaderText, privateNeedle); + CheckNotContains(installSourcePath, installSourceText, privateNeedle); + CheckNotContains(installHeaderPath, installHeaderText, privateNeedle); + CheckNotContains(packageSourcePath, packageSourceText, privateNeedle); + CheckNotContains(packageHeaderPath, packageHeaderText, privateNeedle); + CheckNotContains(microcodeSourcePath, microcodeSourceText, privateNeedle); + CheckNotContains(microcodeHeaderPath, microcodeHeaderText, privateNeedle); + CheckNotContains(planSourcePath, planSourceText, privateNeedle); + CheckNotContains(planHeaderPath, planHeaderText, privateNeedle); + CheckNotContains(providerSourcePath, providerSourceText, privateNeedle); + CheckNotContains(providerHeaderPath, providerHeaderText, privateNeedle); + CheckNotContains(raProviderSourcePath, raProviderSourceText, privateNeedle); + CheckNotContains(raProviderHeaderPath, raProviderHeaderText, privateNeedle); + CheckNotContains(repositorySourcePath, repositorySourceText, privateNeedle); + CheckNotContains(repositoryHeaderPath, repositoryHeaderText, privateNeedle); + CheckNotContains(allocatorSourcePath, allocatorSourceText, privateNeedle); + CheckNotContains(allocatorHeaderPath, allocatorHeaderText, privateNeedle); + CheckNotContains(specsSourcePath, specsSourceText, privateNeedle); + CheckNotContains(specsHeaderPath, specsHeaderText, privateNeedle); + CheckNotContains(sourcePath, sourceText, privateNeedle); + CheckNotContains(headerPath, headerText, privateNeedle); + } + + for (const auto& loaderPrivateNeedle : { + "#include ", + "#include \"hccl.h\"", + "pkg_inc/hcomm", + "pkg_inc/hccl", + "include/hccl", + "libhcomm", + "libhccl_v2", + "libhccl_fwk", + "libmc2_client", + "HcclGetCcuTaskInfo", + "HcomGetCcuTaskInfo", + }) { + CheckNotContains(hccpTypesHeaderPath, hccpTypesHeaderText, loaderPrivateNeedle); + CheckNotContains(hccpLoaderHeaderPath, hccpLoaderHeaderText, loaderPrivateNeedle); + CheckNotContains(hccpLoaderSourcePath, hccpLoaderSourceText, loaderPrivateNeedle); + } + + CheckTrackedFilesContainNeedleOnly( + "public CCU runtime launch ABI", + "src/comm src/include", + "rtCCULaunch", + {sourcePath}); + CheckTrackedFilesContainNeedleOnly( + "public CCU runtime kernel header", + "src/comm src/include", + "runtime/kernel.h", + {sourcePath}); +} + +void TestDirectCcuHeadersOwnStandardDependencies() +{ + const std::string commHeaderPath = "src/comm/tilexr_comm.h"; + const std::string abiConstantsPath = "src/comm/ccu/tilexr_ccu_abi_constants.h"; + const std::string hccpTypesPath = "src/comm/ccu/tilexr_ccu_hccp_types.h"; + const auto commHeaderText = ReadFile(commHeaderPath); + const auto abiConstantsText = ReadFile(abiConstantsPath); + const auto hccpTypesText = ReadFile(hccpTypesPath); + + CheckContains(commHeaderPath, commHeaderText, "#include "); + CheckContains(abiConstantsPath, abiConstantsText, "#include "); + CheckContains(hccpTypesPath, hccpTypesText, "#include "); + CheckContains(hccpTypesPath, hccpTypesText, "tilexr_ccu_abi_constants.h"); + CheckContains(hccpTypesPath, hccpTypesText, "TileXRCcuEndpointRouteProviderFunc"); + CheckNotContains(hccpTypesPath, hccpTypesText, "tilexr_ccu_driver_adapter.h"); + CheckNotContains(hccpTypesPath, hccpTypesText, "tilexr_ccu_lower_layer_payloads.h"); + CheckNotContains(abiConstantsPath, abiConstantsText, "udma/"); + CheckNotContains(hccpTypesPath, hccpTypesText, "udma/"); + CheckNotContains(hccpTypesPath, hccpTypesText, ""); + CheckNotContains(hccpTypesPath, hccpTypesText, "dlopen"); + CheckNotContains(hccpTypesPath, hccpTypesText, "dlsym"); + CheckNotContains(hccpTypesPath, hccpTypesText, "std::string"); +} + } // namespace int main() @@ -191,6 +946,12 @@ int main() TestRuntimeEnvDoesNotPrependCannDevlib(); TestRootCMakeRespectsAscendDriverOverride(); TestCommBuildIncludesProfilingHeaders(); + TestChipNameResolverRecognizesAscend950PrVariants(); + TestCommRuntimeDoesNotUseHcommOrHcclV2(); + TestRootCMakeHcclIncludesAreNotTileCommSurface(); + TestCommDirectCcuInstallAttemptDoesNotSubmit(); + TestCcuRuntimeSubmitBoundaryUsesPublicRuntimeOnly(); + TestDirectCcuHeadersOwnStandardDependencies(); if (g_failures != 0) { std::cerr << g_failures << " TileXR source guard checks failed" << std::endl; From c09eca077d71591fa7eedd9fdafa483890bdd3b3 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Mon, 6 Jul 2026 19:13:25 +0800 Subject: [PATCH 02/41] docs: design CCU API isolation --- .../2026-07-06-ccu-api-isolation-design.md | 237 ++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-06-ccu-api-isolation-design.md diff --git a/docs/superpowers/specs/2026-07-06-ccu-api-isolation-design.md b/docs/superpowers/specs/2026-07-06-ccu-api-isolation-design.md new file mode 100644 index 00000000..fe4b478e --- /dev/null +++ b/docs/superpowers/specs/2026-07-06-ccu-api-isolation-design.md @@ -0,0 +1,237 @@ +# TileXR CCU API Isolation Design + +## Context + +The `codex/direct-ccu-rebased` branch currently implements direct CCU support under `src/comm/ccu`, but the public and +core communication surfaces are still polluted by CCU-specific declarations: + +- `src/include/tilexr_api.h` declares direct CCU handles, constants, option structs, report structs, task structs, and + entry points. +- `src/comm/comm_wrap.cpp` contains both generic TileXR C API wrappers and direct CCU public API bridge code. +- `src/comm/tilexr_comm.h` directly includes multiple CCU internal headers and exposes CCU-specific methods and state + on `TileXRComm`. + +This makes direct CCU look like part of the baseline TileXR communication API. The desired model is that CCU remains +available in the same `tile-comm` library, but it is an internal communication backend. Users should only enable the +backend during communicator setup and select it through normal collective communication controls. They should not call +CCU resource, repository, task-prepare, submit, or readback APIs directly. + +## Goals + +- Remove all CCU-specific public declarations from `tilexr_api.h`. +- Remove the direct CCU public C API bridge from `comm_wrap.cpp`. +- Hide CCU runtime state and orchestration behind a CCU-owned context instead of exposing it directly on + `TileXRComm`. +- Model CCU as an internal C++ backend class rather than a user-facing C API. +- Add a generic collective backend selection surface so users can request `AUTO`, `AIV`, `UDMA`, or `CCU` collective + execution without seeing backend resource details. +- Keep the build and link model simple: direct CCU stays in `libtile-comm.so`; no separate shared library is introduced + in this change. +- Preserve current direct CCU behavior for internal probes and hardware validation through CCU-owned test hooks or + internal C++ helpers. + +## Non-Goals + +- Do not split CCU into a new library or optional package target. +- Do not redesign the CCU lower-layer resource allocation, repository install, or runtime launch algorithms. +- Do not promote direct CCU prepare/submit/readback operations as public user APIs in this pass. +- Do not introduce a generic backend manager abstraction before there is more than one optional collective backend. +- Do not change non-CCU TileXR communication, UDMA, SDMA, collectives, or EP behavior. + +## Public API Layout + +`src/include/tilexr_api.h` remains the baseline TileXR C API header. It should contain only generic communication, +UDMA, SDMA, DFX, and common lifecycle declarations. It must not contain these strings: + +- `CCU` +- `Ccu` +- `DirectCcu` +- `TILEXR_DIRECT_CCU` + +No installed `tilexr_ccu_api.h` is introduced. A CCU-specific public header would be premature because the intended +user flow does not include direct CCU task preparation or submission. + +The only user-visible direction for CCU should be through generic runtime configuration surfaces: + +- communicator initialization can enable optional backends without exposing CCU resource structs or task descriptors; +- collective calls can select a communication mode/backend using the collective API surface, not a direct CCU API. + +To keep `tilexr_api.h` free of CCU symbols, communicator initialization should enable the CCU backend through one of +these generic mechanisms: + +- an environment/config string such as a backend allowlist; or +- a future generic init-options API whose header names do not encode CCU-specific task concepts. + +Typed CCU selection belongs in the collective API layer. A future collective options surface can live in +`tilexr_collectives.h`, for example: + +```cpp +enum TileXRCollectiveBackend { + TILEXR_COLLECTIVE_BACKEND_AUTO = 0, + TILEXR_COLLECTIVE_BACKEND_AIV = 1, + TILEXR_COLLECTIVE_BACKEND_UDMA = 2, + TILEXR_COLLECTIVE_BACKEND_CCU = 3, +}; + +struct TileXRCollectiveOptions { + /* Null or zero-initialized options select AUTO. */ + TileXRCollectiveBackend backend = TILEXR_COLLECTIVE_BACKEND_AUTO; +}; + +int TileXRAllGatherEx( + void* sendBuf, + void* recvBuf, + int64_t sendCount, + TileXR::TileXRDataType dataType, + TileXRCommPtr comm, + aclrtStream stream, + const TileXRCollectiveOptions* options); +``` + +The exact naming can change during implementation, but the public surface must expose only backend selection and must +not expose repository, SQE, XN, CKE, instruction, or task-preparation details. + +If the implementation needs a temporary internal seam for smoke probes, it should live under `src/comm/ccu` or +`tests/ccu`, should not be installed, and should not be documented as user API. + +## Source Layout + +The generic wrapper file, `src/comm/comm_wrap.cpp`, keeps only baseline TileXR API implementations. Direct CCU +prepare/submit/readback C wrappers are removed from this file instead of moved to an installed CCU header. + +The CCU source tree owns internal C++ entry points for backend use and validation. These entry points should be +organized around a class, not a broad C API facade. + +Because CCU is currently the only new backend in this change that needs a dedicated runtime context, `TileXRComm` can +own a single CCU backend context directly. Do not add a generic `TileXRCommBackends` manager in this pass. `AIV` maps +to the existing AIV/default collective path, and `UDMA` maps to existing UDMA-capable collective paths when available. +If a later backend needs the same lifecycle pattern as CCU, that will be the right time to introduce a manager. + +## `TileXRComm` Boundary + +`TileXRComm` should stop exposing CCU internals as public methods and direct member fields. A CCU-owned backend class +should hold the direct CCU runtime state currently stored on `TileXRComm`, including: + +- `TileXRCcuDirectRuntime` +- cached basic info and reports +- lower-layer template, snapshot, plan, routes, and plan reports +- direct CCU allgather round state +- prepare/install/readback helper methods used by collectives or internal validation + +The preferred implementation is `src/comm/ccu/tilexr_ccu_backend.{h,cpp}` with a class named `TileXRCcuBackend`. +`TileXRComm` owns this context as an opaque `std::unique_ptr` and exposes only narrow internal accessors needed by +communicator initialization and collective dispatch, for example: + +- initialize or disable the CCU backend according to communicator configuration; +- query whether the CCU backend is available; +- dispatch a collective operation through the CCU backend when the collective layer selects that mode; +- pass rank, rank size, device id, uid, and socket-exchange facilities needed by the backend. + +If a temporary accessor is needed during migration, it must be clearly CCU-scoped and not added to `tilexr_api.h`. + +The backend class should expose cohesive methods rather than mirroring the old public C functions one-for-one. Example +shape: + +```cpp +class TileXRCcuBackend { +public: + int Init(const TileXRCcuBackendOptions& options); + void Shutdown(); + bool Available() const; + + int PrepareCollective(const TileXRCcuCollectiveRequest& request, TileXRCcuCollectivePlan* plan); + int SubmitCollective(const TileXRCcuCollectivePlan& plan, void* stream); + + int PrepareMemoryCopyForTest(const TileXRCcuMemoryCopyRequest& request, TileXRCcuPreparedTasks* tasks); + int ReadInstructionsForTest(...); +}; +``` + +The exact method names can differ, but the boundary should express backend lifecycle and collective execution rather +than a user-facing direct CCU API. + +## CCU Internal Structure + +`TileXRCcuBackend` is the facade used by `TileXRComm` and the collective dispatch layer. Internally it should separate +control-plane setup from collective execution: + +- `TileXRCcuRuntimeSession`: owns runtime availability, basic info refresh, driver adapter creation, RA/HCCP state, + lower-layer transport exchange, and shutdown. +- `TileXRCcuCollectivePlanner`: turns a typed collective request into CCU resource allocation, lower-layer install + plan, repository image, launch package, and submit task plan. +- `TileXRCcuExecutor`: submits prepared CCU tasks to a stream, handles synchronization/reporting policy, and maps + runtime failures to TileXR error codes. + +This split keeps the existing low-level implementation reusable while preventing `TileXRComm` from knowing about +resource repositories, SQE layout, XN/CKE allocation, or diagnostic readback. + +## Backend Selection And Fallback + +Collective dispatch must distinguish "enabled" from "selected". If `TileXRCollectiveOptions` is `nullptr` or +zero-initialized, the selected backend is `AUTO`. + +- `AUTO`: choose the best available supported backend without surfacing backend-specific errors for skipped candidates. + The default policy is CCU when the communicator enabled and initialized CCU and the requested collective is supported, + then UDMA when UDMA is initialized and supports the requested collective, then the existing AIV path. +- `AIV`: always use the existing AIV/default collective path and do not touch CCU backend state. It also must not require + UDMA. +- `UDMA`: require a UDMA-backed collective path. If UDMA was not initialized, return `TILEXR_ERROR_NOT_INITIALIZED`. + If UDMA is initialized but the requested collective, datatype, topology, rank count, or hardware state is unsupported, + return `TILEXR_ERROR_NOT_SUPPORT`. Do not silently fall back. +- `CCU`: require CCU. If the communicator did not enable or initialize the backend, return + `TILEXR_ERROR_NOT_INITIALIZED`. If CCU is initialized but the requested collective, datatype, topology, rank count, + or hardware state is unsupported, return `TILEXR_ERROR_NOT_SUPPORT`. Do not silently fall back. + +The old collective entry points keep their current behavior and are equivalent to `AUTO` unless a later compatibility +decision says otherwise. New `*Ex` entry points can carry explicit `TileXRCollectiveOptions`. + +## Test Hooks + +Diagnostic-only behavior such as memory-copy task preparation and instruction readback should not be normal backend +API. These helpers should be available only through one of these mechanisms: + +- `#if defined(TILEXR_CCU_TESTING)` declarations inside CCU implementation files; or +- private test helpers under `tests/ccu`. + +Production backend methods should be limited to lifecycle, capability query, collective planning, and collective +submission. + +## Build And Install + +`src/comm/CMakeLists.txt` should add the new CCU backend/context sources to `tile-comm`. + +Install headers should not include a CCU-specific public API header. + +No separate library or install component is introduced. + +## Tests + +Update the existing CCU tests so they encode the new boundary: + +- The baseline public header test asserts `tilexr_api.h` contains no CCU symbols. +- The old CCU public C API compile probe is removed or converted into an internal C++ backend probe. +- The smoke probe uses internal CCU backend/test hooks or the eventual generic collective-mode selection path; it does + not include an installed CCU API header. +- Existing checks that private hcomm/hccl symbols and runtime launch structs do not leak to installed public headers + are retained against `tilexr_api.h` and other installed headers. +- A source-guard asserts `comm_wrap.cpp` does not define direct CCU prepare/submit/readback wrappers. +- CCU implementation tests that previously looked for CCU methods directly on `TileXRComm` should be adjusted to the + new backend/context boundary. +- Collective API tests should verify default-unset `AUTO`, explicit `AUTO`, forced `AIV`, forced `UDMA`, and forced + `CCU` fallback/error behavior using fake backend states before relying on hardware smoke runs. + +## Acceptance Criteria + +- `rg -n "CCU|Ccu|DirectCcu|TILEXR_DIRECT_CCU" src/include/tilexr_api.h` returns no matches. +- No installed public header named `tilexr_ccu_api.h` is added. +- The old direct CCU public API compile probe is removed or no longer treats CCU as external user API. +- `comm_wrap.cpp` contains no direct CCU API bridge implementation. +- CCU state is no longer directly stored as many fields on `TileXRComm`; it is owned by the CCU backend/context class. +- There is no generic backend manager abstraction in this pass; `TileXRComm` owns only the CCU backend context. +- Installed public headers do not expose `TileXRDirectCcu*`, `PrepareDirectCcu`, `SubmitPrepared`, `Repository`, `SQE`, + `XN`, `CKE`, or CCU task descriptors. `tilexr_collectives.h` may expose high-level backend enum values for `AUTO`, + `AIV`, `UDMA`, and `CCU`. +- Unset collective options select `AUTO`. +- Forced `UDMA` and forced `CCU` collective modes do not silently fall back when the selected backend is unavailable or + unsupported. +- The focused CCU unit/source-guard tests pass. From 4b1a7b4225571002ba0af4fb73534f71484b5415 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 10:14:54 +0800 Subject: [PATCH 03/41] docs: plan CCU API isolation --- .../plans/2026-07-07-ccu-api-isolation.md | 1631 +++++++++++++++++ 1 file changed, 1631 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-07-ccu-api-isolation.md diff --git a/docs/superpowers/plans/2026-07-07-ccu-api-isolation.md b/docs/superpowers/plans/2026-07-07-ccu-api-isolation.md new file mode 100644 index 00000000..bf5820ce --- /dev/null +++ b/docs/superpowers/plans/2026-07-07-ccu-api-isolation.md @@ -0,0 +1,1631 @@ +# CCU API Isolation Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Refactor direct CCU from a public task API into an internal collective backend selected through `AUTO`, `AIV`, `UDMA`, or `CCU` collective options. + +**Architecture:** Keep `tilexr_api.h` free of CCU symbols. Add high-level backend selection to `tilexr_collectives.h`, route collective calls through a small dispatcher, and make CCU an internal `TileXRCcuBackend` owned opaquely by `TileXRComm`. Inside CCU, separate runtime/session concerns from planning and task execution. + +**Tech Stack:** C++14, C ABI-compatible exported functions, CMake, existing TileXR collectives tests, Python source-guard tests, Ascend runtime stubs where already used by this branch. + +## Global Constraints + +- Do not install or introduce a public `tilexr_ccu_api.h`. +- `src/include/tilexr_api.h` must not contain `CCU`, `Ccu`, `DirectCcu`, or `TILEXR_DIRECT_CCU`. +- Public headers must not expose `TileXRDirectCcu*`, `PrepareDirectCcu`, `SubmitPrepared`, `Repository`, `SQE`, `XN`, `CKE`, or CCU task descriptors. +- `tilexr_collectives.h` may expose high-level backend enum values for `AUTO`, `AIV`, `UDMA`, and `CCU`. +- Null or zero-initialized collective options select `AUTO`. +- Forced `UDMA` and forced `CCU` must not silently fall back when unavailable or unsupported. +- Do not add a generic backend manager in this pass; `TileXRComm` owns only the CCU backend context. +- Preserve existing AIV collective behavior for old entry points by making old entry points call the new `*Ex` path with `AUTO`. + +--- + +## File Structure + +- `src/include/tilexr_collectives.h`: public high-level backend enum, collective options, and `*Ex` collective APIs. +- `src/collectives/host/tilexr_collectives.cpp`: shared validation, `*Ex` implementations, old API forwarding, and backend dispatch. +- `src/collectives/host/collective_backend.h`: internal backend-selection helpers for `AUTO`, `AIV`, `UDMA`, and `CCU`. +- `src/collectives/host/collective_backend.cpp`: dispatch implementation and fake-state hooks for focused unit tests. +- `src/comm/tilexr_comm.h`: remove direct CCU includes/state; forward-declare `TileXRCcuBackend`; expose narrow internal CCU backend accessors. +- `src/comm/tilexr_comm.cpp`: delegate CCU lifecycle and direct CCU helper logic to `TileXRCcuBackend`. +- `src/comm/comm_wrap.cpp`: remove direct CCU public C API bridge and all direct CCU public constants/helpers. +- `src/comm/ccu/tilexr_ccu_backend.h`: internal CCU backend facade used by `TileXRComm` and collective dispatch. +- `src/comm/ccu/tilexr_ccu_backend.cpp`: backend facade implementation, initially moving behavior from `TileXRComm`. +- `src/comm/ccu/tilexr_ccu_runtime_session.h/.cpp`: runtime availability, driver adapter, basic info, RA/HCCP exchange, shutdown. +- `src/comm/ccu/tilexr_ccu_collective_planner.h/.cpp`: convert typed collective requests to existing CCU install/submit plans. +- `src/comm/ccu/tilexr_ccu_executor.h/.cpp`: submit prepared CCU tasks and map runtime failures to TileXR error codes. +- `src/comm/CMakeLists.txt`: add new CCU/backend sources; keep CCU linked into `tile-comm`. +- `tests/collectives/unit/test_tilexr_collective_backend_options.cpp`: compile/runtime checks for public options and dispatch fallback semantics. +- `tests/ccu/test_tilexr_ccu_public_comm_api.py`: invert old public API tests into source guards for no public CCU task API. +- `tests/ccu/test_tilexr_ccu_backend_boundary.py`: source guards for `TileXRCcuBackend` ownership and no direct CCU fields on `TileXRComm`. +- Existing CCU tests under `tests/ccu`: update references from `TileXRComm` direct CCU methods to backend facade or `TILEXR_CCU_TESTING` hooks. + +--- + +### Task 1: Public Collective Backend Options And `*Ex` API + +**Files:** +- Modify: `src/include/tilexr_collectives.h` +- Modify: `src/collectives/host/tilexr_collectives.cpp` +- Create: `tests/collectives/unit/test_tilexr_collective_backend_options.cpp` +- Modify: `tests/collectives/CMakeLists.txt` + +**Interfaces:** +- Produces: + - `enum TileXRCollectiveBackend` + - `struct TileXRCollectiveOptions` + - `TileXRAllGatherEx`, `TileXRAllToAllEx`, `TileXRAllReduceEx`, `TileXRReduceScatterEx`, `TileXRBroadcastEx` + - `TileXRProfileProbeEx` +- Consumes: + - Existing `TileXRAllGather`, `TileXRAllToAll`, `TileXRAllReduce`, `TileXRReduceScatter`, `TileXRBroadcast`, `TileXRProfileProbe` + +- [ ] **Step 1: Write the failing public header test** + +Create `tests/collectives/unit/test_tilexr_collective_backend_options.cpp`: + +```cpp +#include "tilexr_collectives.h" + +#include + +namespace { + +static_assert(TILEXR_COLLECTIVE_BACKEND_AUTO == 0, "AUTO must be zero for zero-initialized options"); +static_assert(TILEXR_COLLECTIVE_BACKEND_AIV == 1, "AIV enum value changed"); +static_assert(TILEXR_COLLECTIVE_BACKEND_UDMA == 2, "UDMA enum value changed"); +static_assert(TILEXR_COLLECTIVE_BACKEND_CCU == 3, "CCU enum value changed"); + +int CheckFunctionPointers() +{ + TileXRCollectiveOptions options {}; + if (options.backend != TILEXR_COLLECTIVE_BACKEND_AUTO) { + return 1; + } + + auto allGather = &TileXRAllGatherEx; + auto allToAll = &TileXRAllToAllEx; + auto allReduce = &TileXRAllReduceEx; + auto reduceScatter = &TileXRReduceScatterEx; + auto broadcast = &TileXRBroadcastEx; + auto profileProbe = &TileXRProfileProbeEx; + + (void)allGather; + (void)allToAll; + (void)allReduce; + (void)reduceScatter; + (void)broadcast; + (void)profileProbe; + return 0; +} + +} // namespace + +int main() +{ + return CheckFunctionPointers(); +} +``` + +- [ ] **Step 2: Register the failing test target** + +In `tests/collectives/CMakeLists.txt`, add near the other unit executables: + +```cmake +add_executable(test_tilexr_collective_backend_options + unit/test_tilexr_collective_backend_options.cpp +) +``` + +After the existing `target_link_libraries(test_tilexr_collectives_header_compile ...)` block, add: + +```cmake +target_link_libraries(test_tilexr_collective_backend_options PRIVATE ${TILEXR_COLLECTIVES_TEST_TARGET}) +``` + +Add `test_tilexr_collective_backend_options` to the `foreach(_tilexr_collectives_link_target ...)` list immediately +after `test_tilexr_collectives_header_compile`. + +Add this test registration immediately after `add_test(NAME test_tilexr_collectives_header_compile ...)`: + +```cmake +add_test(NAME test_tilexr_collective_backend_options COMMAND test_tilexr_collective_backend_options) +``` + +Add `test_tilexr_collective_backend_options` to the `install(TARGETS ...)` list immediately after +`test_tilexr_collectives_header_compile`. + +- [ ] **Step 3: Run test to verify it fails** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options -j$(nproc) +``` + +Expected: compile failure mentioning `TILEXR_COLLECTIVE_BACKEND_AUTO` or `TileXRAllGatherEx` is not declared. + +- [ ] **Step 4: Add public enum, options, and `*Ex` declarations** + +In `src/include/tilexr_collectives.h`, add inside `extern "C"` before the function declarations: + +```cpp +enum TileXRCollectiveBackend { + TILEXR_COLLECTIVE_BACKEND_AUTO = 0, + TILEXR_COLLECTIVE_BACKEND_AIV = 1, + TILEXR_COLLECTIVE_BACKEND_UDMA = 2, + TILEXR_COLLECTIVE_BACKEND_CCU = 3, +}; + +struct TileXRCollectiveOptions { + TileXRCollectiveBackend backend; +}; +``` + +Then declare the `*Ex` variants: + +```cpp +int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); +int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); +int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options); +int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, + const TileXRCollectiveOptions *options); +int TileXRBroadcastEx(void *buf, int64_t count, + TileXR::TileXRDataType dataType, int root, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options); +int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); +``` + +- [ ] **Step 5: Add minimal `*Ex` implementations and old API forwarding** + +In `src/collectives/host/tilexr_collectives.cpp`, add helper: + +```cpp +TileXRCollectiveBackend SelectedBackend(const TileXRCollectiveOptions *options) +{ + return options == nullptr ? TILEXR_COLLECTIVE_BACKEND_AUTO : options->backend; +} +``` + +Rename each existing function body to the corresponding `*Ex` function and accept `const TileXRCollectiveOptions *options`. At the top of each `*Ex` after validation, call: + +```cpp +const TileXRCollectiveBackend backend = SelectedBackend(options); +(void)backend; +``` + +Then make the old function forward to `nullptr` options: + +```cpp +int TileXRAllGather(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream) +{ + return TileXRAllGatherEx(sendBuf, recvBuf, sendCount, dataType, comm, stream, nullptr); +} +``` + +Repeat the same forwarding pattern for all existing collective entry points. + +- [ ] **Step 6: Run test to verify it passes** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options -j$(nproc) +ctest --test-dir build -R test_tilexr_collective_backend_options --output-on-failure +``` + +Expected: target builds and the test passes. + +- [ ] **Step 7: Commit** + +```bash +git add src/include/tilexr_collectives.h src/collectives/host/tilexr_collectives.cpp tests/collectives/CMakeLists.txt tests/collectives/unit/test_tilexr_collective_backend_options.cpp +git commit -m "feat: add collective backend options" +``` + +--- + +### Task 2: Backend Dispatch Semantics For `AUTO`, `AIV`, `UDMA`, And `CCU` + +**Files:** +- Create: `src/collectives/host/collective_backend.h` +- Create: `src/collectives/host/collective_backend.cpp` +- Modify: `src/collectives/host/tilexr_collectives.cpp` +- Modify: `src/collectives/CMakeLists.txt` +- Modify: `tests/collectives/CMakeLists.txt` +- Modify: `tests/collectives/unit/test_tilexr_collective_backend_options.cpp` + +**Interfaces:** +- Consumes: + - `TileXRCollectiveBackend` + - `TileXRCollectiveOptions` + - Existing AIV launch path in `tilexr_collectives.cpp` +- Produces: + - `TileXRCollectives::Host::CollectiveRequest` + - `TileXRCollectives::Host::DispatchCollective` + - `TileXRCollectives::Host::SetBackendTestState` + - `TileXRCollectives::Host::ResetBackendTestState` + +- [ ] **Step 1: Extend test with fake backend-state cases** + +Add this include after `#include "tilexr_collectives.h"`: + +```cpp +#include "collective_backend.h" +``` + +Append this function before `main()`: + +```cpp + +int CheckBackendDispatch() +{ + using TileXRCollectives::Host::BackendTestState; + using TileXRCollectives::Host::CollectiveRequest; + using TileXRCollectives::Host::DispatchCollective; + using TileXRCollectives::Host::ResetBackendTestState; + using TileXRCollectives::Host::SetBackendTestState; + + CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL_GATHER; + request.sendBuf = reinterpret_cast(0x1000); + request.recvBuf = reinterpret_cast(0x2000); + request.count = 1; + request.dataType = TileXR::TILEXR_DATA_TYPE_INT32; + request.comm = reinterpret_cast(0x3000); + request.stream = nullptr; + + BackendTestState state {}; + state.aivReturn = TileXR::TILEXR_SUCCESS; + state.udmaInitialized = false; + state.ccuInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_AUTO) != TileXR::TILEXR_SUCCESS) { + return 2; + } + + state.udmaInitialized = true; + state.udmaSupported = true; + state.udmaReturn = TileXR::TILEXR_SUCCESS; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_UDMA) != TileXR::TILEXR_SUCCESS) { + return 3; + } + + state.udmaInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_UDMA) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { + return 4; + } + + state.ccuInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { + return 5; + } + + ResetBackendTestState(); + return 0; +} +``` + +Change `main()` to: + +```cpp +int main() +{ + const int pointerRet = CheckFunctionPointers(); + if (pointerRet != 0) { + return pointerRet; + } + return CheckBackendDispatch(); +} +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options -j$(nproc) +``` + +Expected: compile failure mentioning `collective_backend.h` is missing. + +- [ ] **Step 3: Add backend dispatch interfaces** + +Create `src/collectives/host/collective_backend.h`: + +```cpp +#ifndef TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H +#define TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H + +#include + +#include "acl/acl_base.h" +#include "tilexr_collectives.h" +#include "tilexr_types.h" + +namespace TileXRCollectives { +namespace Host { + +struct CollectiveRequest { + TileXR::TileXRType type = TileXR::TileXRType::ALL_GATHER; + void *sendBuf = nullptr; + void *recvBuf = nullptr; + int64_t count = 0; + TileXR::TileXRDataType dataType = TileXR::TILEXR_DATA_TYPE_RESERVED; + TileXR::TileXRReduceOp reduceOp = TileXR::TILEXR_REDUCE_RESERVED; + int root = 0; + TileXRCommPtr comm = nullptr; + aclrtStream stream = nullptr; +}; + +struct BackendTestState { + bool enabled = false; + bool udmaInitialized = false; + bool udmaSupported = false; + int udmaReturn = TileXR::TILEXR_ERROR_NOT_SUPPORT; + bool ccuInitialized = false; + bool ccuSupported = false; + int ccuReturn = TileXR::TILEXR_ERROR_NOT_SUPPORT; + int aivReturn = TileXR::TILEXR_SUCCESS; +}; + +int DispatchCollective(const CollectiveRequest &request, TileXRCollectiveBackend backend); +void SetBackendTestState(const BackendTestState &state); +void ResetBackendTestState(); + +} // namespace Host +} // namespace TileXRCollectives + +#endif // TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H +``` + +- [ ] **Step 4: Add minimal dispatch implementation** + +Create `src/collectives/host/collective_backend.cpp`: + +```cpp +#include "collective_backend.h" + +namespace TileXRCollectives { +namespace Host { +namespace { + +BackendTestState g_testState {}; + +int DispatchAiv(const CollectiveRequest&) +{ + return g_testState.enabled ? g_testState.aivReturn : TileXR::TILEXR_SUCCESS; +} + +int DispatchUdma(const CollectiveRequest&) +{ + if (!g_testState.enabled || !g_testState.udmaInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.udmaSupported ? g_testState.udmaReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; +} + +int DispatchCcu(const CollectiveRequest&) +{ + if (!g_testState.enabled || !g_testState.ccuInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.ccuSupported ? g_testState.ccuReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; +} + +} // namespace + +int DispatchCollective(const CollectiveRequest &request, TileXRCollectiveBackend backend) +{ + if (request.comm == nullptr || request.sendBuf == nullptr || request.recvBuf == nullptr || request.count <= 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + + switch (backend) { + case TILEXR_COLLECTIVE_BACKEND_AIV: + return DispatchAiv(request); + case TILEXR_COLLECTIVE_BACKEND_UDMA: + return DispatchUdma(request); + case TILEXR_COLLECTIVE_BACKEND_CCU: + return DispatchCcu(request); + case TILEXR_COLLECTIVE_BACKEND_AUTO: + default: + if (g_testState.enabled && g_testState.ccuInitialized && g_testState.ccuSupported) { + return DispatchCcu(request); + } + if (g_testState.enabled && g_testState.udmaInitialized && g_testState.udmaSupported) { + return DispatchUdma(request); + } + return DispatchAiv(request); + } +} + +void SetBackendTestState(const BackendTestState &state) +{ + g_testState = state; + g_testState.enabled = true; +} + +void ResetBackendTestState() +{ + g_testState = BackendTestState {}; +} + +} // namespace Host +} // namespace TileXRCollectives +``` + +- [ ] **Step 5: Build/link the dispatch source** + +In `src/collectives/CMakeLists.txt`, add `host/collective_backend.cpp` and `host/collective_backend.h` to the `tilexr-collectives` source list. + +In `tests/collectives/CMakeLists.txt`, add a private include directory for the new test: + +```cmake +target_include_directories(test_tilexr_collective_backend_options PRIVATE + ${TILEXR_ROOT}/src/collectives/host +) +``` + +- [ ] **Step 6: Route `*Ex` calls through dispatch without changing AIV behavior** + +In `tilexr_collectives.cpp`, include `collective_backend.h`. + +For each `*Ex`, after existing validation and loopback handling, build a `CollectiveRequest` and call `DispatchCollective`. In this task, keep AIV launch by returning the existing launch result for `AIV` and `AUTO`; forced `UDMA`/`CCU` semantics are exercised by tests through fake dispatch state. + +Use this local helper to keep the old AIV launch path explicit: + +```cpp +bool UsesForcedNonAivBackend(TileXRCollectiveBackend backend) +{ + return backend == TILEXR_COLLECTIVE_BACKEND_UDMA || backend == TILEXR_COLLECTIVE_BACKEND_CCU; +} +``` + +In `TileXRAllGatherEx`, before the AIV launch: + +```cpp +if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL_GATHER; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = sendCount; + request.dataType = dataType; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); +} +``` + +Repeat this pattern for the other collectives: + +```cpp +// TileXRAllToAllEx +request.type = TileXR::TileXRType::ALL2ALL; +request.count = sendCount; + +// TileXRAllReduceEx +request.type = TileXR::TileXRType::ALL_REDUCE; +request.count = count; +request.reduceOp = op; + +// TileXRReduceScatterEx +request.type = TileXR::TileXRType::REDUCE_SCATTER; +request.count = recvCount; +request.reduceOp = op; + +// TileXRBroadcastEx +request.type = TileXR::TileXRType::BROADCAST; +request.sendBuf = buf; +request.recvBuf = buf; +request.count = count; +request.root = root; + +// TileXRProfileProbeEx +request.type = TileXR::TileXRType::PROFILE_PROBE; +request.count = count; +``` + +- [ ] **Step 7: Run focused tests** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options -j$(nproc) +ctest --test-dir build -R test_tilexr_collective_backend_options --output-on-failure +``` + +Expected: test passes. + +- [ ] **Step 8: Commit** + +```bash +git add src/collectives/host/collective_backend.h src/collectives/host/collective_backend.cpp src/collectives/host/tilexr_collectives.cpp src/collectives/CMakeLists.txt tests/collectives/CMakeLists.txt tests/collectives/unit/test_tilexr_collective_backend_options.cpp +git commit -m "feat: route collective backend selection" +``` + +--- + +### Task 3: Remove Direct CCU Public API From Installed Headers And `comm_wrap.cpp` + +**Files:** +- Modify: `src/include/tilexr_api.h` +- Modify: `src/comm/comm_wrap.cpp` +- Modify: `tests/ccu/test_tilexr_ccu_public_comm_api.py` +- Delete: `tests/ccu/ccu_public_direct_api_compile_probe.c` +- Modify: `tests/ccu/test_tilexr_ccu_public_api_compile_probe.py` + +**Interfaces:** +- Consumes: + - Existing generic TileXR C API declarations. +- Produces: + - A clean `tilexr_api.h`. + - No direct CCU public wrapper implementation in `comm_wrap.cpp`. + +- [ ] **Step 1: Rewrite public source-guard test** + +Replace `tests/ccu/test_tilexr_ccu_public_comm_api.py` with: + +```python +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +PUBLIC_HEADERS = [ + REPO_ROOT / "src" / "include" / "tilexr_api.h", + REPO_ROOT / "src" / "include" / "tilexr_types.h", + REPO_ROOT / "src" / "include" / "tilexr_collectives.h", +] +CORE_API_HEADER = REPO_ROOT / "src" / "include" / "tilexr_api.h" +COMM_WRAP = REPO_ROOT / "src" / "comm" / "comm_wrap.cpp" + + +class TileXRCcuPublicCommApiTest(unittest.TestCase): + def test_core_api_header_has_no_ccu_symbols(self): + header = CORE_API_HEADER.read_text(encoding="utf-8") + for needle in ["CCU", "Ccu", "DirectCcu", "TILEXR_DIRECT_CCU"]: + with self.subTest(needle=needle): + self.assertNotIn(needle, header) + + def test_installed_public_headers_do_not_expose_low_level_ccu_model(self): + forbidden = [ + "TileXRDirectCcu", + "PrepareDirectCcu", + "SubmitPrepared", + "Repository", + "SQE", + " XN", + " CKE", + "TaskInfo", + "rtCCULaunch", + "rtCcuTaskInfo_t", + "hcomm", + "hccl", + ] + for path in PUBLIC_HEADERS: + text = path.read_text(encoding="utf-8") + for needle in forbidden: + with self.subTest(path=path.name, needle=needle): + self.assertNotIn(needle, text) + + def test_collectives_header_only_exposes_high_level_backend_names(self): + text = (REPO_ROOT / "src" / "include" / "tilexr_collectives.h").read_text(encoding="utf-8") + for needle in [ + "TILEXR_COLLECTIVE_BACKEND_AUTO", + "TILEXR_COLLECTIVE_BACKEND_AIV", + "TILEXR_COLLECTIVE_BACKEND_UDMA", + "TILEXR_COLLECTIVE_BACKEND_CCU", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, text) + + def test_comm_wrap_has_no_direct_ccu_public_bridge(self): + wrapper = COMM_WRAP.read_text(encoding="utf-8") + for needle in [ + "TileXRCommInitRankDirectCcuWithDomain", + "TileXRCommPrepareDirectCcu", + "TileXRCommPrepareDirectCcuMemoryCopy", + "TileXRDirectCcuGetPreparedTask", + "TileXRDirectCcuSubmitPrepared", + "TileXRCommReadDirectCcuInstructions", + "TileXRDirectCcuDestroyPrepared", + "TileXRDirectCcuPreparedTasks", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, wrapper) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_public_comm_api.py +``` + +Expected: failures showing CCU symbols in `tilexr_api.h` and `comm_wrap.cpp`. + +- [ ] **Step 3: Remove CCU declarations from `tilexr_api.h`** + +Delete these declaration groups from `src/include/tilexr_api.h`: + +```cpp +typedef void *TileXRDirectCcuPreparedTasksPtr; +#define TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES 2048 +... +int TileXRCommInitRankDirectCcuWithDomain(...); +... +int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared); +``` + +Keep `TileXRCommPtr`, UDMA/SDMA APIs, DFX APIs, and generic comm lifecycle APIs unchanged. + +- [ ] **Step 4: Remove direct CCU bridge from `comm_wrap.cpp`** + +Delete the anonymous-namespace direct CCU helpers and public wrapper functions: + +```cpp +TileXRDirectCcuPreparedTasks +CopyDirectCcuMessage +FillPublicPrepareReport +FillPublicSubmitReport +FillPublicInstructionReadbackReport +RepositoryInstallWindowFromPublic +RepositoryInstallDataLenModeFromPublic +RepositoryMemoryAllocModeFromPublic +InstallOrderFromPublic +MakeDirectCcuOptions +PreparedHandle +MemoryCopyDirectionFromPublic +TileXRCommInitRankDirectCcuWithDomain +TileXRCommPrepareDirectCcu +TileXRCommPrepareDirectCcuMemoryCopy +TileXRDirectCcuGetPreparedTask +TileXRDirectCcuSubmitPrepared +TileXRDirectCcuSubmitPreparedTask +TileXRCommReadDirectCcuInstructions +TileXRDirectCcuCreatePreparedForTest +TileXRDirectCcuDestroyPrepared +``` + +After removal, `comm_wrap.cpp` should include no CCU internal headers and no direct CCU constants. + +- [ ] **Step 5: Remove external direct CCU compile probe** + +Delete `tests/ccu/ccu_public_direct_api_compile_probe.c`. + +Replace `tests/ccu/test_tilexr_ccu_public_api_compile_probe.py` with a guard that asserts the deleted file is gone: + +```python +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +class TileXRCcuPublicApiCompileProbeTest(unittest.TestCase): + def test_external_direct_ccu_public_probe_removed(self): + self.assertFalse((REPO_ROOT / "tests" / "ccu" / "ccu_public_direct_api_compile_probe.c").exists()) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 6: Run source guards** + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_public_comm_api.py +python3 tests/ccu/test_tilexr_ccu_public_api_compile_probe.py +``` + +Expected: both pass. + +- [ ] **Step 7: Commit** + +```bash +git add src/include/tilexr_api.h src/comm/comm_wrap.cpp tests/ccu/test_tilexr_ccu_public_comm_api.py tests/ccu/test_tilexr_ccu_public_api_compile_probe.py +git rm tests/ccu/ccu_public_direct_api_compile_probe.c +git commit -m "refactor: remove public direct CCU API" +``` + +--- + +### Task 4: Introduce `TileXRCcuBackend` Facade And Move CCU State Out Of `TileXRComm` + +**Files:** +- Create: `src/comm/ccu/tilexr_ccu_backend.h` +- Create: `src/comm/ccu/tilexr_ccu_backend.cpp` +- Modify: `src/comm/tilexr_comm.h` +- Modify: `src/comm/tilexr_comm.cpp` +- Modify: `src/comm/CMakeLists.txt` +- Create: `tests/ccu/test_tilexr_ccu_backend_boundary.py` + +**Interfaces:** +- Produces: + - `class TileXRCcuBackend` + - `struct TileXRCcuBackendOptions` + - `TileXRComm::GetCcuBackendForCollectives()` + - `TileXRComm::EnableCcuBackendForTest()` +- Consumes: + - Existing CCU implementation headers under `src/comm/ccu` + +- [ ] **Step 1: Write backend-boundary source guard** + +Create `tests/ccu/test_tilexr_ccu_backend_boundary.py`: + +```python +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMM_HEADER = REPO_ROOT / "src" / "comm" / "tilexr_comm.h" +BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" +BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" + + +class TileXRCcuBackendBoundaryTest(unittest.TestCase): + def test_backend_files_exist(self): + self.assertTrue(BACKEND_HEADER.exists()) + self.assertTrue(BACKEND_SOURCE.exists()) + + def test_tilexr_comm_header_owns_only_opaque_backend(self): + header = COMM_HEADER.read_text(encoding="utf-8") + self.assertIn("class TileXRCcuBackend;", header) + self.assertIn("std::unique_ptr ccuBackend_", header) + for needle in [ + "tilexr_ccu_direct_orchestrator.h", + "tilexr_ccu_direct_runtime.h", + "tilexr_ccu_lower_layer_plan_builder.h", + "TileXRCcuDirectRuntime", + "directCcuBasicInfo_", + "directCcuLowerLayerPlan_", + "directCcuVerifiedEndpointRoutes_", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, header) + + def test_backend_header_exposes_facade_not_public_c_api(self): + header = BACKEND_HEADER.read_text(encoding="utf-8") + self.assertIn("class TileXRCcuBackend", header) + self.assertIn("struct TileXRCcuBackendOptions", header) + self.assertIn("PrepareCollective", header) + self.assertIn("SubmitCollective", header) + for needle in [ + "TileXRDirectCcuPreparedTasksPtr", + "TileXRCommPrepareDirectCcu", + "TileXRDirectCcuSubmitPrepared", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, header) + + +if __name__ == "__main__": + unittest.main() +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +``` + +Expected: failure because backend files do not exist and `TileXRComm` still exposes direct CCU state. + +- [ ] **Step 3: Add backend facade header** + +Create `src/comm/ccu/tilexr_ccu_backend.h`: + +```cpp +#ifndef TILEXR_CCU_BACKEND_H +#define TILEXR_CCU_BACKEND_H + +#include +#include +#include +#include + +#include "acl/acl_base.h" +#include "tilexr_types.h" + +namespace TileXR { + +class TileXRComm; +class TileXRCcuRuntimeSession; +class TileXRCcuCollectivePlanner; +class TileXRCcuExecutor; + +struct TileXRCcuBackendOptions { + int rank = 0; + int rankSize = 0; + int devId = 0; + std::string uid; + TileXRComm *comm = nullptr; +}; + +struct TileXRCcuCollectiveRequest { + TileXRType type = TileXRType::ALL_GATHER; + void *sendBuf = nullptr; + void *recvBuf = nullptr; + int64_t count = 0; + TileXRDataType dataType = TILEXR_DATA_TYPE_RESERVED; + TileXRReduceOp reduceOp = TILEXR_REDUCE_RESERVED; + int root = 0; + aclrtStream stream = nullptr; +}; + +struct TileXRCcuCollectivePlan { + bool ready = false; +}; + +class TileXRCcuBackend { +public: + TileXRCcuBackend(); + ~TileXRCcuBackend(); + + TileXRCcuBackend(const TileXRCcuBackend&) = delete; + TileXRCcuBackend& operator=(const TileXRCcuBackend&) = delete; + + int Init(const TileXRCcuBackendOptions &options); + void Shutdown(); + bool Available() const; + bool Supports(const TileXRCcuCollectiveRequest &request) const; + int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); + int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); + +private: + TileXRCcuBackendOptions options_; + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_BACKEND_H +``` + +- [ ] **Step 4: Add minimal backend source** + +Create `src/comm/ccu/tilexr_ccu_backend.cpp`: + +```cpp +#include "ccu/tilexr_ccu_backend.h" + +namespace TileXR { + +TileXRCcuBackend::TileXRCcuBackend() = default; +TileXRCcuBackend::~TileXRCcuBackend() +{ + Shutdown(); +} + +int TileXRCcuBackend::Init(const TileXRCcuBackendOptions &options) +{ + options_ = options; + initialized_ = true; + return TILEXR_SUCCESS; +} + +void TileXRCcuBackend::Shutdown() +{ + initialized_ = false; +} + +bool TileXRCcuBackend::Available() const +{ + return initialized_; +} + +bool TileXRCcuBackend::Supports(const TileXRCcuCollectiveRequest &request) const +{ + return initialized_ && request.type == TileXRType::ALL_GATHER; +} + +int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan) +{ + if (plan == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!initialized_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (!Supports(request)) { + return TILEXR_ERROR_NOT_SUPPORT; + } + *plan = TileXRCcuCollectivePlan {}; + plan->ready = true; + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream) +{ + if (!initialized_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + return plan.ready ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +} // namespace TileXR +``` + +- [ ] **Step 5: Change `TileXRComm` ownership to opaque backend** + +In `src/comm/tilexr_comm.h`, remove CCU private includes: + +```cpp +#include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" +``` + +Add forward declaration in namespace `TileXR`: + +```cpp +class TileXRCcuBackend; +``` + +Replace direct CCU public/private method declarations with: + +```cpp +int InitCcuBackend(); +TileXRCcuBackend *GetCcuBackendForCollectives(); +const TileXRCcuBackend *GetCcuBackendForCollectives() const; +``` + +Replace all direct CCU member fields with: + +```cpp +std::unique_ptr ccuBackend_; +``` + +- [ ] **Step 6: Wire `TileXRComm` implementation** + +In `src/comm/tilexr_comm.cpp`, include: + +```cpp +#include "ccu/tilexr_ccu_backend.h" +``` + +Add: + +```cpp +int TileXRComm::InitCcuBackend() +{ + if (ccuBackend_ == nullptr) { + ccuBackend_.reset(new (std::nothrow) TileXRCcuBackend()); + if (ccuBackend_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + } + TileXRCcuBackendOptions options {}; + options.rank = rank_; + options.rankSize = rankSize_; + options.devId = devId_; + options.uid = uid_; + options.comm = this; + return ccuBackend_->Init(options); +} + +TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives() +{ + return ccuBackend_.get(); +} + +const TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives() const +{ + return ccuBackend_.get(); +} +``` + +For this task, delete the old direct CCU methods from `TileXRComm` or move their bodies into `tilexr_ccu_backend.cpp` behind private helper functions. Keep `Init()` and `InitThread()` compiling by replacing direct CCU runtime init calls with `InitCcuBackend()` only when the new generic config says CCU is enabled. + +- [ ] **Step 7: Add source to build** + +In `src/comm/CMakeLists.txt`, add: + +```cmake + ccu/tilexr_ccu_backend.h + ccu/tilexr_ccu_backend.cpp +``` + +- [ ] **Step 8: Run source guard and build** + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +source scripts/common_env.sh +cmake --build build --target tile-comm -j$(nproc) +``` + +Expected: source guard passes and `tile-comm` builds. + +- [ ] **Step 9: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_backend.h src/comm/ccu/tilexr_ccu_backend.cpp src/comm/tilexr_comm.h src/comm/tilexr_comm.cpp src/comm/CMakeLists.txt tests/ccu/test_tilexr_ccu_backend_boundary.py +git commit -m "refactor: introduce internal CCU backend" +``` + +--- + +### Task 5: Split CCU Backend Internals Into Runtime Session, Planner, And Executor + +**Files:** +- Create: `src/comm/ccu/tilexr_ccu_runtime_session.h` +- Create: `src/comm/ccu/tilexr_ccu_runtime_session.cpp` +- Create: `src/comm/ccu/tilexr_ccu_collective_planner.h` +- Create: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` +- Create: `src/comm/ccu/tilexr_ccu_executor.h` +- Create: `src/comm/ccu/tilexr_ccu_executor.cpp` +- Modify: `src/comm/ccu/tilexr_ccu_backend.h` +- Modify: `src/comm/ccu/tilexr_ccu_backend.cpp` +- Modify: `src/comm/CMakeLists.txt` +- Modify: `tests/ccu/test_tilexr_ccu_backend_boundary.py` + +**Interfaces:** +- Consumes: + - `TileXRCcuBackend` + - existing direct CCU lower-level files. +- Produces: + - `TileXRCcuRuntimeSession` + - `TileXRCcuCollectivePlanner` + - `TileXRCcuExecutor` + +- [ ] **Step 1: Extend boundary test** + +Append to `test_tilexr_ccu_backend_boundary.py`: + +```python + def test_backend_internals_are_split(self): + expected = [ + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.h", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.h", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.h", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.cpp", + ] + for path in expected: + with self.subTest(path=path.name): + self.assertTrue(path.exists()) +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +``` + +Expected: failures for missing split files. + +- [ ] **Step 3: Add runtime session** + +Create `src/comm/ccu/tilexr_ccu_runtime_session.h`: + +```cpp +#ifndef TILEXR_CCU_RUNTIME_SESSION_H +#define TILEXR_CCU_RUNTIME_SESSION_H + +#include "ccu/tilexr_ccu_backend.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession { +public: + int Init(const TileXRCcuBackendOptions &options); + void Shutdown(); + bool Available() const; + +private: + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_RUNTIME_SESSION_H +``` + +Create `src/comm/ccu/tilexr_ccu_runtime_session.cpp`: + +```cpp +#include "ccu/tilexr_ccu_runtime_session.h" + +namespace TileXR { + +int TileXRCcuRuntimeSession::Init(const TileXRCcuBackendOptions&) +{ + initialized_ = true; + return TILEXR_SUCCESS; +} + +void TileXRCcuRuntimeSession::Shutdown() +{ + initialized_ = false; +} + +bool TileXRCcuRuntimeSession::Available() const +{ + return initialized_; +} + +} // namespace TileXR +``` + +- [ ] **Step 4: Add collective planner** + +Create `src/comm/ccu/tilexr_ccu_collective_planner.h`: + +```cpp +#ifndef TILEXR_CCU_COLLECTIVE_PLANNER_H +#define TILEXR_CCU_COLLECTIVE_PLANNER_H + +#include "ccu/tilexr_ccu_backend.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession; + +class TileXRCcuCollectivePlanner { +public: + bool Supports(const TileXRCcuRuntimeSession &session, const TileXRCcuCollectiveRequest &request) const; + int Prepare(const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request, + TileXRCcuCollectivePlan *plan) const; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_COLLECTIVE_PLANNER_H +``` + +Create `src/comm/ccu/tilexr_ccu_collective_planner.cpp`: + +```cpp +#include "ccu/tilexr_ccu_collective_planner.h" + +#include "ccu/tilexr_ccu_runtime_session.h" + +namespace TileXR { + +bool TileXRCcuCollectivePlanner::Supports( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request) const +{ + return session.Available() && request.type == TileXRType::ALL_GATHER; +} + +int TileXRCcuCollectivePlanner::Prepare( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request, + TileXRCcuCollectivePlan *plan) const +{ + if (plan == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!session.Available()) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (!Supports(session, request)) { + return TILEXR_ERROR_NOT_SUPPORT; + } + *plan = TileXRCcuCollectivePlan {}; + plan->ready = true; + return TILEXR_SUCCESS; +} + +} // namespace TileXR +``` + +- [ ] **Step 5: Add executor** + +Create `src/comm/ccu/tilexr_ccu_executor.h`: + +```cpp +#ifndef TILEXR_CCU_EXECUTOR_H +#define TILEXR_CCU_EXECUTOR_H + +#include "acl/acl_base.h" +#include "ccu/tilexr_ccu_backend.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession; + +class TileXRCcuExecutor { +public: + int Submit(const TileXRCcuRuntimeSession &session, const TileXRCcuCollectivePlan &plan, aclrtStream stream) const; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_EXECUTOR_H +``` + +Create `src/comm/ccu/tilexr_ccu_executor.cpp`: + +```cpp +#include "ccu/tilexr_ccu_executor.h" + +#include "ccu/tilexr_ccu_runtime_session.h" + +namespace TileXR { + +int TileXRCcuExecutor::Submit( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectivePlan &plan, + aclrtStream) +{ + if (!session.Available()) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + return plan.ready ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +} // namespace TileXR +``` + +- [ ] **Step 6: Refactor backend facade to delegate** + +In `tilexr_ccu_backend.h`, replace the `bool initialized_` member with: + +```cpp +std::unique_ptr runtimeSession_; +std::unique_ptr planner_; +std::unique_ptr executor_; +``` + +In `tilexr_ccu_backend.cpp`, include the three new headers and change the methods: + +```cpp +TileXRCcuBackend::TileXRCcuBackend() + : runtimeSession_(new (std::nothrow) TileXRCcuRuntimeSession()), + planner_(new (std::nothrow) TileXRCcuCollectivePlanner()), + executor_(new (std::nothrow) TileXRCcuExecutor()) +{ +} + +int TileXRCcuBackend::Init(const TileXRCcuBackendOptions &options) +{ + if (runtimeSession_ == nullptr || planner_ == nullptr || executor_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + options_ = options; + return runtimeSession_->Init(options); +} + +void TileXRCcuBackend::Shutdown() +{ + if (runtimeSession_ != nullptr) { + runtimeSession_->Shutdown(); + } +} + +bool TileXRCcuBackend::Available() const +{ + return runtimeSession_ != nullptr && runtimeSession_->Available(); +} + +bool TileXRCcuBackend::Supports(const TileXRCcuCollectiveRequest &request) const +{ + return runtimeSession_ != nullptr && planner_ != nullptr && planner_->Supports(*runtimeSession_, request); +} + +int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan) +{ + if (runtimeSession_ == nullptr || planner_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + return planner_->Prepare(*runtimeSession_, request, plan); +} + +int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream) +{ + if (runtimeSession_ == nullptr || executor_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + return executor_->Submit(*runtimeSession_, plan, stream); +} +``` + +- [ ] **Step 7: Add files to CMake and run build** + +In `src/comm/CMakeLists.txt`, add all six new files. + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +source scripts/common_env.sh +cmake --build build --target tile-comm -j$(nproc) +``` + +Expected: source guard passes and `tile-comm` builds. + +- [ ] **Step 8: Move existing CCU logic into split classes** + +Move existing code from `TileXRComm` and current CCU files into the split classes with these ownership rules: + +```text +TileXRCcuRuntimeSession: + InitDirectCcuRuntime + RefreshDirectCcuBasicInfo + RegisterCcuResourceRmaBuffer + ExportRemoteCcuRmaBuffers + lower-layer transport exchange + +TileXRCcuCollectivePlanner: + PrepareDirectCcuInstallAttempt + PrepareDirectCcuMemoryCopyInstallAttempt only under TILEXR_CCU_TESTING + FillDirectCcuLowerLayerPlanFromAllocation + lower-layer install plan generation + +TileXRCcuExecutor: + TileXRCcuSubmitPreparedTasks + runtime submit/report mapping + instruction readback only under TILEXR_CCU_TESTING +``` + +After each moved method, run: + +```bash +source scripts/common_env.sh +cmake --build build --target tile-comm -j$(nproc) +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +``` + +Expected: build and source guard continue to pass. + +- [ ] **Step 9: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_runtime_session.h src/comm/ccu/tilexr_ccu_runtime_session.cpp src/comm/ccu/tilexr_ccu_collective_planner.h src/comm/ccu/tilexr_ccu_collective_planner.cpp src/comm/ccu/tilexr_ccu_executor.h src/comm/ccu/tilexr_ccu_executor.cpp src/comm/ccu/tilexr_ccu_backend.h src/comm/ccu/tilexr_ccu_backend.cpp src/comm/CMakeLists.txt tests/ccu/test_tilexr_ccu_backend_boundary.py +git commit -m "refactor: split CCU backend internals" +``` + +--- + +### Task 6: Connect Forced CCU/UDMA Dispatch To Real Backends And Verify Guards + +**Files:** +- Modify: `src/collectives/host/collective_backend.cpp` +- Modify: `src/collectives/host/tilexr_collectives.cpp` +- Modify: `src/comm/tilexr_comm.h` +- Modify: `src/comm/tilexr_comm.cpp` +- Modify: `tests/collectives/unit/test_tilexr_collective_backend_options.cpp` +- Modify: `tests/ccu/test_tilexr_ccu_public_comm_api.py` + +**Interfaces:** +- Consumes: + - `TileXRComm::GetCcuBackendForCollectives()` + - `TileXRCcuBackend::PrepareCollective` + - `TileXRCcuBackend::SubmitCollective` +- Produces: + - Real forced `CCU` dispatch path. + - Real forced `UDMA` error path until UDMA-backed collectives are implemented. + +- [ ] **Step 1: Add test assertions for forced modes** + +In `tests/collectives/unit/test_tilexr_collective_backend_options.cpp`, add cases: + +```cpp + state.ccuInitialized = true; + state.ccuSupported = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_ERROR_NOT_SUPPORT) { + return 6; + } + + state.ccuSupported = true; + state.ccuReturn = TileXR::TILEXR_SUCCESS; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_SUCCESS) { + return 7; + } +``` + +- [ ] **Step 2: Run test** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options -j$(nproc) +ctest --test-dir build -R test_tilexr_collective_backend_options --output-on-failure +``` + +Expected: test passes in fake-state mode. + +- [ ] **Step 3: Implement real CCU dispatch when fake state is disabled** + +In `collective_backend.cpp`, include `tilexr_comm.h` and `ccu/tilexr_ccu_backend.h`. + +Update `DispatchCcu`: + +```cpp +int DispatchCcu(const CollectiveRequest &request) +{ + if (g_testState.enabled) { + if (!g_testState.ccuInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.ccuSupported ? g_testState.ccuReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; + } + + auto *comm = static_cast(request.comm); + if (comm == nullptr || comm->GetCcuBackendForCollectives() == nullptr) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + TileXR::TileXRCcuBackend *backend = comm->GetCcuBackendForCollectives(); + TileXR::TileXRCcuCollectiveRequest ccuRequest {}; + ccuRequest.type = request.type; + ccuRequest.sendBuf = request.sendBuf; + ccuRequest.recvBuf = request.recvBuf; + ccuRequest.count = request.count; + ccuRequest.dataType = request.dataType; + ccuRequest.reduceOp = request.reduceOp; + ccuRequest.root = request.root; + ccuRequest.stream = request.stream; + + TileXR::TileXRCcuCollectivePlan plan {}; + const int prepareRet = backend->PrepareCollective(ccuRequest, &plan); + if (prepareRet != TileXR::TILEXR_SUCCESS) { + return prepareRet; + } + return backend->SubmitCollective(plan, request.stream); +} +``` + +- [ ] **Step 4: Keep UDMA forced mode explicit** + +Until a UDMA-backed collective path exists, keep `DispatchUdma` returning: + +```cpp +return TileXR::TILEXR_ERROR_NOT_SUPPORT; +``` + +when UDMA is initialized but no matching collective backend exists. Do not route to AIV. + +- [ ] **Step 5: Run focused tests and build** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target tilexr-collectives test_tilexr_collective_backend_options -j$(nproc) +ctest --test-dir build -R test_tilexr_collective_backend_options --output-on-failure +python3 tests/ccu/test_tilexr_ccu_public_comm_api.py +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +``` + +Expected: build passes and all listed tests pass. + +- [ ] **Step 6: Commit** + +```bash +git add src/collectives/host/collective_backend.cpp src/collectives/host/tilexr_collectives.cpp src/comm/tilexr_comm.h src/comm/tilexr_comm.cpp tests/collectives/unit/test_tilexr_collective_backend_options.cpp tests/ccu/test_tilexr_ccu_public_comm_api.py +git commit -m "feat: connect CCU collective backend dispatch" +``` + +--- + +### Task 7: Final Verification + +**Files:** +- No new files. +- Verify all files touched in Tasks 1-6. + +**Interfaces:** +- Consumes all previous task outputs. +- Produces verified refactor state. + +- [ ] **Step 1: Run public surface guards** + +Run: + +```bash +rg -n "CCU|Ccu|DirectCcu|TILEXR_DIRECT_CCU" src/include/tilexr_api.h +``` + +Expected: no output and exit code `1`. + +Run: + +```bash +python3 tests/ccu/test_tilexr_ccu_public_comm_api.py +python3 tests/ccu/test_tilexr_ccu_public_api_compile_probe.py +python3 tests/ccu/test_tilexr_ccu_backend_boundary.py +``` + +Expected: all pass. + +- [ ] **Step 2: Run collective backend tests** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target test_tilexr_collective_backend_options test_tilexr_collectives_header_compile test_tilexr_collectives_api -j$(nproc) +ctest --test-dir build -R "test_tilexr_collective_backend_options|test_tilexr_collectives_header_compile|test_tilexr_collectives_api" --output-on-failure +``` + +Expected: all listed tests pass. + +- [ ] **Step 3: Build core libraries** + +Run: + +```bash +source scripts/common_env.sh +cmake --build build --target tile-comm tilexr-collectives -j$(nproc) +``` + +Expected: both targets build. + +- [ ] **Step 4: Inspect remaining CCU leakage** + +Run: + +```bash +rg -n "TileXRDirectCcu|PrepareDirectCcu|SubmitPrepared|TILEXR_DIRECT_CCU|TileXRCommInitRankDirectCcu" src tests +``` + +Expected: matches only in intentionally retained internal CCU implementation tests guarded by `TILEXR_CCU_TESTING`, or no matches after test migration. No matches in installed public headers or `comm_wrap.cpp`. + +- [ ] **Step 5: Confirm no verification-only changes remain** + +Run: + +```bash +git status --short +``` + +Expected: no output. If this command lists files, inspect them with `git diff` and either commit the intentional fix +with the exact files shown by `git status --short`, or revert generated artifacts that are not source changes. From ac8bc420d691ac01e61edbb92f381e0ff2e8ca88 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 10:34:08 +0800 Subject: [PATCH 04/41] feat: add collective backend options --- src/collectives/host/tilexr_collectives.cpp | 96 +++++++++++++++---- src/include/tilexr_collectives.h | 30 ++++++ tests/collectives/CMakeLists.txt | 11 +++ ...test_tilexr_collective_backend_options.cpp | 49 ++++++++++ 4 files changed, 168 insertions(+), 18 deletions(-) create mode 100644 tests/collectives/unit/test_tilexr_collective_backend_options.cpp diff --git a/src/collectives/host/tilexr_collectives.cpp b/src/collectives/host/tilexr_collectives.cpp index d91ed87c..96049baf 100644 --- a/src/collectives/host/tilexr_collectives.cpp +++ b/src/collectives/host/tilexr_collectives.cpp @@ -66,16 +66,23 @@ int LoopbackCopy(void *sendBuf, void *recvBuf, int64_t bytes, aclrtStream stream return ret == ACL_SUCCESS ? TileXR::TILEXR_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; } +TileXRCollectiveBackend SelectedBackend(const TileXRCollectiveOptions *options) +{ + return options == nullptr ? TILEXR_COLLECTIVE_BACKEND_AUTO : options->backend; +} + } // namespace -int TileXRAllGather(void *sendBuf, void *recvBuf, int64_t sendCount, - TileXR::TileXRDataType dataType, TileXRCommPtr comm, - aclrtStream stream) +int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options) { int ret = ValidateCommon(sendBuf, recvBuf, sendCount, dataType, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -93,14 +100,23 @@ int TileXRAllGather(void *sendBuf, void *recvBuf, int64_t sendCount, sendBuf, recvBuf, sendCount, dataType, blockDim, stream); } -int TileXRAllToAll(void *sendBuf, void *recvBuf, int64_t sendCount, - TileXR::TileXRDataType dataType, TileXRCommPtr comm, - aclrtStream stream) +int TileXRAllGather(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream) +{ + return TileXRAllGatherEx(sendBuf, recvBuf, sendCount, dataType, comm, stream, nullptr); +} + +int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options) { int ret = ValidateCommon(sendBuf, recvBuf, sendCount, dataType, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -133,14 +149,23 @@ int TileXRAllToAll(void *sendBuf, void *recvBuf, int64_t sendCount, sendBuf, recvBuf, kernelCount, dataType, blockDim, stream); } -int TileXRAllReduce(void *sendBuf, void *recvBuf, int64_t count, - TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, - TileXRCommPtr comm, aclrtStream stream) +int TileXRAllToAll(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream) +{ + return TileXRAllToAllEx(sendBuf, recvBuf, sendCount, dataType, comm, stream, nullptr); +} + +int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options) { int ret = ValidateReduce(sendBuf, recvBuf, count, dataType, op, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -159,14 +184,24 @@ int TileXRAllReduce(void *sendBuf, void *recvBuf, int64_t count, TileXRCollectives::Host::CollectiveLaunchAttrs { static_cast(op), 0 }); } -int TileXRReduceScatter(void *sendBuf, void *recvBuf, int64_t recvCount, - TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, - TileXRCommPtr comm, aclrtStream stream) +int TileXRAllReduce(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream) +{ + return TileXRAllReduceEx(sendBuf, recvBuf, count, dataType, op, comm, stream, nullptr); +} + +int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, + const TileXRCollectiveOptions *options) { int ret = ValidateReduce(sendBuf, recvBuf, recvCount, dataType, op, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -195,14 +230,23 @@ int TileXRReduceScatter(void *sendBuf, void *recvBuf, int64_t recvCount, TileXRCollectives::Host::CollectiveLaunchAttrs { static_cast(op), 0 }); } -int TileXRBroadcast(void *buf, int64_t count, - TileXR::TileXRDataType dataType, int root, - TileXRCommPtr comm, aclrtStream stream) +int TileXRReduceScatter(void *sendBuf, void *recvBuf, int64_t recvCount, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream) +{ + return TileXRReduceScatterEx(sendBuf, recvBuf, recvCount, dataType, op, comm, stream, nullptr); +} + +int TileXRBroadcastEx(void *buf, int64_t count, + TileXR::TileXRDataType dataType, int root, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options) { int ret = ValidateBroadcastLocal(buf, count, dataType, root, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -227,14 +271,23 @@ int TileXRBroadcast(void *buf, int64_t count, TileXRCollectives::Host::CollectiveLaunchAttrs { 0, root }); } -int TileXRProfileProbe(void *sendBuf, void *recvBuf, int64_t count, - TileXR::TileXRDataType dataType, TileXRCommPtr comm, - aclrtStream stream) +int TileXRBroadcast(void *buf, int64_t count, + TileXR::TileXRDataType dataType, int root, + TileXRCommPtr comm, aclrtStream stream) +{ + return TileXRBroadcastEx(buf, count, dataType, root, comm, stream, nullptr); +} + +int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options) { int ret = ValidateCommon(sendBuf, recvBuf, count, dataType, comm); if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + const TileXRCollectiveBackend backend = SelectedBackend(options); + (void)backend; TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -250,3 +303,10 @@ int TileXRProfileProbe(void *sendBuf, void *recvBuf, int64_t count, return TileXRCollectives::Host::LaunchCollectiveKernel(comm, TileXR::TileXRType::PROFILE_PROBE, context, sendBuf, recvBuf, bytes, TileXR::TILEXR_DATA_TYPE_INT8, blockDim, stream); } + +int TileXRProfileProbe(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream) +{ + return TileXRProfileProbeEx(sendBuf, recvBuf, count, dataType, comm, stream, nullptr); +} diff --git a/src/include/tilexr_collectives.h b/src/include/tilexr_collectives.h index 50fa7828..b2bac393 100644 --- a/src/include/tilexr_collectives.h +++ b/src/include/tilexr_collectives.h @@ -20,21 +20,51 @@ // The collectives public API is currently C++ header-compatible because it reuses TileXR namespace datatypes. extern "C" { +enum TileXRCollectiveBackend { + TILEXR_COLLECTIVE_BACKEND_AUTO = 0, + TILEXR_COLLECTIVE_BACKEND_AIV = 1, + TILEXR_COLLECTIVE_BACKEND_UDMA = 2, + TILEXR_COLLECTIVE_BACKEND_CCU = 3, +}; + +struct TileXRCollectiveOptions { + TileXRCollectiveBackend backend; +}; + +int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); int TileXRAllGather(void *sendBuf, void *recvBuf, int64_t sendCount, TileXR::TileXRDataType dataType, TileXRCommPtr comm, aclrtStream stream); +int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); int TileXRAllToAll(void *sendBuf, void *recvBuf, int64_t sendCount, TileXR::TileXRDataType dataType, TileXRCommPtr comm, aclrtStream stream); +int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options); int TileXRAllReduce(void *sendBuf, void *recvBuf, int64_t count, TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, TileXRCommPtr comm, aclrtStream stream); +int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, + TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, + TileXRCommPtr comm, aclrtStream stream, + const TileXRCollectiveOptions *options); int TileXRReduceScatter(void *sendBuf, void *recvBuf, int64_t recvCount, TileXR::TileXRDataType dataType, TileXR::TileXRReduceOp op, TileXRCommPtr comm, aclrtStream stream); +int TileXRBroadcastEx(void *buf, int64_t count, + TileXR::TileXRDataType dataType, int root, + TileXRCommPtr comm, aclrtStream stream, const TileXRCollectiveOptions *options); int TileXRBroadcast(void *buf, int64_t count, TileXR::TileXRDataType dataType, int root, TileXRCommPtr comm, aclrtStream stream); +int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, + TileXR::TileXRDataType dataType, TileXRCommPtr comm, + aclrtStream stream, const TileXRCollectiveOptions *options); int TileXRProfileProbe(void *sendBuf, void *recvBuf, int64_t count, TileXR::TileXRDataType dataType, TileXRCommPtr comm, aclrtStream stream); diff --git a/tests/collectives/CMakeLists.txt b/tests/collectives/CMakeLists.txt index 0dd5f9ef..f2c8b029 100644 --- a/tests/collectives/CMakeLists.txt +++ b/tests/collectives/CMakeLists.txt @@ -90,6 +90,10 @@ add_executable(test_tilexr_collectives_header_compile unit/test_tilexr_collectives_header_compile.cpp ) +add_executable(test_tilexr_collective_backend_options + unit/test_tilexr_collective_backend_options.cpp +) + add_executable(test_tilexr_perf_trace_layout unit/test_tilexr_perf_trace_layout.cpp ) @@ -243,6 +247,10 @@ target_link_libraries(test_tilexr_collectives_header_compile ${TILEXR_COLLECTIVES_TEST_TARGET} ) +target_link_libraries(test_tilexr_collective_backend_options + ${TILEXR_COLLECTIVES_TEST_TARGET} +) + target_link_libraries(test_tilexr_perf_trace_layout ${TILEXR_COMM_TEST_TARGET} ) @@ -295,6 +303,7 @@ target_link_libraries(test_prepare_host_launch_context foreach(_tilexr_collectives_link_target test_tilexr_collectives_header_compile + test_tilexr_collective_backend_options test_tilexr_perf_trace_layout test_collective_perf_report test_collective_perf_session @@ -312,6 +321,7 @@ add_test(NAME test_tilexr_collectives_api COMMAND test_tilexr_collectives_api) add_test(NAME test_tilexr_collectives_kernel_ownership COMMAND test_tilexr_collectives_kernel_ownership) add_test(NAME test_tilexr_collectives_tools_sources COMMAND test_tilexr_collectives_tools_sources) add_test(NAME test_tilexr_collectives_header_compile COMMAND test_tilexr_collectives_header_compile) +add_test(NAME test_tilexr_collective_backend_options COMMAND test_tilexr_collective_backend_options) add_test(NAME test_tilexr_perf_trace_layout COMMAND test_tilexr_perf_trace_layout) add_test(NAME test_collective_perf_report COMMAND test_collective_perf_report) add_test(NAME test_collective_perf_session COMMAND test_collective_perf_session) @@ -338,6 +348,7 @@ install(TARGETS test_tilexr_collectives_kernel_ownership test_tilexr_collectives_tools_sources test_tilexr_collectives_header_compile + test_tilexr_collective_backend_options test_tilexr_perf_trace_layout test_collective_perf_report test_collective_perf_session diff --git a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp new file mode 100644 index 00000000..cd3a977b --- /dev/null +++ b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2024-2026 TileXR Project + * This file is a part of the CANN Open Software. + * Licensed under CANN Open Software License Agreement Version 1.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#include "tilexr_collectives.h" + +#include + +namespace { + +static_assert(TILEXR_COLLECTIVE_BACKEND_AUTO == 0, "AUTO must be zero for zero-initialized options"); +static_assert(TILEXR_COLLECTIVE_BACKEND_AIV == 1, "AIV enum value changed"); +static_assert(TILEXR_COLLECTIVE_BACKEND_UDMA == 2, "UDMA enum value changed"); +static_assert(TILEXR_COLLECTIVE_BACKEND_CCU == 3, "CCU enum value changed"); + +int CheckFunctionPointers() +{ + TileXRCollectiveOptions options {}; + if (options.backend != TILEXR_COLLECTIVE_BACKEND_AUTO) { + return 1; + } + + auto allGather = &TileXRAllGatherEx; + auto allToAll = &TileXRAllToAllEx; + auto allReduce = &TileXRAllReduceEx; + auto reduceScatter = &TileXRReduceScatterEx; + auto broadcast = &TileXRBroadcastEx; + auto profileProbe = &TileXRProfileProbeEx; + + (void)allGather; + (void)allToAll; + (void)allReduce; + (void)reduceScatter; + (void)broadcast; + (void)profileProbe; + return 0; +} + +} // namespace + +int main() +{ + return CheckFunctionPointers(); +} From 30154488ac2d792ee8e7ebba0b5fdfdb0b2fb725 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 10:47:28 +0800 Subject: [PATCH 05/41] feat: route collective backend selection --- src/collectives/CMakeLists.txt | 2 + src/collectives/host/collective_backend.cpp | 81 +++++++++++++++++++ src/collectives/host/collective_backend.h | 52 ++++++++++++ src/collectives/host/tilexr_collectives.cpp | 75 +++++++++++++++++ tests/collectives/CMakeLists.txt | 4 + ...test_tilexr_collective_backend_options.cpp | 57 ++++++++++++- 6 files changed, 270 insertions(+), 1 deletion(-) create mode 100644 src/collectives/host/collective_backend.cpp create mode 100644 src/collectives/host/collective_backend.h diff --git a/src/collectives/CMakeLists.txt b/src/collectives/CMakeLists.txt index 258f29ef..b73bda17 100644 --- a/src/collectives/CMakeLists.txt +++ b/src/collectives/CMakeLists.txt @@ -23,12 +23,14 @@ add_custom_command( ) set(TILEXR_COLLECTIVES_SOURCE_FILE + host/collective_backend.cpp host/collective_launcher.cpp host/collective_utils.cpp host/collective_kernel.cpp host/perf_trace_report.cpp host/perf_trace_session.cpp host/tilexr_collectives.cpp + host/collective_backend.h ${TILEXR_COLLECTIVES_KERNEL_EMBED_CPP} ) diff --git a/src/collectives/host/collective_backend.cpp b/src/collectives/host/collective_backend.cpp new file mode 100644 index 00000000..8f897a19 --- /dev/null +++ b/src/collectives/host/collective_backend.cpp @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2024-2026 TileXR Project + * This file is a part of the CANN Open Software. + * Licensed under CANN Open Software License Agreement Version 1.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#include "collective_backend.h" + +namespace TileXRCollectives { +namespace Host { +namespace { + +BackendTestState g_testState {}; + +int DispatchAiv(const CollectiveRequest &request) +{ + (void)request; + return g_testState.enabled ? g_testState.aivReturn : TileXR::TILEXR_SUCCESS; +} + +int DispatchUdma(const CollectiveRequest &request) +{ + (void)request; + if (!g_testState.enabled || !g_testState.udmaInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.udmaSupported ? g_testState.udmaReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; +} + +int DispatchCcu(const CollectiveRequest &request) +{ + (void)request; + if (!g_testState.enabled || !g_testState.ccuInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.ccuSupported ? g_testState.ccuReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; +} + +} // namespace + +int DispatchCollective(const CollectiveRequest &request, TileXRCollectiveBackend backend) +{ + if (request.comm == nullptr || request.sendBuf == nullptr || request.recvBuf == nullptr || request.count <= 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + + switch (backend) { + case TILEXR_COLLECTIVE_BACKEND_AIV: + return DispatchAiv(request); + case TILEXR_COLLECTIVE_BACKEND_UDMA: + return DispatchUdma(request); + case TILEXR_COLLECTIVE_BACKEND_CCU: + return DispatchCcu(request); + case TILEXR_COLLECTIVE_BACKEND_AUTO: + default: + if (g_testState.enabled && g_testState.ccuInitialized && g_testState.ccuSupported) { + return DispatchCcu(request); + } + if (g_testState.enabled && g_testState.udmaInitialized && g_testState.udmaSupported) { + return DispatchUdma(request); + } + return DispatchAiv(request); + } +} + +void SetBackendTestState(const BackendTestState &state) +{ + g_testState = state; + g_testState.enabled = true; +} + +void ResetBackendTestState() +{ + g_testState = BackendTestState {}; +} + +} // namespace Host +} // namespace TileXRCollectives diff --git a/src/collectives/host/collective_backend.h b/src/collectives/host/collective_backend.h new file mode 100644 index 00000000..cd897fe5 --- /dev/null +++ b/src/collectives/host/collective_backend.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024-2026 TileXR Project + * This file is a part of the CANN Open Software. + * Licensed under CANN Open Software License Agreement Version 1.0 (the "License"). + * Please refer to the License for details. You may not use this file except in compliance with the License. + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. + * See LICENSE in the root of the software repository for the full text of the License. + */ +#ifndef TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H +#define TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H + +#include + +#include "acl/acl_base.h" +#include "tilexr_collectives.h" +#include "tilexr_types.h" + +namespace TileXRCollectives { +namespace Host { + +struct CollectiveRequest { + TileXR::TileXRType type = TileXR::TileXRType::ALL_GATHER; + void *sendBuf = nullptr; + void *recvBuf = nullptr; + int64_t count = 0; + TileXR::TileXRDataType dataType = TileXR::TILEXR_DATA_TYPE_RESERVED; + TileXR::TileXRReduceOp reduceOp = TileXR::TILEXR_REDUCE_RESERVED; + int root = 0; + TileXRCommPtr comm = nullptr; + aclrtStream stream = nullptr; +}; + +struct BackendTestState { + bool enabled = false; + bool udmaInitialized = false; + bool udmaSupported = false; + int udmaReturn = TileXR::TILEXR_ERROR_NOT_SUPPORT; + bool ccuInitialized = false; + bool ccuSupported = false; + int ccuReturn = TileXR::TILEXR_ERROR_NOT_SUPPORT; + int aivReturn = TileXR::TILEXR_SUCCESS; +}; + +int DispatchCollective(const CollectiveRequest &request, TileXRCollectiveBackend backend); +void SetBackendTestState(const BackendTestState &state); +void ResetBackendTestState(); + +} // namespace Host +} // namespace TileXRCollectives + +#endif // TILEXR_COLLECTIVES_HOST_COLLECTIVE_BACKEND_H diff --git a/src/collectives/host/tilexr_collectives.cpp b/src/collectives/host/tilexr_collectives.cpp index 96049baf..024f7feb 100644 --- a/src/collectives/host/tilexr_collectives.cpp +++ b/src/collectives/host/tilexr_collectives.cpp @@ -10,6 +10,7 @@ #include #include "acl/acl_rt.h" +#include "collective_backend.h" #include "collective_kernel.h" #include "collective_launcher.h" #include "collective_utils.h" @@ -71,6 +72,11 @@ TileXRCollectiveBackend SelectedBackend(const TileXRCollectiveOptions *options) return options == nullptr ? TILEXR_COLLECTIVE_BACKEND_AUTO : options->backend; } +bool UsesForcedNonAivBackend(TileXRCollectiveBackend backend) +{ + return backend == TILEXR_COLLECTIVE_BACKEND_UDMA || backend == TILEXR_COLLECTIVE_BACKEND_CCU; +} + } // namespace int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, @@ -94,6 +100,17 @@ int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, if (context.hostArgs->rankSize <= 1) { return LoopbackCopy(sendBuf, recvBuf, bytes, stream); } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL_GATHER; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = sendCount; + request.dataType = dataType; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const uint32_t blockDim = TileXRCollectives::Host::GetAllGatherBlockNum(*context.hostArgs, bytes); return TileXRCollectives::Host::LaunchCollectiveKernel(comm, TileXR::TileXRType::ALL_GATHER, context, @@ -135,6 +152,17 @@ int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, if (sendCount > std::numeric_limits::max() / static_cast(rankSize)) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL2ALL; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = sendCount; + request.dataType = dataType; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const int64_t kernelCount = sendCount * static_cast(rankSize); const int64_t kernelBytes = TileXRCollectives::Host::CountToBytes(kernelCount, dataType); @@ -177,6 +205,18 @@ int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, if (context.hostArgs->rankSize <= 1) { return LoopbackCopy(sendBuf, recvBuf, bytes, stream); } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL_REDUCE; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = count; + request.dataType = dataType; + request.reduceOp = op; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const uint32_t blockDim = TileXRCollectives::Host::GetAllReduceBlockNum(*context.hostArgs, bytes); return TileXRCollectives::Host::LaunchCollectiveKernel(comm, TileXR::TileXRType::ALL_REDUCE, context, @@ -217,6 +257,18 @@ int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, if (recvCount > std::numeric_limits::max() / static_cast(rankSize)) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::REDUCE_SCATTER; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = recvCount; + request.dataType = dataType; + request.reduceOp = op; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const int64_t inputCount = recvCount * static_cast(rankSize); const int64_t inputBytes = TileXRCollectives::Host::CountToBytes(inputCount, dataType); @@ -261,6 +313,18 @@ int TileXRBroadcastEx(void *buf, int64_t count, if (context.hostArgs->rankSize <= 1) { return TileXR::TILEXR_SUCCESS; } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::BROADCAST; + request.sendBuf = buf; + request.recvBuf = buf; + request.count = count; + request.dataType = dataType; + request.root = root; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const uint32_t blockDim = TileXRCollectives::Host::GetBroadcastBlockNum(*context.hostArgs, bytes); if (blockDim == 0) { @@ -294,6 +358,17 @@ int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, if (ret != TileXR::TILEXR_SUCCESS) { return ret; } + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::PROFILE_PROBE; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = count; + request.dataType = dataType; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } const int64_t bytes = TileXRCollectives::Host::CountToBytes(count, dataType); const uint32_t blockDim = TileXRCollectives::Host::GetProfileProbeBlockNum(*context.hostArgs, bytes); diff --git a/tests/collectives/CMakeLists.txt b/tests/collectives/CMakeLists.txt index f2c8b029..090baf12 100644 --- a/tests/collectives/CMakeLists.txt +++ b/tests/collectives/CMakeLists.txt @@ -94,6 +94,10 @@ add_executable(test_tilexr_collective_backend_options unit/test_tilexr_collective_backend_options.cpp ) +target_include_directories(test_tilexr_collective_backend_options PRIVATE + ${TILEXR_ROOT}/src/collectives/host +) + add_executable(test_tilexr_perf_trace_layout unit/test_tilexr_perf_trace_layout.cpp ) diff --git a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp index cd3a977b..b94d6b72 100644 --- a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp +++ b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp @@ -8,6 +8,7 @@ * See LICENSE in the root of the software repository for the full text of the License. */ #include "tilexr_collectives.h" +#include "collective_backend.h" #include @@ -41,9 +42,63 @@ int CheckFunctionPointers() return 0; } +int CheckBackendDispatch() +{ + using TileXRCollectives::Host::BackendTestState; + using TileXRCollectives::Host::CollectiveRequest; + using TileXRCollectives::Host::DispatchCollective; + using TileXRCollectives::Host::ResetBackendTestState; + using TileXRCollectives::Host::SetBackendTestState; + + CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL_GATHER; + request.sendBuf = reinterpret_cast(0x1000); + request.recvBuf = reinterpret_cast(0x2000); + request.count = 1; + request.dataType = TileXR::TILEXR_DATA_TYPE_INT32; + request.comm = reinterpret_cast(0x3000); + request.stream = nullptr; + + BackendTestState state {}; + state.aivReturn = TileXR::TILEXR_SUCCESS; + state.udmaInitialized = false; + state.ccuInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_AUTO) != TileXR::TILEXR_SUCCESS) { + return 2; + } + + state.udmaInitialized = true; + state.udmaSupported = true; + state.udmaReturn = TileXR::TILEXR_SUCCESS; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_UDMA) != TileXR::TILEXR_SUCCESS) { + return 3; + } + + state.udmaInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_UDMA) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { + return 4; + } + + state.ccuInitialized = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { + return 5; + } + + ResetBackendTestState(); + return 0; +} + } // namespace int main() { - return CheckFunctionPointers(); + const int pointerRet = CheckFunctionPointers(); + if (pointerRet != 0) { + return pointerRet; + } + return CheckBackendDispatch(); } From f9f17c28137c1fdc89ba273c47fab42e719ee6b4 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 11:02:17 +0800 Subject: [PATCH 06/41] fix: enforce forced collective backend dispatch --- src/collectives/host/tilexr_collectives.cpp | 132 +++++++++--------- ...test_tilexr_collective_backend_options.cpp | 38 ++++- 2 files changed, 100 insertions(+), 70 deletions(-) diff --git a/src/collectives/host/tilexr_collectives.cpp b/src/collectives/host/tilexr_collectives.cpp index 024f7feb..bf52a81a 100644 --- a/src/collectives/host/tilexr_collectives.cpp +++ b/src/collectives/host/tilexr_collectives.cpp @@ -88,18 +88,6 @@ int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; - - TileXRCollectives::Host::HostLaunchContext context; - ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); - if (ret != TileXR::TILEXR_SUCCESS) { - return ret; - } - - const int64_t bytes = TileXRCollectives::Host::CountToBytes(sendCount, dataType); - if (context.hostArgs->rankSize <= 1) { - return LoopbackCopy(sendBuf, recvBuf, bytes, stream); - } if (UsesForcedNonAivBackend(backend)) { TileXRCollectives::Host::CollectiveRequest request {}; request.type = TileXR::TileXRType::ALL_GATHER; @@ -112,6 +100,17 @@ int TileXRAllGatherEx(void *sendBuf, void *recvBuf, int64_t sendCount, return TileXRCollectives::Host::DispatchCollective(request, backend); } + TileXRCollectives::Host::HostLaunchContext context; + ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); + if (ret != TileXR::TILEXR_SUCCESS) { + return ret; + } + + const int64_t bytes = TileXRCollectives::Host::CountToBytes(sendCount, dataType); + if (context.hostArgs->rankSize <= 1) { + return LoopbackCopy(sendBuf, recvBuf, bytes, stream); + } + const uint32_t blockDim = TileXRCollectives::Host::GetAllGatherBlockNum(*context.hostArgs, bytes); return TileXRCollectives::Host::LaunchCollectiveKernel(comm, TileXR::TileXRType::ALL_GATHER, context, sendBuf, recvBuf, sendCount, dataType, blockDim, stream); @@ -133,7 +132,17 @@ int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::ALL2ALL; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = sendCount; + request.dataType = dataType; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -152,17 +161,6 @@ int TileXRAllToAllEx(void *sendBuf, void *recvBuf, int64_t sendCount, if (sendCount > std::numeric_limits::max() / static_cast(rankSize)) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (UsesForcedNonAivBackend(backend)) { - TileXRCollectives::Host::CollectiveRequest request {}; - request.type = TileXR::TileXRType::ALL2ALL; - request.sendBuf = sendBuf; - request.recvBuf = recvBuf; - request.count = sendCount; - request.dataType = dataType; - request.comm = comm; - request.stream = stream; - return TileXRCollectives::Host::DispatchCollective(request, backend); - } const int64_t kernelCount = sendCount * static_cast(rankSize); const int64_t kernelBytes = TileXRCollectives::Host::CountToBytes(kernelCount, dataType); @@ -193,18 +191,6 @@ int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; - - TileXRCollectives::Host::HostLaunchContext context; - ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); - if (ret != TileXR::TILEXR_SUCCESS) { - return ret; - } - - const int64_t bytes = TileXRCollectives::Host::CountToBytes(count, dataType); - if (context.hostArgs->rankSize <= 1) { - return LoopbackCopy(sendBuf, recvBuf, bytes, stream); - } if (UsesForcedNonAivBackend(backend)) { TileXRCollectives::Host::CollectiveRequest request {}; request.type = TileXR::TileXRType::ALL_REDUCE; @@ -218,6 +204,17 @@ int TileXRAllReduceEx(void *sendBuf, void *recvBuf, int64_t count, return TileXRCollectives::Host::DispatchCollective(request, backend); } + TileXRCollectives::Host::HostLaunchContext context; + ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); + if (ret != TileXR::TILEXR_SUCCESS) { + return ret; + } + + const int64_t bytes = TileXRCollectives::Host::CountToBytes(count, dataType); + if (context.hostArgs->rankSize <= 1) { + return LoopbackCopy(sendBuf, recvBuf, bytes, stream); + } + const uint32_t blockDim = TileXRCollectives::Host::GetAllReduceBlockNum(*context.hostArgs, bytes); return TileXRCollectives::Host::LaunchCollectiveKernel(comm, TileXR::TileXRType::ALL_REDUCE, context, sendBuf, recvBuf, count, dataType, blockDim, stream, @@ -241,7 +238,18 @@ int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::REDUCE_SCATTER; + request.sendBuf = sendBuf; + request.recvBuf = recvBuf; + request.count = recvCount; + request.dataType = dataType; + request.reduceOp = op; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -257,18 +265,6 @@ int TileXRReduceScatterEx(void *sendBuf, void *recvBuf, int64_t recvCount, if (recvCount > std::numeric_limits::max() / static_cast(rankSize)) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (UsesForcedNonAivBackend(backend)) { - TileXRCollectives::Host::CollectiveRequest request {}; - request.type = TileXR::TileXRType::REDUCE_SCATTER; - request.sendBuf = sendBuf; - request.recvBuf = recvBuf; - request.count = recvCount; - request.dataType = dataType; - request.reduceOp = op; - request.comm = comm; - request.stream = stream; - return TileXRCollectives::Host::DispatchCollective(request, backend); - } const int64_t inputCount = recvCount * static_cast(rankSize); const int64_t inputBytes = TileXRCollectives::Host::CountToBytes(inputCount, dataType); @@ -298,7 +294,18 @@ int TileXRBroadcastEx(void *buf, int64_t count, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; + if (UsesForcedNonAivBackend(backend)) { + TileXRCollectives::Host::CollectiveRequest request {}; + request.type = TileXR::TileXRType::BROADCAST; + request.sendBuf = buf; + request.recvBuf = buf; + request.count = count; + request.dataType = dataType; + request.root = root; + request.comm = comm; + request.stream = stream; + return TileXRCollectives::Host::DispatchCollective(request, backend); + } TileXRCollectives::Host::HostLaunchContext context; ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); @@ -313,18 +320,6 @@ int TileXRBroadcastEx(void *buf, int64_t count, if (context.hostArgs->rankSize <= 1) { return TileXR::TILEXR_SUCCESS; } - if (UsesForcedNonAivBackend(backend)) { - TileXRCollectives::Host::CollectiveRequest request {}; - request.type = TileXR::TileXRType::BROADCAST; - request.sendBuf = buf; - request.recvBuf = buf; - request.count = count; - request.dataType = dataType; - request.root = root; - request.comm = comm; - request.stream = stream; - return TileXRCollectives::Host::DispatchCollective(request, backend); - } const uint32_t blockDim = TileXRCollectives::Host::GetBroadcastBlockNum(*context.hostArgs, bytes); if (blockDim == 0) { @@ -351,13 +346,6 @@ int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, return ret; } const TileXRCollectiveBackend backend = SelectedBackend(options); - (void)backend; - - TileXRCollectives::Host::HostLaunchContext context; - ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); - if (ret != TileXR::TILEXR_SUCCESS) { - return ret; - } if (UsesForcedNonAivBackend(backend)) { TileXRCollectives::Host::CollectiveRequest request {}; request.type = TileXR::TileXRType::PROFILE_PROBE; @@ -370,6 +358,12 @@ int TileXRProfileProbeEx(void *sendBuf, void *recvBuf, int64_t count, return TileXRCollectives::Host::DispatchCollective(request, backend); } + TileXRCollectives::Host::HostLaunchContext context; + ret = TileXRCollectives::Host::PrepareHostLaunchContext(comm, context); + if (ret != TileXR::TILEXR_SUCCESS) { + return ret; + } + const int64_t bytes = TileXRCollectives::Host::CountToBytes(count, dataType); const uint32_t blockDim = TileXRCollectives::Host::GetProfileProbeBlockNum(*context.hostArgs, bytes); if (blockDim == 0) { diff --git a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp index b94d6b72..f86346dd 100644 --- a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp +++ b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp @@ -58,6 +58,10 @@ int CheckBackendDispatch() request.dataType = TileXR::TILEXR_DATA_TYPE_INT32; request.comm = reinterpret_cast(0x3000); request.stream = nullptr; + int sendValue = 1; + int recvValue = 0; + TileXRCollectiveOptions options {}; + options.backend = TILEXR_COLLECTIVE_BACKEND_AUTO; BackendTestState state {}; state.aivReturn = TileXR::TILEXR_SUCCESS; @@ -82,10 +86,42 @@ int CheckBackendDispatch() return 4; } + state.udmaInitialized = true; + state.udmaSupported = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_UDMA) != TileXR::TILEXR_ERROR_NOT_SUPPORT) { + return 5; + } + state.ccuInitialized = false; SetBackendTestState(state); if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { - return 5; + return 6; + } + + state.ccuInitialized = true; + state.ccuSupported = false; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_ERROR_NOT_SUPPORT) { + return 7; + } + + options.backend = TILEXR_COLLECTIVE_BACKEND_UDMA; + state.udmaInitialized = false; + state.udmaSupported = false; + SetBackendTestState(state); + if (TileXRAllGatherEx(&sendValue, &recvValue, 1, TileXR::TILEXR_DATA_TYPE_INT32, request.comm, nullptr, &options) != + TileXR::TILEXR_ERROR_NOT_INITIALIZED) { + return 8; + } + + options.backend = TILEXR_COLLECTIVE_BACKEND_CCU; + state.ccuInitialized = true; + state.ccuSupported = false; + SetBackendTestState(state); + if (TileXRAllGatherEx(&sendValue, &recvValue, 1, TileXR::TILEXR_DATA_TYPE_INT32, request.comm, nullptr, &options) != + TileXR::TILEXR_ERROR_NOT_SUPPORT) { + return 9; } ResetBackendTestState(); From 411f56551d54e74c2b45875bf953cefa6b240efd Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 11:14:15 +0800 Subject: [PATCH 07/41] refactor: remove public direct CCU API --- src/comm/comm_wrap.cpp | 448 ------------------ src/include/tilexr_api.h | 136 ------ .../ccu/ccu_public_direct_api_compile_probe.c | 59 --- ...est_tilexr_ccu_public_api_compile_probe.py | 78 +-- tests/ccu/test_tilexr_ccu_public_comm_api.py | 183 ++----- 5 files changed, 47 insertions(+), 857 deletions(-) delete mode 100644 tests/ccu/ccu_public_direct_api_compile_probe.c diff --git a/src/comm/comm_wrap.cpp b/src/comm/comm_wrap.cpp index 4eb717dc..27b18716 100644 --- a/src/comm/comm_wrap.cpp +++ b/src/comm/comm_wrap.cpp @@ -11,10 +11,7 @@ #include #include #include -#include #include -#include -#include #include #include "tilexr_comm.h" @@ -24,180 +21,6 @@ using namespace std; using namespace TileXR; -namespace { - -constexpr const char* TILEXR_DIRECT_CCU_PUBLIC_PROVIDER = "tilexr-public-direct-ccu"; -constexpr const char* TILEXR_DIRECT_CCU_BARRIER_MODE_ENV = "TILEXR_CCU_DIRECT_BARRIER_MODE"; -constexpr uint32_t TILEXR_DIRECT_CCU_MEMORY_COPY_INSTRUCTION_COUNT = 7U; - -struct TileXRDirectCcuPreparedTasks { - ~TileXRDirectCcuPreparedTasks() - { - (void)TileXRCcuReleaseDirectInstallAttemptResources(attempt); - } - - TileXRCcuDirectInstallAttempt attempt; -}; - -void CopyDirectCcuMessage(const std::string& message, char* output) -{ - if (output == nullptr) { - return; - } - std::memset(output, 0, TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES); - if (message.empty()) { - return; - } - std::strncpy(output, message.c_str(), TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES - 1); -} - -void FillPublicPrepareReport( - const TileXRCcuDirectInstallReport& source, - TileXRDirectCcuPrepareReport* report) -{ - if (report == nullptr) { - return; - } - *report = TileXRDirectCcuPrepareReport {}; - report->pipelineBuilt = source.pipelineBuilt; - report->installAttempted = source.installAttempted; - report->installSucceeded = source.installSucceeded; - report->submitReady = source.submitReady; - report->requiredInstallSurfaceCount = source.requiredInstallSurfaceCount; - report->publicVerifiedInstallSurfaceCount = source.publicVerifiedInstallSurfaceCount; - report->missingInstallSurfaceCount = source.missingInstallSurfaceCount; - report->taskCount = source.taskCount; - report->submitTaskCount = source.submitTaskCount; - CopyDirectCcuMessage(source.message, report->message); -} - -void FillPublicSubmitReport( - const TileXRCcuDirectSubmitReport& source, - TileXRDirectCcuSubmitReport* report) -{ - if (report == nullptr) { - return; - } - *report = TileXRDirectCcuSubmitReport {}; - report->submitted = source.submitted; - report->taskCount = source.taskCount; - report->submittedTaskCount = source.submittedTaskCount; - CopyDirectCcuMessage(source.message, report->message); -} - -void FillPublicInstructionReadbackReport( - uint32_t readInstructionCount, - const TileXRCcuDriverAdapterReport& source, - TileXRDirectCcuInstructionReadbackReport* report) -{ - if (report == nullptr) { - return; - } - *report = TileXRDirectCcuInstructionReadbackReport {}; - report->readbackAttempted = true; - report->readInstructionCount = readInstructionCount; - report->opcode = source.opcode; - report->driverRet = source.driverRet; - report->opRet = source.opRet; - CopyDirectCcuMessage(source.message, report->message); -} - -TileXRCcuRepositoryInstallWindow RepositoryInstallWindowFromPublic(uint32_t value) -{ - return value == TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY ? - TileXRCcuRepositoryInstallWindow::FullRepository : - TileXRCcuRepositoryInstallWindow::Mission; -} - -TileXRCcuRepositoryInstallDataLenMode RepositoryInstallDataLenModeFromPublic(uint32_t value) -{ - return value == TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES ? - TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes : - TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; -} - -TileXRCcuRepositoryMemoryAllocMode RepositoryMemoryAllocModeFromPublic(uint32_t value) -{ - if (value == TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM) { - return TileXRCcuRepositoryMemoryAllocMode::RtHbm; - } - return value == TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3 ? - TileXRCcuRepositoryMemoryAllocMode::AclModule3 : - TileXRCcuRepositoryMemoryAllocMode::Acl; -} - -TileXRCcuInstallOrder InstallOrderFromPublic(uint32_t value) -{ - return value == TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST ? - TileXRCcuInstallOrder::InstallLowerLayerFirst : - TileXRCcuInstallOrder::RepositoryFirst; -} - -TileXRCcuDirectInstallOptions MakeDirectCcuOptions(const TileXRDirectCcuPrepareOptions& publicOptions) -{ - TileXRCcuDirectInstallOptions options; - options.syncResourceCount = publicOptions.syncResourceCount; - options.sqeArgCount = publicOptions.sqeArgCount; - options.syncInstructionCount = publicOptions.syncInstructionCount; - options.bindingsPerSyncResource = publicOptions.bindingsPerSyncResource; - options.missionStartId = publicOptions.missionStartId; - options.instructionStartId = publicOptions.instructionStartId; - options.missionInstructionStartId = publicOptions.missionInstructionStartId; - options.xnStartId = publicOptions.xnStartId; - options.gsaStartId = publicOptions.gsaStartId; - options.remoteXnStartId = publicOptions.remoteXnStartId; - options.remoteXnCount = publicOptions.remoteXnCount; - options.ckeStartId = publicOptions.ckeStartId; - options.channelStartId = publicOptions.channelStartId; - options.localWaitCkeStartId = publicOptions.localWaitCkeStartId; - options.localWaitCkeCount = publicOptions.localWaitCkeCount; - options.remoteNotifyCkeStartId = publicOptions.remoteNotifyCkeStartId; - options.remoteNotifyCkeCount = publicOptions.remoteNotifyCkeCount; - options.repositoryInstallOptions.window = - RepositoryInstallWindowFromPublic(publicOptions.repositoryInstallWindow); - options.repositoryInstallOptions.dataLenMode = - RepositoryInstallDataLenModeFromPublic(publicOptions.repositoryInstallDataLenMode); - options.repositoryMemoryAllocMode = - RepositoryMemoryAllocModeFromPublic(publicOptions.repositoryMemoryAllocMode); - options.installOrder = InstallOrderFromPublic(publicOptions.installOrder); - options.deviceId = publicOptions.deviceId; - options.rank = publicOptions.rank; - const char* barrierMode = std::getenv(TILEXR_DIRECT_CCU_BARRIER_MODE_ENV); - if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke") == 0) { - options.barrierMode = TileXRCcuBarrierMode::SyncCke; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke_set_wait") == 0) { - options.barrierMode = TileXRCcuBarrierMode::SyncCkeSetWait; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_cke_post_only") == 0) { - options.barrierMode = TileXRCcuBarrierMode::SyncCkePostOnly; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "local_cke") == 0) { - options.barrierMode = TileXRCcuBarrierMode::LocalCke; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "local_cke_post_only") == 0) { - options.barrierMode = TileXRCcuBarrierMode::LocalCkePostOnly; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_xn_post_only") == 0) { - options.barrierMode = TileXRCcuBarrierMode::SyncXnPostOnly; - } else if (barrierMode != nullptr && std::strcmp(barrierMode, "sync_xn_load_post_only") == 0) { - options.barrierMode = TileXRCcuBarrierMode::SyncXnLoadPostOnly; - } - options.provider = (publicOptions.provider == nullptr || publicOptions.provider[0] == '\0') ? - TILEXR_DIRECT_CCU_PUBLIC_PROVIDER : - publicOptions.provider; - return options; -} - -TileXRDirectCcuPreparedTasks* PreparedHandle(TileXRDirectCcuPreparedTasksPtr prepared) -{ - return static_cast(prepared); -} - -TileXRCcuMemoryCopyDirection MemoryCopyDirectionFromPublic(uint32_t direction) -{ - return direction == TILEXR_DIRECT_CCU_MEMORY_COPY_LOCAL_TO_REMOTE ? - TileXRCcuMemoryCopyDirection::LocalToRemote : - TileXRCcuMemoryCopyDirection::RemoteToLocal; -} - -} // namespace - int TileXRCommInitRankLocal(int rankSize, int rank, TileXRCommPtr *comm) { TILEXR_LOG(INFO) << "using tilexr c++ api! rank" << rank; @@ -292,29 +115,6 @@ int TileXRCommInitRankWithDomain(int commDomain, int rankSize, int rank, TileXRC return TileXRCommInitRankWithCustDomainSize(commDomain, minBufferSize, rankSize, rank, comm); } -int TileXRCommInitRankDirectCcuWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm) -{ - TILEXR_LOG(INFO) << "using tilexr direct CCU only api! rank" << rank; - if (comm == nullptr) { - TILEXR_LOG(ERROR) << "tilexr direct CCU only comm ptr is nullptr!"; - return TILEXR_ERROR_INTERNAL; - } - *comm = nullptr; - unique_ptr c(new (std::nothrow) TileXRComm(rank, rankSize, commDomain, TILEXR_COMM_BUFFER_SIZE)); - if (c == nullptr) { - TILEXR_LOG(ERROR) << "TileXRComm create failed for direct CCU only init. rank : " - << rank << ", rankSize : " << rankSize; - return TILEXR_ERROR_INTERNAL; - } - int ret = c->InitDirectCcuOnly(); - if (ret != TILEXR_SUCCESS) { - TILEXR_LOG(ERROR) << "tilexr direct CCU only init failed! ret: " << ret; - return ret; - } - *comm = c.release(); - return TILEXR_SUCCESS; -} - int TileXRGetCommArgsDev(TileXRCommPtr comm, GM_ADDR &commArgsPtr) { if (comm == nullptr) { @@ -412,254 +212,6 @@ int TileXRGetSDMAWorkspaceDev(TileXRCommPtr comm, GM_ADDR *workspace) return TILEXR_SUCCESS; } -int TileXRCommPrepareDirectCcu( - TileXRCommPtr comm, - const TileXRDirectCcuPrepareOptions *options, - TileXRDirectCcuPreparedTasksPtr *prepared, - TileXRDirectCcuPrepareReport *report) -{ - if (prepared != nullptr) { - *prepared = nullptr; - } - if (report != nullptr) { - *report = TileXRDirectCcuPrepareReport {}; - } - if (comm == nullptr || options == nullptr || prepared == nullptr) { - TILEXR_LOG(ERROR) << "TileXRCommPrepareDirectCcu invalid input"; - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); - if (handle == nullptr) { - return TILEXR_ERROR_INTERNAL; - } - - auto* tilexr = static_cast(comm); - TileXRCcuDirectInstallReport internalReport; - TileXRCcuDirectInstallOptions internalOptions = MakeDirectCcuOptions(*options); - const int ret = tilexr->PrepareDirectCcuInstallAttempt(internalOptions, &handle->attempt, &internalReport); - FillPublicPrepareReport(internalReport, report); - const bool installDiagnosticReady = - internalReport.pipelineBuilt && - internalReport.installAttempted && - internalReport.installSucceeded && - !internalReport.submitReady; - if (ret != TILEXR_SUCCESS && !installDiagnosticReady) { - return ret; - } - - *prepared = handle.release(); - return TILEXR_SUCCESS; -} - -int TileXRCommPrepareDirectCcuMemoryCopy( - TileXRCommPtr comm, - const TileXRDirectCcuMemoryCopyPrepareOptions *options, - TileXRDirectCcuPreparedTasksPtr *prepared, - TileXRDirectCcuPrepareReport *report) -{ - if (prepared != nullptr) { - *prepared = nullptr; - } - if (report != nullptr) { - *report = TileXRDirectCcuPrepareReport {}; - } - if (comm == nullptr || options == nullptr || prepared == nullptr) { - TILEXR_LOG(ERROR) << "TileXRCommPrepareDirectCcuMemoryCopy invalid input"; - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); - if (handle == nullptr) { - return TILEXR_ERROR_INTERNAL; - } - - TileXRCcuDirectInstallOptions internalOptions = MakeDirectCcuOptions(options->prepare); - internalOptions.sqeArgCount = 0; - internalOptions.syncResourceCount = 1; - internalOptions.syncInstructionCount = std::max( - internalOptions.syncInstructionCount, - TILEXR_DIRECT_CCU_MEMORY_COPY_INSTRUCTION_COUNT); - internalOptions.bindingsPerSyncResource = - internalOptions.bindingsPerSyncResource == 0 ? 1 : internalOptions.bindingsPerSyncResource; - const TileXRCcuMemoryCopyDirection direction = MemoryCopyDirectionFromPublic(options->direction); - - auto* tilexr = static_cast(comm); - TileXRCcuDirectInstallReport internalReport; - const int ret = tilexr->PrepareDirectCcuMemoryCopyInstallAttempt( - internalOptions, - options->localSourceAddr, - options->localDestinationAddr, - options->bytes, - options->peerRank, - direction, - &handle->attempt, - &internalReport); - FillPublicPrepareReport(internalReport, report); - const bool installDiagnosticReady = - internalReport.pipelineBuilt && - internalReport.installAttempted && - internalReport.installSucceeded && - !internalReport.submitReady; - if (ret != TILEXR_SUCCESS && !installDiagnosticReady) { - return ret; - } - - *prepared = handle.release(); - return TILEXR_SUCCESS; -} - -int TileXRDirectCcuGetPreparedTask( - TileXRDirectCcuPreparedTasksPtr prepared, - uint32_t index, - TileXRDirectCcuTaskInfo *task) -{ - if (prepared == nullptr || task == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - const auto* handle = PreparedHandle(prepared); - if (index >= handle->attempt.submitTasks.size()) { - return TILEXR_ERROR_NOT_FOUND; - } - - const TileXRCcuTask& source = handle->attempt.submitTasks[index]; - *task = TileXRDirectCcuTaskInfo {}; - task->dieId = source.dieId; - task->missionId = source.missionId; - task->timeout = source.timeout; - task->instStartId = source.instStartId; - task->instCnt = source.instCnt; - task->key = source.key; - task->argSize = source.argSize; - std::memcpy(task->args, source.args, sizeof(task->args)); - return TILEXR_SUCCESS; -} - -int TileXRDirectCcuSubmitPrepared( - TileXRDirectCcuPreparedTasksPtr prepared, - void *stream, - TileXRDirectCcuSubmitReport *report) -{ - if (report != nullptr) { - *report = TileXRDirectCcuSubmitReport {}; - } - if (prepared == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - auto* handle = PreparedHandle(prepared); - TileXRCcuDirectSubmitReport internalReport; - const int ret = TileXRCcuSubmitPreparedTasks( - handle->attempt.submitTasks, - stream, - nullptr, - nullptr, - &internalReport); - FillPublicSubmitReport(internalReport, report); - return ret; -} - -int TileXRDirectCcuSubmitPreparedTask( - TileXRDirectCcuPreparedTasksPtr prepared, - uint32_t index, - void *stream, - TileXRDirectCcuSubmitReport *report) -{ - if (report != nullptr) { - *report = TileXRDirectCcuSubmitReport {}; - } - if (prepared == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - auto* handle = PreparedHandle(prepared); - if (index >= handle->attempt.submitTasks.size()) { - if (report != nullptr) { - report->taskCount = static_cast(handle->attempt.submitTasks.size()); - CopyDirectCcuMessage("selected direct CCU submit task is missing", report->message); - } - return TILEXR_ERROR_NOT_FOUND; - } - - std::vector selectedTasks(1U); - selectedTasks[0] = handle->attempt.submitTasks[index]; - TileXRCcuDirectSubmitReport internalReport; - const int ret = TileXRCcuSubmitPreparedTasks( - selectedTasks, - stream, - nullptr, - nullptr, - &internalReport); - FillPublicSubmitReport(internalReport, report); - return ret; -} - -int TileXRCommReadDirectCcuInstructions( - TileXRCommPtr comm, - uint8_t dieId, - uint16_t instructionStartId, - uint32_t instructionCount, - TileXRDirectCcuInstructionWords *instructions, - TileXRDirectCcuInstructionReadbackReport *report) -{ - if (report != nullptr) { - *report = TileXRDirectCcuInstructionReadbackReport {}; - } - if (comm == nullptr || instructions == nullptr || instructionCount == 0 || - instructionCount > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - auto* tilexr = static_cast(comm); - TileXRCcuDriverAdapterReport internalReport; - const uint32_t instructionBytes = - instructionCount * static_cast(sizeof(TileXRDirectCcuInstructionWords)); - const int ret = tilexr->ReadDirectCcuInstructionsForDebug( - dieId, - instructionStartId, - instructions, - instructionCount, - instructionBytes, - &internalReport); - FillPublicInstructionReadbackReport(instructionCount, internalReport, report); - return ret; -} - -#if defined(TILEXR_CCU_TESTING) -extern "C" TileXRDirectCcuPreparedTasksPtr TileXRDirectCcuCreatePreparedForTest( - const TileXRDirectCcuTaskInfo* tasks, - uint32_t taskCount) -{ - if (tasks == nullptr || taskCount == 0) { - return nullptr; - } - std::unique_ptr handle(new (std::nothrow) TileXRDirectCcuPreparedTasks); - if (handle == nullptr) { - return nullptr; - } - handle->attempt.submitTasks.reserve(taskCount); - for (uint32_t i = 0; i < taskCount; ++i) { - TileXRCcuTask task {}; - task.dieId = tasks[i].dieId; - task.missionId = tasks[i].missionId; - task.timeout = tasks[i].timeout; - task.instStartId = tasks[i].instStartId; - task.instCnt = tasks[i].instCnt; - task.key = tasks[i].key; - task.argSize = tasks[i].argSize; - std::memcpy(task.args, tasks[i].args, sizeof(task.args)); - handle->attempt.submitTasks.push_back(task); - } - return handle.release(); -} -#endif - -int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared) -{ - if (prepared == nullptr) { - return TILEXR_INVALID_VALUE; - } - delete PreparedHandle(prepared); - return TILEXR_SUCCESS; -} - void TileXRPrintDFX2Log(TileXRCommPtr comm) { if (comm == nullptr) { diff --git a/src/include/tilexr_api.h b/src/include/tilexr_api.h index d71faf8e..13cf8251 100644 --- a/src/include/tilexr_api.h +++ b/src/include/tilexr_api.h @@ -27,106 +27,10 @@ extern "C" { #endif // __cplusplus typedef void *TileXRCommPtr; -typedef void *TileXRDirectCcuPreparedTasksPtr; typedef uint32_t TileXRUDMAMemHandle; #define TILEXRUNIQUE_ID_BYTES 128 -#define TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES 2048 -#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION 0U -#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY 1U -#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES 0U -#define TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES 1U -#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL 0U -#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3 1U -#define TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM 2U -#define TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST 0U -#define TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST 1U -#define TILEXR_DIRECT_CCU_MEMORY_COPY_REMOTE_TO_LOCAL 0U -#define TILEXR_DIRECT_CCU_MEMORY_COPY_LOCAL_TO_REMOTE 1U -#define TILEXR_DIRECT_CCU_SQE_ARGS_LEN 13U typedef struct { char internal[TILEXRUNIQUE_ID_BYTES]; } TileXRUniqueId; -typedef struct TileXRDirectCcuPrepareOptions { - uint32_t syncResourceCount; - /* 0 is a valid explicit no-SQE-load mode; use TILEXR_DIRECT_CCU_SQE_ARGS_LEN for full SQE args. */ - uint32_t sqeArgCount; - uint32_t syncInstructionCount; - uint32_t bindingsPerSyncResource; - uint16_t missionStartId; - uint16_t instructionStartId; - uint16_t missionInstructionStartId; - uint16_t xnStartId; - uint16_t gsaStartId; - uint16_t remoteXnStartId; - uint16_t remoteXnCount; - uint16_t ckeStartId; - uint16_t channelStartId; - uint16_t localWaitCkeStartId; - uint16_t localWaitCkeCount; - uint16_t remoteNotifyCkeStartId; - uint16_t remoteNotifyCkeCount; - uint32_t repositoryInstallWindow; - uint32_t repositoryInstallDataLenMode; - uint32_t repositoryMemoryAllocMode; - uint32_t installOrder; - uint32_t deviceId; - uint32_t rank; - const char *provider; -} TileXRDirectCcuPrepareOptions; - -typedef struct TileXRDirectCcuMemoryCopyPrepareOptions { - TileXRDirectCcuPrepareOptions prepare; - uint64_t localSourceAddr; - uint64_t localDestinationAddr; - uint64_t bytes; - uint32_t peerRank; - uint32_t direction; -} TileXRDirectCcuMemoryCopyPrepareOptions; - -typedef struct TileXRDirectCcuPrepareReport { - bool pipelineBuilt; - bool installAttempted; - bool installSucceeded; - bool submitReady; - uint32_t requiredInstallSurfaceCount; - uint32_t publicVerifiedInstallSurfaceCount; - uint32_t missingInstallSurfaceCount; - uint32_t taskCount; - uint32_t submitTaskCount; - char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; -} TileXRDirectCcuPrepareReport; - -typedef struct TileXRDirectCcuSubmitReport { - bool submitted; - uint32_t taskCount; - uint32_t submittedTaskCount; - char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; -} TileXRDirectCcuSubmitReport; - -typedef struct TileXRDirectCcuTaskInfo { - uint8_t dieId; - uint8_t missionId; - uint16_t timeout; - uint16_t instStartId; - uint16_t instCnt; - uint32_t key; - uint32_t argSize; - uint64_t args[TILEXR_DIRECT_CCU_SQE_ARGS_LEN]; -} TileXRDirectCcuTaskInfo; - -#define TILEXR_DIRECT_CCU_INSTRUCTION_WORDS 4U -typedef struct TileXRDirectCcuInstructionWords { - uint64_t words[TILEXR_DIRECT_CCU_INSTRUCTION_WORDS]; -} TileXRDirectCcuInstructionWords; - -typedef struct TileXRDirectCcuInstructionReadbackReport { - bool readbackAttempted; - uint32_t readInstructionCount; - uint32_t opcode; - int driverRet; - int opRet; - char message[TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES]; -} TileXRDirectCcuInstructionReadbackReport; - int TileXRGetUniqueId(TileXRUniqueId *uniqueId, int commDomain); int TileXRCommInitRankLocal(int rankSize, int rank, TileXRCommPtr *comm); @@ -137,8 +41,6 @@ int TileXRCommInitRankWithCustDomainSize(int commDomain, int bufferSize, int ran int TileXRCommInitRankWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm); -int TileXRCommInitRankDirectCcuWithDomain(int commDomain, int rankSize, int rank, TileXRCommPtr *comm); - #ifdef __cplusplus int TileXRGetCommArgsDev(TileXRCommPtr comm, GM_ADDR &commArgsPtr); @@ -161,44 +63,6 @@ int TileXRSDMAAvailable(TileXRCommPtr comm, bool *available); int TileXRGetSDMAWorkspaceDev(TileXRCommPtr comm, GM_ADDR *workspace); -int TileXRCommPrepareDirectCcu( - TileXRCommPtr comm, - const TileXRDirectCcuPrepareOptions *options, - TileXRDirectCcuPreparedTasksPtr *prepared, - TileXRDirectCcuPrepareReport *report); - -int TileXRCommPrepareDirectCcuMemoryCopy( - TileXRCommPtr comm, - const TileXRDirectCcuMemoryCopyPrepareOptions *options, - TileXRDirectCcuPreparedTasksPtr *prepared, - TileXRDirectCcuPrepareReport *report); - -int TileXRDirectCcuGetPreparedTask( - TileXRDirectCcuPreparedTasksPtr prepared, - uint32_t index, - TileXRDirectCcuTaskInfo *task); - -int TileXRDirectCcuSubmitPrepared( - TileXRDirectCcuPreparedTasksPtr prepared, - void *stream, - TileXRDirectCcuSubmitReport *report); - -int TileXRDirectCcuSubmitPreparedTask( - TileXRDirectCcuPreparedTasksPtr prepared, - uint32_t index, - void *stream, - TileXRDirectCcuSubmitReport *report); - -int TileXRCommReadDirectCcuInstructions( - TileXRCommPtr comm, - uint8_t dieId, - uint16_t instructionStartId, - uint32_t instructionCount, - TileXRDirectCcuInstructionWords *instructions, - TileXRDirectCcuInstructionReadbackReport *report); - -int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared); - void TileXRPrintDFX2Log(TileXRCommPtr comm); int TileXRCommInit(int rank, int rankSize, TileXRCommPtr *comms); diff --git a/tests/ccu/ccu_public_direct_api_compile_probe.c b/tests/ccu/ccu_public_direct_api_compile_probe.c deleted file mode 100644 index fb866496..00000000 --- a/tests/ccu/ccu_public_direct_api_compile_probe.c +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2026 TileXR Project - * - * External-user compile probe for the public direct CCU API. - * - * This file intentionally includes only tilexr_api.h from TileXR. It is - * compiled to an object file, not linked, so it verifies public declarations - * without requiring Ascend runtime or NPU hardware. - */ - -#include "tilexr_api.h" - -#ifndef TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES -#error "TILEXR_DIRECT_CCU_REPORT_MESSAGE_BYTES must be public" -#endif - -int TileXRPublicDirectCcuApiCompileProbe(void) -{ - TileXRCommPtr comm = 0; - TileXRDirectCcuPreparedTasksPtr prepared = 0; - TileXRDirectCcuPrepareOptions options = {0}; - TileXRDirectCcuPrepareReport prepareReport = {0}; - TileXRDirectCcuSubmitReport submitReport = {0}; - TileXRDirectCcuTaskInfo task = {0}; - - int (*prepareFn)( - TileXRCommPtr, - const TileXRDirectCcuPrepareOptions *, - TileXRDirectCcuPreparedTasksPtr *, - TileXRDirectCcuPrepareReport *) = &TileXRCommPrepareDirectCcu; - int (*getTaskFn)( - TileXRDirectCcuPreparedTasksPtr, - uint32_t, - TileXRDirectCcuTaskInfo *) = &TileXRDirectCcuGetPreparedTask; - int (*submitFn)( - TileXRDirectCcuPreparedTasksPtr, - void *, - TileXRDirectCcuSubmitReport *) = &TileXRDirectCcuSubmitPrepared; - int (*destroyFn)(TileXRDirectCcuPreparedTasksPtr) = &TileXRDirectCcuDestroyPrepared; - - options.syncResourceCount = 1; - options.syncInstructionCount = 2; - options.bindingsPerSyncResource = 1; - options.gsaStartId = 510; - options.provider = "tilexr-public-direct-ccu-compile-probe"; - - prepareReport.message[0] = '\0'; - submitReport.message[0] = '\0'; - task.args[0] = 0; - - (void)comm; - (void)prepared; - (void)prepareFn; - (void)getTaskFn; - (void)submitFn; - (void)destroyFn; - return (int)(options.syncResourceCount + options.gsaStartId + prepareReport.taskCount + - submitReport.taskCount + task.argSize); -} diff --git a/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py b/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py index da73b900..49eff720 100644 --- a/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py +++ b/tests/ccu/test_tilexr_ccu_public_api_compile_probe.py @@ -3,90 +3,16 @@ # Copyright (c) 2026 TileXR Project # -import shutil -import subprocess -import tempfile import unittest from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[2] -PROBE_C = REPO_ROOT / "tests" / "ccu" / "ccu_public_direct_api_compile_probe.c" -INCLUDE_DIR = REPO_ROOT / "src" / "include" class TileXRCcuPublicApiCompileProbeTest(unittest.TestCase): - def test_probe_source_uses_only_public_api_header(self): - source = PROBE_C.read_text(encoding="utf-8") - - self.assertIn('#include "tilexr_api.h"', source) - for needle in [ - '#include "tilexr_comm.h"', - '#include "ccu/', - "#include ", source) - self.assertNotIn("TileXRCcuDirectInstallAttempt", source) - self.assertNotIn("TileXRCcuSubmitPreparedTasks", source) - if __name__ == "__main__": unittest.main() From f62c4f86335274f65d69c243778c830c588259b2 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 11:43:23 +0800 Subject: [PATCH 08/41] refactor: introduce internal CCU backend --- src/comm/CMakeLists.txt | 2 + src/comm/ccu/tilexr_ccu_backend.cpp | 61 + src/comm/ccu/tilexr_ccu_backend.h | 67 ++ src/comm/tilexr_comm.cpp | 1065 +---------------- src/comm/tilexr_comm.h | 86 +- tests/ccu/test_tilexr_ccu_backend_boundary.py | 53 + 6 files changed, 217 insertions(+), 1117 deletions(-) create mode 100644 src/comm/ccu/tilexr_ccu_backend.cpp create mode 100644 src/comm/ccu/tilexr_ccu_backend.h create mode 100644 tests/ccu/test_tilexr_ccu_backend_boundary.py diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index 91c436fd..648d383c 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -108,6 +108,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_abi_constants.h ccu/tilexr_ccu_barrier_program.h ccu/tilexr_ccu_barrier_program.cpp + ccu/tilexr_ccu_backend.h + ccu/tilexr_ccu_backend.cpp ccu/tilexr_ccu_direct_orchestrator.h ccu/tilexr_ccu_direct_orchestrator.cpp ccu/tilexr_ccu_direct_runtime.h diff --git a/src/comm/ccu/tilexr_ccu_backend.cpp b/src/comm/ccu/tilexr_ccu_backend.cpp new file mode 100644 index 00000000..7a649295 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_backend.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#include "ccu/tilexr_ccu_backend.h" + +namespace TileXR { + +TileXRCcuBackend::TileXRCcuBackend() = default; + +TileXRCcuBackend::~TileXRCcuBackend() +{ + Shutdown(); +} + +int TileXRCcuBackend::Init(const TileXRCcuBackendOptions &options) +{ + options_ = options; + initialized_ = true; + return TILEXR_SUCCESS; +} + +void TileXRCcuBackend::Shutdown() +{ + initialized_ = false; +} + +bool TileXRCcuBackend::Available() const +{ + return initialized_; +} + +bool TileXRCcuBackend::Supports(const TileXRCcuCollectiveRequest &request) const +{ + return initialized_ && request.type == TileXRType::ALL_GATHER; +} + +int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan) +{ + if (plan == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!initialized_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (!Supports(request)) { + return TILEXR_ERROR_NOT_SUPPORT; + } + *plan = TileXRCcuCollectivePlan {}; + plan->ready = true; + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream) +{ + if (!initialized_) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + return plan.ready ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_backend.h b/src/comm/ccu/tilexr_ccu_backend.h new file mode 100644 index 00000000..123507b2 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_backend.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#ifndef TILEXR_CCU_BACKEND_H +#define TILEXR_CCU_BACKEND_H + +#include +#include +#include +#include + +#include "acl/acl_base.h" +#include "tilexr_types.h" + +namespace TileXR { + +class TileXRComm; +class TileXRCcuRuntimeSession; +class TileXRCcuCollectivePlanner; +class TileXRCcuExecutor; + +struct TileXRCcuBackendOptions { + int rank = 0; + int rankSize = 0; + int devId = 0; + std::string uid; + TileXRComm *comm = nullptr; +}; + +struct TileXRCcuCollectiveRequest { + TileXRType type = TileXRType::ALL_GATHER; + void *sendBuf = nullptr; + void *recvBuf = nullptr; + int64_t count = 0; + TileXRDataType dataType = TILEXR_DATA_TYPE_RESERVED; + TileXRReduceOp reduceOp = TILEXR_REDUCE_RESERVED; + int root = 0; + aclrtStream stream = nullptr; +}; + +struct TileXRCcuCollectivePlan { + bool ready = false; +}; + +class TileXRCcuBackend { +public: + TileXRCcuBackend(); + ~TileXRCcuBackend(); + + TileXRCcuBackend(const TileXRCcuBackend&) = delete; + TileXRCcuBackend& operator=(const TileXRCcuBackend&) = delete; + + int Init(const TileXRCcuBackendOptions &options); + void Shutdown(); + bool Available() const; + bool Supports(const TileXRCcuCollectiveRequest &request) const; + int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); + int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); + +private: + TileXRCcuBackendOptions options_; + bool initialized_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_BACKEND_H diff --git a/src/comm/tilexr_comm.cpp b/src/comm/tilexr_comm.cpp index 7c01300d..889db05e 100644 --- a/src/comm/tilexr_comm.cpp +++ b/src/comm/tilexr_comm.cpp @@ -9,8 +9,7 @@ */ #include "tilexr_comm.h" #include "tilexr_internal.h" -#include "ccu/tilexr_ccu_memory_program.h" -#include "ccu/tilexr_ccu_repository.h" +#include "ccu/tilexr_ccu_backend.h" #include "sdma/tilexr_sdma_transport.h" #include "udma/tilexr_udma_transport.h" @@ -50,145 +49,14 @@ using namespace chrono; namespace TileXR { constexpr int HCCL_IPC_PID_ARRAY_SIZE = 1; // 固定每次只传一个PID数据 constexpr int TILEXR_INIT_TIMEOUT = 600; -constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; static map g_localPeerMemMap; static map g_devList; -struct TileXRThreadAllGatherState { - std::vector data[TILEXR_MAX_RANK_SIZE]; - uint64_t arrivals = 0; - uint64_t departures = 0; - size_t bytes = 0; -}; -static map g_directCcuAllGatherStates; static std::mutex g_mtx; static std::mutex g_udmaMtx; static bool g_udmaUnavailable = false; -static std::mutex g_ccuDirectRuntimeMtx; -static bool g_ccuDirectRuntimeUnavailable = false; -static std::string g_ccuDirectRuntimeUnavailableMessage; static std::mutex g_sdmaMtx; static bool g_sdmaUnavailable = false; -uint8_t SelectDirectCcuInstallDieId() -{ - const char *text = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); - if (text == nullptr || text[0] == '\0') { - return 0; - } - char *end = nullptr; - const unsigned long parsed = std::strtoul(text, &end, 0); - if (end == text || *end != '\0' || parsed > 1UL) { - return 0; - } - return static_cast(parsed); -} - -uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) -{ - if (peerRouteCount == 0) { - return 0; - } - return static_cast(peerLocalIndex) + - static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); -} - -uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) -{ - return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); -} - -uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) -{ - if (syncRouteCount == 0) { - return 0; - } - return syncRouteCount; -} - -uint16_t SelectDirectCcuChannelBoundRemoteXnId( - uint16_t remoteXnStartId, - size_t peerLocalIndex, - uint32_t syncIndex, - size_t peerRouteCount) -{ - return static_cast( - static_cast(remoteXnStartId) + - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); -} - -uint16_t SelectDirectCcuRemoteNotifyCkeId(uint16_t remoteNotifyCkeStartId, size_t routeIndex) -{ - return static_cast(static_cast(remoteNotifyCkeStartId) + routeIndex); -} - -std::string ProcessDirectCcuRuntimeUnavailableMessage() -{ - lock_guard lock(g_ccuDirectRuntimeMtx); - if (!g_ccuDirectRuntimeUnavailable) { - return {}; - } - return g_ccuDirectRuntimeUnavailableMessage.empty() ? - "direct CCU runtime unavailable after process-level init failure" : - "direct CCU runtime unavailable after process-level init failure: " + - g_ccuDirectRuntimeUnavailableMessage; -} - -struct DirectCcuMemoryCopyEndpoint { - uint64_t sourceAddr = 0; - uint64_t sourceToken = 0; - uint64_t destinationAddr = 0; - uint64_t destinationToken = 0; - uint64_t bytes = 0; - uint32_t rank = 0; - uint32_t valid = 0; -}; - -int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) -{ - if (addr == 0 || bytes == 0 || packedToken == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - *packedToken = 0; - rtMemUbTokenInfo info {}; - info.va = addr; - info.size = bytes; - const rtError_t ret = rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info); - if (ret != RT_ERROR_NONE) { - return TILEXR_ERROR_MKIRT; - } - constexpr uint32_t tokenIdRightShift = 8U; - const uint32_t tokenId = info.tokenId >> tokenIdRightShift; - *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); - return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; -} - -int BuildDirectCcuLocalMemoryCopyEndpoint( - uint32_t rank, - uint64_t sourceAddr, - uint64_t destinationAddr, - uint64_t bytes, - DirectCcuMemoryCopyEndpoint *endpoint) -{ - if (endpoint == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - *endpoint = DirectCcuMemoryCopyEndpoint {}; - endpoint->rank = rank; - endpoint->bytes = bytes; - endpoint->sourceAddr = sourceAddr; - endpoint->destinationAddr = destinationAddr; - int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); - if (ret != TILEXR_SUCCESS) { - return ret; - } - ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); - if (ret != TILEXR_SUCCESS) { - return ret; - } - endpoint->valid = 1; - return TILEXR_SUCCESS; -} - // 如果是互联的链路，返回false； 对910B2C那些不互联的链路，返回true bool SkipUnusedChannel910B2C(int curRank, int peerRank, ChipName chipName) @@ -304,61 +172,37 @@ int TileXRComm::InitUDMA() return TILEXR_SUCCESS; } -int TileXRComm::InitDirectCcuRuntime() +int TileXRComm::InitCcuBackend() { - if (rankSize_ <= 1) { - TILEXR_LOG(INFO) << "direct CCU runtime skipped for single-rank communicator"; - return TILEXR_SUCCESS; - } - - lock_guard lock(g_ccuDirectRuntimeMtx); - if (g_ccuDirectRuntimeUnavailable) { - TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; - return TILEXR_SUCCESS; - } - - ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime()); - if (ccuDirectRuntime_ == nullptr) { - TILEXR_LOG(WARN) << "TileXRCcuDirectRuntime allocation failed, direct CCU disabled"; - return TILEXR_SUCCESS; + if (ccuBackend_ == nullptr) { + ccuBackend_.reset(new (nothrow) TileXRCcuBackend()); + if (ccuBackend_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } } - TileXRCcuDirectRuntimeOptions options {}; + TileXRCcuBackendOptions options {}; options.rank = rank_; options.rankSize = rankSize_; options.devId = devId_; - options.allGather = &TileXRComm::DirectCcuAllGatherCallback; - options.allGatherUserData = this; - TileXRCcuDirectRuntimeReport runtimeReport; - const int ret = ccuDirectRuntime_->Init(options, &runtimeReport); - if (ret != TILEXR_SUCCESS || !ccuDirectRuntime_->IsAvailable()) { - TILEXR_LOG(WARN) << "TileXR direct CCU runtime init failed: " << ret - << ", logicDevId " << runtimeReport.logicDevId - << ", devicePhyId " << runtimeReport.devicePhyId - << ", hdcType " << runtimeReport.hdcType - << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) - << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) - << ", " << runtimeReport.message << ", direct CCU disabled"; - g_ccuDirectRuntimeUnavailable = true; - g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; - ResetDirectCcuBasicInfo(); - ccuDirectRuntime_.reset(); - return TILEXR_SUCCESS; - } + options.uid = uid_; + options.comm = this; + return ccuBackend_->Init(options); +} - const int ccuInfoRet = RefreshDirectCcuBasicInfo(0); - if (ccuInfoRet != TILEXR_SUCCESS && ccuInfoRet != TILEXR_ERROR_NOT_FOUND) { - TILEXR_LOG(WARN) << "direct CCU basic info refresh failed after runtime init: " << ccuInfoRet - << ", " << directCcuBasicInfoReport_.message; - } +TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives() +{ + return ccuBackend_.get(); +} - TILEXR_LOG(INFO) << "InitDirectCcuRuntime success, rank " << rank_ << "/" << rankSize_ - << " logicDevId " << runtimeReport.logicDevId - << " devicePhyId " << runtimeReport.devicePhyId - << " hdcType " << runtimeReport.hdcType - << " raInitialized " << (runtimeReport.raInitialized ? 1 : 0) - << " ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0); - return TILEXR_SUCCESS; +const TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives() const +{ + return ccuBackend_.get(); +} + +int TileXRComm::EnableCcuBackendForTest() +{ + return InitCcuBackend(); } int TileXRComm::InitSDMA() @@ -506,23 +350,6 @@ void TileXRComm::FreeUDMARegistry() udmaRegistry_ = TileXRUDMARegistry {}; } -void TileXRComm::ResetDirectCcuBasicInfo() -{ - directCcuBasicInfoValid_ = false; - directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - directCcuBasicInfo_ = TileXRCcuBasicInfo {}; - directCcuBasicInfoReport_ = TileXRCcuDriverAdapterReport {}; -} - -void TileXRComm::ResetDirectCcuLowerLayerPlan() -{ - directCcuLowerLayerPlanValid_ = false; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - directCcuLowerLayerSnapshot_ = TileXRCcuLowerLayerTransportSnapshot {}; - directCcuLowerLayerPlan_ = TileXRCcuLowerLayerInstallPlan {}; - directCcuLowerLayerPlanReport_ = TileXRCcuLowerLayerPlanBuilderReport {}; -} - int TileXRComm::RegisterUDMAMemory(GM_ADDR localPtr, size_t bytes, TileXRUDMAMemHandle *handle) { if (!inited_) { @@ -642,796 +469,6 @@ const TileXRUDMARegistry* TileXRComm::GetUDMARegistryHost() const return UDMARegistryValid(&udmaRegistry_, rankSize_) ? &udmaRegistry_ : nullptr; } -int TileXRComm::RefreshDirectCcuBasicInfo(uint8_t dieId) -{ - ResetDirectCcuBasicInfo(); - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuBasicInfoReport_.message = "direct CCU runtime is unavailable for basic info"; - directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuBasicInfoStatus_; - } - - TileXRCcuBasicInfo basicInfo; - TileXRCcuDriverAdapterReport report; - const int ret = ccuDirectRuntime_->QueryBasicInfo(dieId, &basicInfo, &report); - directCcuBasicInfoReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuBasicInfoStatus_ = ret; - return directCcuBasicInfoStatus_; - } - - directCcuBasicInfo_ = basicInfo; - directCcuBasicInfoReport_.message = "direct CCU basic info cached"; - directCcuBasicInfoValid_ = true; - directCcuBasicInfoStatus_ = TILEXR_SUCCESS; - return TILEXR_SUCCESS; -} - -bool TileXRComm::HasDirectCcuBasicInfo() const -{ - return directCcuBasicInfoValid_; -} - -int TileXRComm::GetDirectCcuBasicInfoStatus() const -{ - return directCcuBasicInfoStatus_; -} - -const TileXRCcuBasicInfo *TileXRComm::GetDirectCcuBasicInfo() const -{ - return directCcuBasicInfoValid_ ? &directCcuBasicInfo_ : nullptr; -} - -const TileXRCcuDriverAdapterReport &TileXRComm::GetDirectCcuBasicInfoReport() const -{ - return directCcuBasicInfoReport_; -} - -int TileXRComm::ConfigureDirectCcuLowerLayerTemplate( - const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot) -{ - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); -} - -int TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes( - const std::vector &verifiedRoutes) -{ - TileXRCcuLowerLayerTransportSnapshot validationSnapshot; - validationSnapshot.routes = verifiedRoutes; - TileXRCcuLowerLayerPlanBuilderReport report; - int ret = TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &validationSnapshot, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return ret; - } - - directCcuVerifiedEndpointRoutes_ = verifiedRoutes; - if (directCcuLowerLayerTemplateConfigured_) { - return RefreshDirectCcuLowerLayerPlan(); - } - return TILEXR_SUCCESS; -} - -int TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute( - const TileXRCcuLowerLayerTransportRoute &route) -{ - TileXRCcuLowerLayerTransportSnapshot validationSnapshot; - validationSnapshot.routes.push_back(route); - TileXRCcuLowerLayerPlanBuilderReport report; - std::vector routes {route}; - int ret = TileXRCcuOverlayVerifiedEndpointRoutes(routes, &validationSnapshot, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; - directCcuLocalVerifiedEndpointRouteValid_ = false; - directCcuLowerLayerPlanStatus_ = ret; - return ret; - } - - directCcuLocalVerifiedEndpointRoute_ = route; - directCcuLocalVerifiedEndpointRouteValid_ = true; - if (ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable()) { - return ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(route); - } - return TILEXR_SUCCESS; -} - -int TileXRComm::ConfigureDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation, - const std::vector &remoteCcuBuffers) -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuBasicInfoValid_) { - directCcuLowerLayerPlanReport_.message = - "direct CCU basic info is unavailable for lower-layer transport template"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot templateSnapshot; - TileXRCcuLowerLayerPlanBuilderReport report; - int ret = TileXRCcuBuildLowerLayerTransportTemplate( - directCcuBasicInfo_, - allocation, - remoteCcuBuffers, - &templateSnapshot, - &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &templateSnapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); -} - -int TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation) -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuBasicInfoValid_) { - directCcuLowerLayerPlanReport_.message = - "direct CCU basic info is unavailable for lower-layer transport template"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuLowerLayerPlanReport_.message = - "direct CCU runtime is unavailable for resource window registration"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - int ret = ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to register direct CCU resource window"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLocalResourceWindowInfo localCcuResourceWindow; - ret = ccuDirectRuntime_->ExportLocalCcuRmaBuffer(&localCcuResourceWindow); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU local resource window token"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - if (directCcuLocalVerifiedEndpointRouteValid_) { - ret = ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(directCcuLocalVerifiedEndpointRoute_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to configure direct CCU local verified endpoint route"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - } else { - TileXRCcuDirectRuntimeReport endpointRouteReport; - ret = ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute(&endpointRouteReport); - if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { - TILEXR_LOG(WARN) << "direct CCU local endpoint route collection failed closed: " - << ret << ", " << endpointRouteReport.message; - } - } - - std::vector remoteCcuBuffers; - ret = ccuDirectRuntime_->ExportRemoteCcuRmaBuffers(&remoteCcuBuffers); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU peer resource window tokens"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = ExchangeDirectCcuRemoteNotifyCke(allocation, &remoteCcuBuffers, &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot templateSnapshot; - TileXRCcuLowerLayerPlanBuilderReport report; - ret = TileXRCcuBuildLowerLayerTransportTemplate( - directCcuBasicInfo_, - allocation, - remoteCcuBuffers, - &templateSnapshot, - &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - templateSnapshot.msidToken.dieId = directCcuBasicInfo_.dieId; - templateSnapshot.msidToken.msId = directCcuBasicInfo_.msId; - templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId; - templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue; - templateSnapshot.msidToken.valid = true; - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &templateSnapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); -} - -int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report) -{ - if (plan == nullptr || report == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - int ret = PrepareDirectCcuLowerLayerTemplateFromAllocation(allocation); - if (ret != TILEXR_SUCCESS) { - *report = directCcuLowerLayerPlanReport_; - return ret; - } - if (!directCcuLowerLayerPlanValid_) { - *report = directCcuLowerLayerPlanReport_; - return TILEXR_ERROR_NOT_FOUND; - } - *plan = directCcuLowerLayerPlan_; - *report = directCcuLowerLayerPlanReport_; - return TILEXR_SUCCESS; -} - -int TileXRComm::ExchangeDirectCcuRemoteNotifyCke( - const TileXRCcuResourceAllocation &allocation, - std::vector *remoteCcuBuffers, - TileXRCcuLowerLayerPlanBuilderReport *report) -{ - if (remoteCcuBuffers == nullptr) { - if (report != nullptr) { - report->message = "missing direct CCU remote notify CKE exchange inputs"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (rankSize_ <= 1 || rank_ < 0 || rank_ >= rankSize_) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - const size_t peerRouteCount = static_cast(rankSize_ - 1); - const size_t syncRouteCount = allocation.remoteXn.num; - if (allocation.localXn.num == 0 || - allocation.localWaitCke.num == 0 || - allocation.remoteNotifyCke.num == 0 || - allocation.remoteXn.num < static_cast(rankSize_ - 1) || - allocation.localWaitCke.num < allocation.remoteXn.num || - allocation.remoteNotifyCke.num < allocation.remoteXn.num || - allocation.channels.num == 0 || - remoteCcuBuffers->size() != peerRouteCount) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - struct PeerResourceExchange { - uint16_t localXnStartId; - uint16_t localXnCount; - uint16_t remoteXnStartId; - uint16_t remoteXnCount; - uint16_t localWaitCkeStartId; - uint16_t localWaitCkeCount; - uint16_t remoteNotifyCkeStartId; - uint16_t remoteNotifyCkeCount; - uint16_t channelStartId; - uint16_t channelCount; - }; - PeerResourceExchange local { - allocation.localXn.startId, - allocation.localXn.num, - allocation.remoteXn.startId, - DirectCcuRemoteXnProofSpan(allocation.remoteXn.num), - allocation.localWaitCke.startId, - allocation.localWaitCke.num, - allocation.remoteNotifyCke.startId, - allocation.remoteNotifyCke.num, - allocation.channels.startId, - allocation.channels.num, - }; - std::vector all(rankSize_); - const int ret = DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to exchange direct CCU peer XN/CKE resources"; - } - return ret; - } - - std::vector peerRanks; - peerRanks.reserve(peerRouteCount); - for (int peer = 0; peer < rankSize_; ++peer) { - if (peer != rank_) { - peerRanks.push_back(peer); - } - } - if (peerRanks.size() != peerRouteCount) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - std::vector peerCcuBuffers = *remoteCcuBuffers; - remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); - - size_t routeIndex = 0; - for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { - const size_t peerBufferIndex = syncIndex % peerRouteCount; - const int peer = peerRanks[peerBufferIndex]; - const PeerResourceExchange &peerResources = all[peer]; - const size_t peerLocalIndex = static_cast(rank_ < peer ? rank_ : rank_ - 1); - const uint32_t peerLocalXnOffset = - SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t selectedRemoteXnOffset = - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t peerLocalWaitCkeOffset = routeIndex; - if (peerResources.localXnCount == 0 || - peerResources.remoteXnCount == 0 || - peerResources.localWaitCkeCount == 0 || - peerResources.channelCount == 0 || - peerLocalXnOffset >= peerResources.localXnCount || - selectedRemoteXnOffset >= peerResources.remoteXnCount || - peerLocalIndex >= peerResources.channelCount || - peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { - if (report != nullptr) { - report->message = "peer direct CCU local XN/CKE resources are incomplete"; - } - return TILEXR_ERROR_NOT_FOUND; - } - uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( - peerResources.remoteXnStartId, - peerLocalIndex, - syncIndex, - peerRouteCount); - const uint16_t peerLocalXnId = - static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); - uint16_t remoteNotifyCke = - static_cast(static_cast(peerResources.localWaitCkeStartId) + - peerLocalWaitCkeOffset); - (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; - (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; - (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; - const bool peerLocalXnOwnerVerified = - static_cast(peerLocalXnId) >= peerResources.localXnStartId && - static_cast(peerLocalXnId) < - static_cast(peerResources.localXnStartId) + peerResources.localXnCount; - const bool notifyCkeOwnerVerified = - static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && - static_cast(remoteNotifyCke) < - static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; - const bool localChannelOwnerVerified = - allocation.channels.num != 0 && - peerLocalXnOwnerVerified && - static_cast(channelBoundRemoteXnId) >= peerResources.remoteXnStartId && - static_cast(channelBoundRemoteXnId) < - static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && - routeIndex < allocation.channels.num && - peerResources.channelStartId != 0 && - peerLocalIndex < peerResources.channelCount; - const bool transportResourceExchangeVerified = - notifyCkeOwnerVerified && - allocation.localWaitCke.num != 0 && - routeIndex < allocation.localWaitCke.num && - peerLocalWaitCkeOffset < peerResources.localWaitCkeCount; - (*remoteCcuBuffers)[routeIndex].channelResourceOwnerVerified = localChannelOwnerVerified; - (*remoteCcuBuffers)[routeIndex].transportResourceExchangeVerified = transportResourceExchangeVerified; - ++routeIndex; - } - return TILEXR_SUCCESS; -} - -int TileXRComm::DirectCcuAllGatherCallback( - const void *sendBuf, - size_t sendBytes, - void *recvBuf, - void *userData) -{ - auto *comm = static_cast(userData); - if (comm == nullptr || sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (comm->socketExchange_ == nullptr) { - return comm->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf); - } - return comm->socketExchange_->AllGather( - static_cast(sendBuf), - sendBytes, - static_cast(recvBuf)); -} - -int TileXRComm::DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) -{ - if (sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0 || rank_ < 0 || - rank_ >= rankSize_ || rankSize_ <= 0 || uid_.empty()) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - const uint64_t round = directCcuThreadAllGatherRound_++; - const std::string key = uid_ + ":ccu:" + std::to_string(round); - auto start = high_resolution_clock::now(); - for (;;) { - { - lock_guard lock(g_mtx); - auto &state = g_directCcuAllGatherStates[key]; - if (state.bytes == 0) { - state.bytes = sendBytes; - } else if (state.bytes != sendBytes) { - g_directCcuAllGatherStates.erase(key); - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (state.data[rank_].empty()) { - state.data[rank_].resize(sendBytes); - std::memcpy(state.data[rank_].data(), sendBuf, sendBytes); - ++state.arrivals; - } - if (state.arrivals == static_cast(rankSize_)) { - auto *output = static_cast(recvBuf); - for (int i = 0; i < rankSize_; ++i) { - std::memcpy(output + static_cast(i) * sendBytes, state.data[i].data(), sendBytes); - } - ++state.departures; - if (state.departures == static_cast(rankSize_)) { - g_directCcuAllGatherStates.erase(key); - } - return TILEXR_SUCCESS; - } - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - lock_guard lock(g_mtx); - g_directCcuAllGatherStates.erase(key); - TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ - << " uid " << uid_ << " round " << round << ", " - << processUnavailableMessage; - return TILEXR_ERROR_NOT_FOUND; - } - this_thread::sleep_for(1ms); - auto elapsed = duration_cast(high_resolution_clock::now() - start); - if (elapsed.count() > TILEXR_INIT_TIMEOUT) { - lock_guard lock(g_mtx); - g_directCcuAllGatherStates.erase(key); - TILEXR_LOG(ERROR) << "direct CCU thread allgather timeout rank " << rank_ << "/" << rankSize_ - << " uid " << uid_ << " round " << round; - return TILEXR_ERROR_TIMEOUT; - } - } -} - -int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report, - void *userData) -{ - auto *comm = static_cast(userData); - if (comm == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - return comm->FillDirectCcuLowerLayerPlanFromAllocation(allocation, plan, report); -} - -int TileXRComm::PrepareDirectCcuInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report) -{ - if (!inited_) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = "TileXRComm is not initialized for direct CCU install attempt"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = processUnavailableMessage; - } - return TILEXR_ERROR_NOT_FOUND; - } - const uint8_t installDieId = SelectDirectCcuInstallDieId(); - if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { - const int ret = RefreshDirectCcuBasicInfo(installDieId); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = directCcuBasicInfoReport_.message; - } - return ret; - } - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = "direct CCU runtime is unavailable for install attempt"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - TileXRCcuDriverAdapter adapter; - TileXRCcuDriverAdapterReport adapterReport; - int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = adapterReport.message; - } - return ret; - } - - TileXRCcuDirectInstallOptions next = options; - next.basicInfo = &directCcuBasicInfo_; - next.offlineOnly = false; - next.driverAdapter = &adapter; - next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); - next.repositoryMemoryUserData = nullptr; - next.lowerLayerPlan = nullptr; - next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback; - next.lowerLayerPlanUserData = this; - if (next.provider.empty()) { - next.provider = "tilexr-comm-direct-ccu"; - } - - return TileXRCcuRunDirectInstallAttempt(next, attempt, report); -} - -int TileXRComm::PrepareDirectCcuMemoryCopyInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - uint64_t localSourceAddr, - uint64_t localDestinationAddr, - uint64_t bytes, - uint32_t peerRank, - TileXRCcuMemoryCopyDirection direction, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report) -{ - if (!inited_) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "TileXRComm is not initialized for direct CCU memory copy install attempt"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - if (localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || - peerRank >= static_cast(rankSize_) || peerRank == static_cast(rank_)) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "invalid direct CCU memory copy endpoint"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = processUnavailableMessage; - } - return TILEXR_ERROR_NOT_FOUND; - } - const uint8_t installDieId = SelectDirectCcuInstallDieId(); - if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { - const int ret = RefreshDirectCcuBasicInfo(installDieId); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = directCcuBasicInfoReport_.message; - } - return ret; - } - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "direct CCU runtime is unavailable for memory copy install attempt"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - DirectCcuMemoryCopyEndpoint localEndpoint; - int ret = BuildDirectCcuLocalMemoryCopyEndpoint( - static_cast(rank_), - localSourceAddr, - localDestinationAddr, - bytes, - &localEndpoint); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to query direct CCU memory copy local buffer token"; - } - return ret; - } - - std::vector allEndpoints(static_cast(rankSize_)); - ret = DirectCcuAllGatherCallback( - &localEndpoint, - sizeof(localEndpoint), - allEndpoints.data(), - this); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to exchange direct CCU memory copy peer endpoints"; - } - return ret; - } - const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; - if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "invalid direct CCU memory copy peer endpoint"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - TileXRCcuDirectMemoryCopySpec memoryCopy; - memoryCopy.direction = direction; - memoryCopy.lengthBytes = bytes; - if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { - memoryCopy.localAddr = localEndpoint.destinationAddr; - memoryCopy.localToken = localEndpoint.destinationToken; - memoryCopy.remoteAddr = peerEndpoint.sourceAddr; - memoryCopy.remoteToken = peerEndpoint.sourceToken; - } else { - memoryCopy.localAddr = localEndpoint.sourceAddr; - memoryCopy.localToken = localEndpoint.sourceToken; - memoryCopy.remoteAddr = peerEndpoint.destinationAddr; - memoryCopy.remoteToken = peerEndpoint.destinationToken; - } - - TileXRCcuDriverAdapter adapter; - TileXRCcuDriverAdapterReport adapterReport; - ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = adapterReport.message; - } - return ret; - } - - TileXRCcuDirectInstallOptions next = options; - next.basicInfo = &directCcuBasicInfo_; - next.offlineOnly = false; - next.driverAdapter = &adapter; - next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); - next.repositoryMemoryUserData = nullptr; - next.lowerLayerPlan = nullptr; - next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback; - next.lowerLayerPlanUserData = this; - next.sqeArgCount = 0; - next.syncResourceCount = 1; - next.syncInstructionCount = std::max( - next.syncInstructionCount, - TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); - next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; - if (next.provider.empty()) { - next.provider = "tilexr-comm-direct-ccu-memory-copy"; - } - - return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); -} - -int TileXRComm::RefreshDirectCcuLowerLayerPlan() -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuLowerLayerTemplateConfigured_) { - directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer template is not configured"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuLowerLayerPlanReport_.message = "direct CCU runtime is unavailable for lower-layer planning"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot snapshot; - int ret = ccuDirectRuntime_->ExportLowerLayerTransportSnapshot(directCcuLowerLayerTemplate_, &snapshot); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU lower-layer transport snapshot"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &snapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerInstallPlan plan; - TileXRCcuLowerLayerPlanBuilderReport report; - ret = TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerSnapshot_ = snapshot; - directCcuLowerLayerPlan_ = plan; - directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer install plan cached"; - directCcuLowerLayerPlanValid_ = true; - directCcuLowerLayerPlanStatus_ = TILEXR_SUCCESS; - return TILEXR_SUCCESS; -} - -bool TileXRComm::HasDirectCcuLowerLayerPlan() const -{ - return directCcuLowerLayerPlanValid_; -} - -int TileXRComm::GetDirectCcuLowerLayerPlanStatus() const -{ - return directCcuLowerLayerPlanStatus_; -} - -const TileXRCcuLowerLayerPlanBuilderReport &TileXRComm::GetDirectCcuLowerLayerPlanReport() const -{ - return directCcuLowerLayerPlanReport_; -} - -const TileXRCcuLowerLayerInstallPlan *TileXRComm::GetDirectCcuLowerLayerPlan() const -{ - return directCcuLowerLayerPlanValid_ ? &directCcuLowerLayerPlan_ : nullptr; -} - -int TileXRComm::ReadDirectCcuInstructionsForDebug( - uint8_t dieId, - uint16_t instructionStartId, - void *instructions, - uint32_t instructionCount, - uint32_t instructionBytes, - TileXRCcuDriverAdapterReport *report) -{ - if (report != nullptr) { - *report = TileXRCcuDriverAdapterReport{}; - } - if (!inited_) { - if (report != nullptr) { - report->message = "TileXRComm is not initialized for direct CCU instruction readback"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - report->message = "direct CCU runtime is unavailable for instruction readback"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - TileXRCcuDriverAdapter adapter; - int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, report); - if (ret != TILEXR_SUCCESS) { - return ret; - } - return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); -} - int TileXRComm::InitCommon() { // enable peer device @@ -1533,10 +570,6 @@ int TileXRComm::Init() if (ret != TILEXR_SUCCESS) { return ret; } - ret = InitDirectCcuRuntime(); - if (ret != TILEXR_SUCCESS) { - return ret; - } ret = InitSDMA(); if (ret != TILEXR_SUCCESS) { return ret; @@ -1554,47 +587,6 @@ int TileXRComm::Init() return TILEXR_SUCCESS; } -int TileXRComm::InitDirectCcuOnly() -{ - if (inited_) { - return TILEXR_SUCCESS; - } - if (rank_ < 0 || rank_ >= rankSize_ || rankSize_ <= 0 || rankSize_ > TILEXR_MAX_RANK_SIZE) { - TILEXR_LOG(ERROR) << "The rank is invalid for direct CCU only init! rank:" << rank_ - << " rankSize:" << rankSize_; - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (TileXRSockExchange::CheckValid(commId_)) { - socketExchange_ = new (nothrow) TileXRSockExchange(rank_, rankSize_, commId_); - } else { - socketExchange_ = new (nothrow) TileXRSockExchange(rank_, rankSize_, commDomain_); - } - if (socketExchange_ == nullptr) { - TILEXR_LOG(ERROR) << "TileXRSockExchange create failed for direct CCU only init. rank : " - << rank_ << " rankSize:" << rankSize_; - return TILEXR_ERROR_INTERNAL; - } - - int ret = GetDev(); - if (ret != TILEXR_SUCCESS) { - TILEXR_LOG(ERROR) << "direct CCU only init context failed! ret: " << ret; - return ret; - } - TILEXR_LOG(INFO) << "direct CCU only init rank " << rank_ << "/" << rankSize_ - << " running devId:" << devId_; - - if (InitCommon() != TILEXR_SUCCESS) { - TILEXR_LOG(ERROR) << "direct CCU only init common failed!"; - return TILEXR_ERROR_INTERNAL; - } - ret = InitDirectCcuRuntime(); - if (ret != TILEXR_SUCCESS) { - return ret; - } - inited_ = true; - return TILEXR_SUCCESS; -} - int TileXRComm::InitThread(const std::string &uid) { if (inited_) { @@ -1650,10 +642,6 @@ int TileXRComm::InitThread(const std::string &uid) // UDMA 主要用于跨进程/跨节点通信，线程模式使用进程内共享内存即可 TILEXR_LOG(DEBUG) << "Thread mode: UDMA initialization skipped (single-process multi-thread scenario)"; - ret = InitDirectCcuRuntime(); - if (ret != TILEXR_SUCCESS) { - return ret; - } ret = InitSDMA(); if (ret != TILEXR_SUCCESS) { return ret; @@ -2015,7 +1003,10 @@ TileXRComm::~TileXRComm() } udmaRegisteredPtr_ = nullptr; udmaInfoDev_ = nullptr; - ResetDirectCcuBasicInfo(); + if (ccuBackend_ != nullptr) { + ccuBackend_->Shutdown(); + ccuBackend_.reset(); + } ResetSDMAState(); } diff --git a/src/comm/tilexr_comm.h b/src/comm/tilexr_comm.h index 279aa929..83d2c6ce 100644 --- a/src/comm/tilexr_comm.h +++ b/src/comm/tilexr_comm.h @@ -20,9 +20,6 @@ #include "../include/tilexr_types.h" #include "../include/tilexr_api.h" #include "../include/comm_args.h" -#include "ccu/tilexr_ccu_direct_orchestrator.h" -#include "ccu/tilexr_ccu_direct_runtime.h" -#include "ccu/tilexr_ccu_lower_layer_plan_builder.h" namespace TileXR { constexpr int IPC_NAME_SIZE = 65; @@ -30,6 +27,7 @@ constexpr int IPC_NAME_SIZE = 65; class TileXRSockExchange; class TileXRUDMATransport; class TileXRSDMATransport; +class TileXRCcuBackend; class TileXRComm { public: TileXRComm(int rank, int rankSize); @@ -39,7 +37,6 @@ class TileXRComm { TileXRComm(const TileXRComm &) = delete; TileXRComm &operator=(const TileXRComm &) = delete; int Init(); - int InitDirectCcuOnly(); int InitThread(const std::string &uid = "default"); int GetRank() const; int GetRankSize() const; @@ -52,44 +49,10 @@ class TileXRComm { int UnregisterUDMAMemory(TileXRUDMAMemHandle handle); GM_ADDR GetUDMARegistryPtr() const; const TileXRUDMARegistry* GetUDMARegistryHost() const; - int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); - bool HasDirectCcuBasicInfo() const; - int GetDirectCcuBasicInfoStatus() const; - const TileXRCcuBasicInfo *GetDirectCcuBasicInfo() const; - const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; - int ConfigureDirectCcuLowerLayerTemplate(const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot); - int ConfigureDirectCcuVerifiedEndpointRoutes( - const std::vector &verifiedRoutes); - int ConfigureDirectCcuLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route); - int ConfigureDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation, - const std::vector &remoteCcuBuffers); - int PrepareDirectCcuLowerLayerTemplateFromAllocation(const TileXRCcuResourceAllocation &allocation); - int RefreshDirectCcuLowerLayerPlan(); - bool HasDirectCcuLowerLayerPlan() const; - int GetDirectCcuLowerLayerPlanStatus() const; - const TileXRCcuLowerLayerPlanBuilderReport &GetDirectCcuLowerLayerPlanReport() const; - const TileXRCcuLowerLayerInstallPlan *GetDirectCcuLowerLayerPlan() const; - int ReadDirectCcuInstructionsForDebug( - uint8_t dieId, - uint16_t instructionStartId, - void *instructions, - uint32_t instructionCount, - uint32_t instructionBytes, - TileXRCcuDriverAdapterReport *report); - int PrepareDirectCcuInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report); - int PrepareDirectCcuMemoryCopyInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - uint64_t localSourceAddr, - uint64_t localDestinationAddr, - uint64_t bytes, - uint32_t peerRank, - TileXRCcuMemoryCopyDirection direction, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report); + int InitCcuBackend(); + TileXRCcuBackend *GetCcuBackendForCollectives(); + const TileXRCcuBackend *GetCcuBackendForCollectives() const; + int EnableCcuBackendForTest(); bool IsSDMAAvailable() const; GM_ADDR GetSDMAWorkspacePtr() const; SDMAInitStatus GetSDMAInitStatus() const; @@ -116,31 +79,9 @@ class TileXRComm { int SyncCommArgs(); int InitDumpAddr(); int InitUDMA(); - int InitDirectCcuRuntime(); int InitSDMA(); int UpdateCommArgsDev(); void FreeUDMARegistry(); - void ResetDirectCcuBasicInfo(); - void ResetDirectCcuLowerLayerPlan(); - int FillDirectCcuLowerLayerPlanFromAllocation( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report); - int ExchangeDirectCcuRemoteNotifyCke( - const TileXRCcuResourceAllocation &allocation, - std::vector *remoteCcuBuffers, - TileXRCcuLowerLayerPlanBuilderReport *report); - static int PrepareDirectCcuLowerLayerPlanCallback( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report, - void *userData); - static int DirectCcuAllGatherCallback( - const void *sendBuf, - size_t sendBytes, - void *recvBuf, - void *userData); - int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); void ResetSDMAState(); private: @@ -171,22 +112,7 @@ class TileXRComm { GM_ADDR udmaRegisteredPtr_ = nullptr; TileXRUDMARegistry udmaRegistry_ = {}; std::unique_ptr udmaTransport_; - std::unique_ptr ccuDirectRuntime_; - bool directCcuBasicInfoValid_ = false; - int directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - TileXRCcuBasicInfo directCcuBasicInfo_ = {}; - TileXRCcuDriverAdapterReport directCcuBasicInfoReport_ = {}; - bool directCcuLowerLayerTemplateConfigured_ = false; - bool directCcuLowerLayerPlanValid_ = false; - int directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerTemplate_ = {}; - TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerSnapshot_ = {}; - TileXRCcuLowerLayerInstallPlan directCcuLowerLayerPlan_ = {}; - TileXRCcuLowerLayerPlanBuilderReport directCcuLowerLayerPlanReport_ = {}; - std::vector directCcuVerifiedEndpointRoutes_ = {}; - TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; - bool directCcuLocalVerifiedEndpointRouteValid_ = false; - uint64_t directCcuThreadAllGatherRound_ = 0; + std::unique_ptr ccuBackend_; GM_ADDR sdmaWorkspaceDev_ = nullptr; SDMAInitStatus sdmaInitStatus_ = SDMAInitStatus::DISABLED_BY_ENV; std::unique_ptr sdmaTransport_; diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py new file mode 100644 index 00000000..5a5a0024 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +COMM_HEADER = REPO_ROOT / "src" / "comm" / "tilexr_comm.h" +BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" +BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" + + +class TileXRCcuBackendBoundaryTest(unittest.TestCase): + def test_backend_files_exist(self): + self.assertTrue(BACKEND_HEADER.exists()) + self.assertTrue(BACKEND_SOURCE.exists()) + + def test_tilexr_comm_header_owns_only_opaque_backend(self): + header = COMM_HEADER.read_text(encoding="utf-8") + self.assertIn("class TileXRCcuBackend;", header) + self.assertIn("std::unique_ptr ccuBackend_", header) + for needle in [ + "tilexr_ccu_direct_orchestrator.h", + "tilexr_ccu_direct_runtime.h", + "tilexr_ccu_lower_layer_plan_builder.h", + "TileXRCcuDirectRuntime", + "directCcuBasicInfo_", + "directCcuLowerLayerPlan_", + "directCcuVerifiedEndpointRoutes_", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, header) + + def test_backend_header_exposes_facade_not_public_c_api(self): + header = BACKEND_HEADER.read_text(encoding="utf-8") + self.assertIn("class TileXRCcuBackend", header) + self.assertIn("struct TileXRCcuBackendOptions", header) + self.assertIn("PrepareCollective", header) + self.assertIn("SubmitCollective", header) + for needle in [ + "TileXRDirectCcuPreparedTasksPtr", + "TileXRCommPrepareDirectCcu", + "TileXRDirectCcuSubmitPrepared", + ]: + with self.subTest(needle=needle): + self.assertNotIn(needle, header) + + +if __name__ == "__main__": + unittest.main() From 775ee7c7a5e78d5a96c0885b5b85d487cc8438fc Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 12:10:49 +0800 Subject: [PATCH 09/41] fix: preserve CCU backend internals --- src/collectives/host/collective_backend.cpp | 31 +- src/comm/ccu/tilexr_ccu_backend.cpp | 1189 ++++++++++++++++- src/comm/ccu/tilexr_ccu_backend.h | 11 +- src/comm/tilexr_comm.cpp | 2 +- tests/ccu/test_tilexr_ccu_backend_boundary.py | 24 + tests/comm/unit/test_tilexr_source_guards.cpp | 62 +- 6 files changed, 1276 insertions(+), 43 deletions(-) diff --git a/src/collectives/host/collective_backend.cpp b/src/collectives/host/collective_backend.cpp index 8f897a19..5b9cff3e 100644 --- a/src/collectives/host/collective_backend.cpp +++ b/src/collectives/host/collective_backend.cpp @@ -9,6 +9,9 @@ */ #include "collective_backend.h" +#include "../../comm/ccu/tilexr_ccu_backend.h" +#include "../../comm/tilexr_comm.h" + namespace TileXRCollectives { namespace Host { namespace { @@ -32,11 +35,33 @@ int DispatchUdma(const CollectiveRequest &request) int DispatchCcu(const CollectiveRequest &request) { - (void)request; - if (!g_testState.enabled || !g_testState.ccuInitialized) { + if (g_testState.enabled) { + if (!g_testState.ccuInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.ccuSupported ? g_testState.ccuReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; + } + + auto *comm = static_cast(request.comm); + TileXR::TileXRCcuBackend *backend = comm->GetCcuBackendForCollectives(); + if (backend == nullptr || !backend->Available()) { return TileXR::TILEXR_ERROR_NOT_INITIALIZED; } - return g_testState.ccuSupported ? g_testState.ccuReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; + TileXR::TileXRCcuCollectiveRequest ccuRequest {}; + ccuRequest.type = request.type; + ccuRequest.sendBuf = request.sendBuf; + ccuRequest.recvBuf = request.recvBuf; + ccuRequest.count = request.count; + ccuRequest.dataType = request.dataType; + ccuRequest.reduceOp = request.reduceOp; + ccuRequest.root = request.root; + ccuRequest.stream = request.stream; + TileXR::TileXRCcuCollectivePlan plan {}; + const int ret = backend->PrepareCollective(ccuRequest, &plan); + if (ret != TileXR::TILEXR_SUCCESS) { + return ret; + } + return backend->SubmitCollective(plan, request.stream); } } // namespace diff --git a/src/comm/ccu/tilexr_ccu_backend.cpp b/src/comm/ccu/tilexr_ccu_backend.cpp index 7a649295..a1efe86c 100644 --- a/src/comm/ccu/tilexr_ccu_backend.cpp +++ b/src/comm/ccu/tilexr_ccu_backend.cpp @@ -3,35 +3,1188 @@ */ #include "ccu/tilexr_ccu_backend.h" +#include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_memory_program.h" +#include "ccu/tilexr_ccu_repository.h" +#include "tilexr_log.h" +#include "tools/socket/tilexr_sock_exchange.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "runtime/dev.h" +#include "runtime/mem.h" +#include "runtime/rts/rts_device.h" + +using namespace std; +using namespace chrono; + namespace TileXR { -TileXRCcuBackend::TileXRCcuBackend() = default; +constexpr int TILEXR_INIT_TIMEOUT = 600; +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; -TileXRCcuBackend::~TileXRCcuBackend() +struct TileXRThreadAllGatherState { + std::vector data[TILEXR_MAX_RANK_SIZE]; + uint64_t arrivals = 0; + uint64_t departures = 0; + size_t bytes = 0; +}; +static map g_directCcuAllGatherStates; +static std::mutex g_mtx; +static std::mutex g_ccuDirectRuntimeMtx; +static bool g_ccuDirectRuntimeUnavailable = false; +static std::string g_ccuDirectRuntimeUnavailableMessage; + +uint8_t SelectDirectCcuInstallDieId() { - Shutdown(); + const char *text = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); + if (text == nullptr || text[0] == '\0') { + return 0; + } + char *end = nullptr; + const unsigned long parsed = std::strtoul(text, &end, 0); + if (end == text || *end != '\0' || parsed > 1UL) { + return 0; + } + return static_cast(parsed); } -int TileXRCcuBackend::Init(const TileXRCcuBackendOptions &options) +uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +{ + if (peerRouteCount == 0) { + return 0; + } + return static_cast(peerLocalIndex) + + static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); +} + +uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) { + return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); +} + +uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) +{ + if (syncRouteCount == 0) { + return 0; + } + return syncRouteCount; +} + +uint16_t SelectDirectCcuChannelBoundRemoteXnId( + uint16_t remoteXnStartId, + size_t peerLocalIndex, + uint32_t syncIndex, + size_t peerRouteCount) +{ + return static_cast( + static_cast(remoteXnStartId) + + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); +} + +uint16_t SelectDirectCcuRemoteNotifyCkeId(uint16_t remoteNotifyCkeStartId, size_t routeIndex) +{ + return static_cast(static_cast(remoteNotifyCkeStartId) + routeIndex); +} + +std::string ProcessDirectCcuRuntimeUnavailableMessage() +{ + lock_guard lock(g_ccuDirectRuntimeMtx); + if (!g_ccuDirectRuntimeUnavailable) { + return {}; + } + return g_ccuDirectRuntimeUnavailableMessage.empty() ? + "direct CCU runtime unavailable after process-level init failure" : + "direct CCU runtime unavailable after process-level init failure: " + + g_ccuDirectRuntimeUnavailableMessage; +} + +struct DirectCcuMemoryCopyEndpoint { + uint64_t sourceAddr = 0; + uint64_t sourceToken = 0; + uint64_t destinationAddr = 0; + uint64_t destinationToken = 0; + uint64_t bytes = 0; + uint32_t rank = 0; + uint32_t valid = 0; +}; + +int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) +{ + if (addr == 0 || bytes == 0 || packedToken == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *packedToken = 0; + rtMemUbTokenInfo info {}; + info.va = addr; + info.size = bytes; + const rtError_t ret = rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info); + if (ret != RT_ERROR_NONE) { + return TILEXR_ERROR_MKIRT; + } + constexpr uint32_t tokenIdRightShift = 8U; + const uint32_t tokenId = info.tokenId >> tokenIdRightShift; + *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); + return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; +} + +int BuildDirectCcuLocalMemoryCopyEndpoint( + uint32_t rank, + uint64_t sourceAddr, + uint64_t destinationAddr, + uint64_t bytes, + DirectCcuMemoryCopyEndpoint *endpoint) +{ + if (endpoint == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *endpoint = DirectCcuMemoryCopyEndpoint {}; + endpoint->rank = rank; + endpoint->bytes = bytes; + endpoint->sourceAddr = sourceAddr; + endpoint->destinationAddr = destinationAddr; + int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + endpoint->valid = 1; + return TILEXR_SUCCESS; +} + +class TileXRCcuBackend::Impl { +public: + int Init(const TileXRCcuBackendOptions &options); + void Shutdown(); + bool Available() const; + + int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); + bool HasDirectCcuBasicInfo() const; + int GetDirectCcuBasicInfoStatus() const; + const TileXRCcuBasicInfo *GetDirectCcuBasicInfo() const; + const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; + int ConfigureDirectCcuLowerLayerTemplate(const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot); + int ConfigureDirectCcuVerifiedEndpointRoutes( + const std::vector &verifiedRoutes); + int ConfigureDirectCcuLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route); + int ConfigureDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers); + int PrepareDirectCcuLowerLayerTemplateFromAllocation(const TileXRCcuResourceAllocation &allocation); + int PrepareDirectCcuInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int RefreshDirectCcuLowerLayerPlan(); + bool HasDirectCcuLowerLayerPlan() const; + int GetDirectCcuLowerLayerPlanStatus() const; + const TileXRCcuLowerLayerPlanBuilderReport &GetDirectCcuLowerLayerPlanReport() const; + const TileXRCcuLowerLayerInstallPlan *GetDirectCcuLowerLayerPlan() const; + int ReadDirectCcuInstructionsForDebug( + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report); + +private: + void ResetDirectCcuBasicInfo(); + void ResetDirectCcuLowerLayerPlan(); + int FillDirectCcuLowerLayerPlanFromAllocation( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report); + int ExchangeDirectCcuRemoteNotifyCke( + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report); + static int PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData); + static int DirectCcuAllGatherCallback(const void *sendBuf, size_t sendBytes, void *recvBuf, void *userData); + int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); + + TileXRCcuBackendOptions options_ = {}; + int rank_ = 0; + int rankSize_ = 0; + int devId_ = 0; + std::string uid_ = {}; + TileXRSockExchange *socketExchange_ = nullptr; + bool initialized_ = false; + std::unique_ptr ccuDirectRuntime_; + bool directCcuBasicInfoValid_ = false; + int directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuBasicInfo directCcuBasicInfo_ = {}; + TileXRCcuDriverAdapterReport directCcuBasicInfoReport_ = {}; + bool directCcuLowerLayerTemplateConfigured_ = false; + bool directCcuLowerLayerPlanValid_ = false; + int directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerTemplate_ = {}; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerSnapshot_ = {}; + TileXRCcuLowerLayerInstallPlan directCcuLowerLayerPlan_ = {}; + TileXRCcuLowerLayerPlanBuilderReport directCcuLowerLayerPlanReport_ = {}; + std::vector directCcuVerifiedEndpointRoutes_ = {}; + TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; + bool directCcuLocalVerifiedEndpointRouteValid_ = false; + uint64_t directCcuThreadAllGatherRound_ = 0; +}; + +void TileXRCcuBackend::Impl::Shutdown() +{ + initialized_ = false; + ResetDirectCcuBasicInfo(); + ResetDirectCcuLowerLayerPlan(); + directCcuLowerLayerTemplateConfigured_ = false; + directCcuLowerLayerTemplate_ = TileXRCcuLowerLayerTransportSnapshot {}; + directCcuVerifiedEndpointRoutes_.clear(); + directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; + directCcuLocalVerifiedEndpointRouteValid_ = false; + directCcuThreadAllGatherRound_ = 0; + if (ccuDirectRuntime_ != nullptr) { + ccuDirectRuntime_->Shutdown(); + ccuDirectRuntime_.reset(); + } + options_ = TileXRCcuBackendOptions {}; + socketExchange_ = nullptr; +} + +bool TileXRCcuBackend::Impl::Available() const +{ + return initialized_ && ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable(); +} + +int TileXRCcuBackend::Impl::Init(const TileXRCcuBackendOptions &options) +{ + Shutdown(); options_ = options; + rank_ = options.rank; + rankSize_ = options.rankSize; + devId_ = options.devId; + uid_ = options.uid; + socketExchange_ = options.exchange; + if (rankSize_ <= 1) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped for single-rank communicator"; + return TILEXR_SUCCESS; + } + + lock_guard lock(g_ccuDirectRuntimeMtx); + if (g_ccuDirectRuntimeUnavailable) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; + return TILEXR_SUCCESS; + } + + ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime()); + if (ccuDirectRuntime_ == nullptr) { + TILEXR_LOG(WARN) << "TileXRCcuDirectRuntime allocation failed, direct CCU disabled"; + return TILEXR_SUCCESS; + } + + TileXRCcuDirectRuntimeOptions runtimeOptions {}; + runtimeOptions.rank = rank_; + runtimeOptions.rankSize = rankSize_; + runtimeOptions.devId = devId_; + runtimeOptions.allGather = &TileXRCcuBackend::Impl::DirectCcuAllGatherCallback; + runtimeOptions.allGatherUserData = this; + TileXRCcuDirectRuntimeReport runtimeReport; + const int ret = ccuDirectRuntime_->Init(runtimeOptions, &runtimeReport); + if (ret != TILEXR_SUCCESS || !ccuDirectRuntime_->IsAvailable()) { + TILEXR_LOG(WARN) << "TileXR direct CCU runtime init failed: " << ret + << ", logicDevId " << runtimeReport.logicDevId + << ", devicePhyId " << runtimeReport.devicePhyId + << ", hdcType " << runtimeReport.hdcType + << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) + << ", " << runtimeReport.message << ", direct CCU disabled"; + g_ccuDirectRuntimeUnavailable = true; + g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; + ResetDirectCcuBasicInfo(); + ccuDirectRuntime_.reset(); + return TILEXR_SUCCESS; + } + + const int ccuInfoRet = RefreshDirectCcuBasicInfo(0); + if (ccuInfoRet != TILEXR_SUCCESS && ccuInfoRet != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU basic info refresh failed after runtime init: " << ccuInfoRet + << ", " << directCcuBasicInfoReport_.message; + } + + TILEXR_LOG(INFO) << "InitDirectCcuRuntime success, rank " << rank_ << "/" << rankSize_ + << " logicDevId " << runtimeReport.logicDevId + << " devicePhyId " << runtimeReport.devicePhyId + << " hdcType " << runtimeReport.hdcType + << " raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << " ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0); initialized_ = true; return TILEXR_SUCCESS; } +void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo() +{ + directCcuBasicInfoValid_ = false; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuBasicInfo_ = TileXRCcuBasicInfo {}; + directCcuBasicInfoReport_ = TileXRCcuDriverAdapterReport {}; +} + +void TileXRCcuBackend::Impl::ResetDirectCcuLowerLayerPlan() +{ + directCcuLowerLayerPlanValid_ = false; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuLowerLayerSnapshot_ = TileXRCcuLowerLayerTransportSnapshot {}; + directCcuLowerLayerPlan_ = TileXRCcuLowerLayerInstallPlan {}; + directCcuLowerLayerPlanReport_ = TileXRCcuLowerLayerPlanBuilderReport {}; +} + +int TileXRCcuBackend::Impl::RefreshDirectCcuBasicInfo(uint8_t dieId) +{ + ResetDirectCcuBasicInfo(); + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuBasicInfoReport_.message = "direct CCU runtime is unavailable for basic info"; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuBasicInfoStatus_; + } + + TileXRCcuBasicInfo basicInfo; + TileXRCcuDriverAdapterReport report; + const int ret = ccuDirectRuntime_->QueryBasicInfo(dieId, &basicInfo, &report); + directCcuBasicInfoReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuBasicInfoStatus_ = ret; + return directCcuBasicInfoStatus_; + } + + directCcuBasicInfo_ = basicInfo; + directCcuBasicInfoReport_.message = "direct CCU basic info cached"; + directCcuBasicInfoValid_ = true; + directCcuBasicInfoStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRCcuBackend::Impl::HasDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_; +} + +int TileXRCcuBackend::Impl::GetDirectCcuBasicInfoStatus() const +{ + return directCcuBasicInfoStatus_; +} + +const TileXRCcuBasicInfo *TileXRCcuBackend::Impl::GetDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_ ? &directCcuBasicInfo_ : nullptr; +} + +const TileXRCcuDriverAdapterReport &TileXRCcuBackend::Impl::GetDirectCcuBasicInfoReport() const +{ + return directCcuBasicInfoReport_; +} + +int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplate( + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot) +{ + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRCcuBackend::Impl::ConfigureDirectCcuVerifiedEndpointRoutes( + const std::vector &verifiedRoutes) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes = verifiedRoutes; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuVerifiedEndpointRoutes_ = verifiedRoutes; + if (directCcuLowerLayerTemplateConfigured_) { + return RefreshDirectCcuLowerLayerPlan(); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::Impl::ConfigureDirectCcuLocalVerifiedEndpointRoute( + const TileXRCcuLowerLayerTransportRoute &route) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes.push_back(route); + TileXRCcuLowerLayerPlanBuilderReport report; + std::vector routes {route}; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(routes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; + directCcuLocalVerifiedEndpointRouteValid_ = false; + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuLocalVerifiedEndpointRoute_ = route; + directCcuLocalVerifiedEndpointRouteValid_ = true; + if (ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable()) { + return ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(route); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers) +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuBasicInfoValid_) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuBuildLowerLayerTransportTemplate( + directCcuBasicInfo_, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation( + const TileXRCcuResourceAllocation &allocation) +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuBasicInfoValid_) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuLowerLayerPlanReport_.message = + "direct CCU runtime is unavailable for resource window registration"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + int ret = ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to register direct CCU resource window"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLocalResourceWindowInfo localCcuResourceWindow; + ret = ccuDirectRuntime_->ExportLocalCcuRmaBuffer(&localCcuResourceWindow); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU local resource window token"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + if (directCcuLocalVerifiedEndpointRouteValid_) { + ret = ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(directCcuLocalVerifiedEndpointRoute_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to configure direct CCU local verified endpoint route"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + } else { + TileXRCcuDirectRuntimeReport endpointRouteReport; + ret = ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute(&endpointRouteReport); + if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU local endpoint route collection failed closed: " + << ret << ", " << endpointRouteReport.message; + } + } + + std::vector remoteCcuBuffers; + ret = ccuDirectRuntime_->ExportRemoteCcuRmaBuffers(&remoteCcuBuffers); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU peer resource window tokens"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = ExchangeDirectCcuRemoteNotifyCke(allocation, &remoteCcuBuffers, &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerTransportTemplate( + directCcuBasicInfo_, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + templateSnapshot.msidToken.dieId = directCcuBasicInfo_.dieId; + templateSnapshot.msidToken.msId = directCcuBasicInfo_.msId; + templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId; + templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue; + templateSnapshot.msidToken.valid = true; + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(); +} + +int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (plan == nullptr || report == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + int ret = PrepareDirectCcuLowerLayerTemplateFromAllocation(allocation); + if (ret != TILEXR_SUCCESS) { + *report = directCcuLowerLayerPlanReport_; + return ret; + } + if (!directCcuLowerLayerPlanValid_) { + *report = directCcuLowerLayerPlanReport_; + return TILEXR_ERROR_NOT_FOUND; + } + *plan = directCcuLowerLayerPlan_; + *report = directCcuLowerLayerPlanReport_; + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke( + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (remoteCcuBuffers == nullptr) { + if (report != nullptr) { + report->message = "missing direct CCU remote notify CKE exchange inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize_ <= 1 || rank_ < 0 || rank_ >= rankSize_) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const size_t peerRouteCount = static_cast(rankSize_ - 1); + const size_t syncRouteCount = allocation.remoteXn.num; + if (allocation.localXn.num == 0 || + allocation.localWaitCke.num == 0 || + allocation.remoteNotifyCke.num == 0 || + allocation.remoteXn.num < static_cast(rankSize_ - 1) || + allocation.localWaitCke.num < allocation.remoteXn.num || + allocation.remoteNotifyCke.num < allocation.remoteXn.num || + allocation.channels.num == 0 || + remoteCcuBuffers->size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + struct PeerResourceExchange { + uint16_t localXnStartId; + uint16_t localXnCount; + uint16_t remoteXnStartId; + uint16_t remoteXnCount; + uint16_t localWaitCkeStartId; + uint16_t localWaitCkeCount; + uint16_t remoteNotifyCkeStartId; + uint16_t remoteNotifyCkeCount; + uint16_t channelStartId; + uint16_t channelCount; + }; + PeerResourceExchange local { + allocation.localXn.startId, + allocation.localXn.num, + allocation.remoteXn.startId, + DirectCcuRemoteXnProofSpan(allocation.remoteXn.num), + allocation.localWaitCke.startId, + allocation.localWaitCke.num, + allocation.remoteNotifyCke.startId, + allocation.remoteNotifyCke.num, + allocation.channels.startId, + allocation.channels.num, + }; + std::vector all(rankSize_); + const int ret = DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to exchange direct CCU peer XN/CKE resources"; + } + return ret; + } + + std::vector peerRanks; + peerRanks.reserve(peerRouteCount); + for (int peer = 0; peer < rankSize_; ++peer) { + if (peer != rank_) { + peerRanks.push_back(peer); + } + } + if (peerRanks.size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector peerCcuBuffers = *remoteCcuBuffers; + remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); + + size_t routeIndex = 0; + for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { + const size_t peerBufferIndex = syncIndex % peerRouteCount; + const int peer = peerRanks[peerBufferIndex]; + const PeerResourceExchange &peerResources = all[peer]; + const size_t peerLocalIndex = static_cast(rank_ < peer ? rank_ : rank_ - 1); + const uint32_t peerLocalXnOffset = + SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t selectedRemoteXnOffset = + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t peerLocalWaitCkeOffset = routeIndex; + if (peerResources.localXnCount == 0 || + peerResources.remoteXnCount == 0 || + peerResources.localWaitCkeCount == 0 || + peerResources.channelCount == 0 || + peerLocalXnOffset >= peerResources.localXnCount || + selectedRemoteXnOffset >= peerResources.remoteXnCount || + peerLocalIndex >= peerResources.channelCount || + peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { + if (report != nullptr) { + report->message = "peer direct CCU local XN/CKE resources are incomplete"; + } + return TILEXR_ERROR_NOT_FOUND; + } + uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( + peerResources.remoteXnStartId, + peerLocalIndex, + syncIndex, + peerRouteCount); + const uint16_t peerLocalXnId = + static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); + uint16_t remoteNotifyCke = + static_cast(static_cast(peerResources.localWaitCkeStartId) + + peerLocalWaitCkeOffset); + (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; + (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; + (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; + const bool peerLocalXnOwnerVerified = + static_cast(peerLocalXnId) >= peerResources.localXnStartId && + static_cast(peerLocalXnId) < + static_cast(peerResources.localXnStartId) + peerResources.localXnCount; + const bool notifyCkeOwnerVerified = + static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && + static_cast(remoteNotifyCke) < + static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; + const bool localChannelOwnerVerified = + allocation.channels.num != 0 && + peerLocalXnOwnerVerified && + static_cast(channelBoundRemoteXnId) >= peerResources.remoteXnStartId && + static_cast(channelBoundRemoteXnId) < + static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && + routeIndex < allocation.channels.num && + peerResources.channelStartId != 0 && + peerLocalIndex < peerResources.channelCount; + const bool transportResourceExchangeVerified = + notifyCkeOwnerVerified && + allocation.localWaitCke.num != 0 && + routeIndex < allocation.localWaitCke.num && + peerLocalWaitCkeOffset < peerResources.localWaitCkeCount; + (*remoteCcuBuffers)[routeIndex].channelResourceOwnerVerified = localChannelOwnerVerified; + (*remoteCcuBuffers)[routeIndex].transportResourceExchangeVerified = transportResourceExchangeVerified; + ++routeIndex; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback( + const void *sendBuf, + size_t sendBytes, + void *recvBuf, + void *userData) +{ + auto *backend = static_cast(userData); + if (backend == nullptr || sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (backend->socketExchange_ == nullptr) { + return backend->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf); + } + return backend->socketExchange_->AllGather( + static_cast(sendBuf), + sendBytes, + static_cast(recvBuf)); +} + +int TileXRCcuBackend::Impl::DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) +{ + if (sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0 || rank_ < 0 || + rank_ >= rankSize_ || rankSize_ <= 0 || uid_.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint64_t round = directCcuThreadAllGatherRound_++; + const std::string key = uid_ + ":ccu:" + std::to_string(round); + auto start = high_resolution_clock::now(); + for (;;) { + { + lock_guard lock(g_mtx); + auto &state = g_directCcuAllGatherStates[key]; + if (state.bytes == 0) { + state.bytes = sendBytes; + } else if (state.bytes != sendBytes) { + g_directCcuAllGatherStates.erase(key); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (state.data[rank_].empty()) { + state.data[rank_].resize(sendBytes); + std::memcpy(state.data[rank_].data(), sendBuf, sendBytes); + ++state.arrivals; + } + if (state.arrivals == static_cast(rankSize_)) { + auto *output = static_cast(recvBuf); + for (int i = 0; i < rankSize_; ++i) { + std::memcpy(output + static_cast(i) * sendBytes, state.data[i].data(), sendBytes); + } + ++state.departures; + if (state.departures == static_cast(rankSize_)) { + g_directCcuAllGatherStates.erase(key); + } + return TILEXR_SUCCESS; + } + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round << ", " + << processUnavailableMessage; + return TILEXR_ERROR_NOT_FOUND; + } + this_thread::sleep_for(1ms); + auto elapsed = duration_cast(high_resolution_clock::now() - start); + if (elapsed.count() > TILEXR_INIT_TIMEOUT) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather timeout rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round; + return TILEXR_ERROR_TIMEOUT; + } + } +} + +int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData) +{ + auto *backend = static_cast(userData); + if (backend == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return backend->FillDirectCcuLowerLayerPlanFromAllocation(allocation, plan, report); +} + +int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!initialized_) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "TileXRCcuBackend is not initialized for direct CCU install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { + const int ret = RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = directCcuBasicInfoReport_.message; + } + return ret; + } + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "direct CCU runtime is unavailable for install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = adapterReport.message; + } + return ret; + } + + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = &directCcuBasicInfo_; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = this; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu"; + } + + return TileXRCcuRunDirectInstallAttempt(next, attempt, report); +} + +int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt( + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!initialized_) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU memory copy install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize_) || peerRank == static_cast(rank_)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { + const int ret = RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = directCcuBasicInfoReport_.message; + } + return ret; + } + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for memory copy install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + int ret = BuildDirectCcuLocalMemoryCopyEndpoint( + static_cast(rank_), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU memory copy local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize_)); + ret = DirectCcuAllGatherCallback( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data(), + this); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU memory copy peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuDirectMemoryCopySpec memoryCopy; + memoryCopy.direction = direction; + memoryCopy.lengthBytes = bytes; + if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { + memoryCopy.localAddr = localEndpoint.destinationAddr; + memoryCopy.localToken = localEndpoint.destinationToken; + memoryCopy.remoteAddr = peerEndpoint.sourceAddr; + memoryCopy.remoteToken = peerEndpoint.sourceToken; + } else { + memoryCopy.localAddr = localEndpoint.sourceAddr; + memoryCopy.localToken = localEndpoint.sourceToken; + memoryCopy.remoteAddr = peerEndpoint.destinationAddr; + memoryCopy.remoteToken = peerEndpoint.destinationToken; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = &directCcuBasicInfo_; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = this; + next.sqeArgCount = 0; + next.syncResourceCount = 1; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-memory-copy"; + } + + return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); +} + +int TileXRCcuBackend::Impl::RefreshDirectCcuLowerLayerPlan() +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuLowerLayerTemplateConfigured_) { + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer template is not configured"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuLowerLayerPlanReport_.message = "direct CCU runtime is unavailable for lower-layer planning"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + int ret = ccuDirectRuntime_->ExportLowerLayerTransportSnapshot(directCcuLowerLayerTemplate_, &snapshot); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU lower-layer transport snapshot"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &snapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerSnapshot_ = snapshot; + directCcuLowerLayerPlan_ = plan; + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer install plan cached"; + directCcuLowerLayerPlanValid_ = true; + directCcuLowerLayerPlanStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRCcuBackend::Impl::HasDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_; +} + +int TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlanStatus() const +{ + return directCcuLowerLayerPlanStatus_; +} + +const TileXRCcuLowerLayerPlanBuilderReport &TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlanReport() const +{ + return directCcuLowerLayerPlanReport_; +} + +const TileXRCcuLowerLayerInstallPlan *TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_ ? &directCcuLowerLayerPlan_ : nullptr; +} + +int TileXRCcuBackend::Impl::ReadDirectCcuInstructionsForDebug( + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report) +{ + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + } + if (!initialized_) { + if (report != nullptr) { + report->message = "TileXRCcuBackend is not initialized for direct CCU instruction readback"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + report->message = "direct CCU runtime is unavailable for instruction readback"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); +} +TileXRCcuBackend::TileXRCcuBackend() : impl_(new (std::nothrow) Impl()) +{ +} + +TileXRCcuBackend::~TileXRCcuBackend() +{ + Shutdown(); +} + +int TileXRCcuBackend::Init(const TileXRCcuBackendOptions &options) +{ + if (impl_ == nullptr) { + impl_.reset(new (std::nothrow) Impl()); + if (impl_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + } + return impl_->Init(options); +} + void TileXRCcuBackend::Shutdown() { - initialized_ = false; + if (impl_ != nullptr) { + impl_->Shutdown(); + } } bool TileXRCcuBackend::Available() const { - return initialized_; + return impl_ != nullptr && impl_->Available(); } bool TileXRCcuBackend::Supports(const TileXRCcuCollectiveRequest &request) const { - return initialized_ && request.type == TileXRType::ALL_GATHER; + (void)request; + return false; } int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan) @@ -39,23 +1192,27 @@ int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &reques if (plan == nullptr) { return TILEXR_ERROR_PARA_CHECK_FAIL; } - if (!initialized_) { + *plan = TileXRCcuCollectivePlan {}; + if (!Available()) { return TILEXR_ERROR_NOT_INITIALIZED; } - if (!Supports(request)) { - return TILEXR_ERROR_NOT_SUPPORT; - } - *plan = TileXRCcuCollectivePlan {}; - plan->ready = true; - return TILEXR_SUCCESS; + (void)request; + return TILEXR_ERROR_NOT_SUPPORT; } int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream) { - if (!initialized_) { + if (!Available()) { return TILEXR_ERROR_NOT_INITIALIZED; } - return plan.ready ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; + return plan.ready ? TILEXR_ERROR_NOT_SUPPORT : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +#ifdef TILEXR_CCU_TESTING +bool TileXRCcuBackend::RuntimeInitializedForTest() const +{ + return Available(); } +#endif } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_backend.h b/src/comm/ccu/tilexr_ccu_backend.h index 123507b2..8177e211 100644 --- a/src/comm/ccu/tilexr_ccu_backend.h +++ b/src/comm/ccu/tilexr_ccu_backend.h @@ -14,7 +14,7 @@ namespace TileXR { -class TileXRComm; +class TileXRSockExchange; class TileXRCcuRuntimeSession; class TileXRCcuCollectivePlanner; class TileXRCcuExecutor; @@ -24,7 +24,7 @@ struct TileXRCcuBackendOptions { int rankSize = 0; int devId = 0; std::string uid; - TileXRComm *comm = nullptr; + TileXRSockExchange *exchange = nullptr; }; struct TileXRCcuCollectiveRequest { @@ -56,10 +56,13 @@ class TileXRCcuBackend { bool Supports(const TileXRCcuCollectiveRequest &request) const; int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); +#ifdef TILEXR_CCU_TESTING + bool RuntimeInitializedForTest() const; +#endif private: - TileXRCcuBackendOptions options_; - bool initialized_ = false; + class Impl; + std::unique_ptr impl_; }; } // namespace TileXR diff --git a/src/comm/tilexr_comm.cpp b/src/comm/tilexr_comm.cpp index 889db05e..82fac246 100644 --- a/src/comm/tilexr_comm.cpp +++ b/src/comm/tilexr_comm.cpp @@ -186,7 +186,7 @@ int TileXRComm::InitCcuBackend() options.rankSize = rankSize_; options.devId = devId_; options.uid = uid_; - options.comm = this; + options.exchange = socketExchange_; return ccuBackend_->Init(options); } diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 5a5a0024..914f8252 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -38,6 +38,7 @@ def test_backend_header_exposes_facade_not_public_c_api(self): header = BACKEND_HEADER.read_text(encoding="utf-8") self.assertIn("class TileXRCcuBackend", header) self.assertIn("struct TileXRCcuBackendOptions", header) + self.assertIn("TileXRSockExchange *exchange", header) self.assertIn("PrepareCollective", header) self.assertIn("SubmitCollective", header) for needle in [ @@ -48,6 +49,29 @@ def test_backend_header_exposes_facade_not_public_c_api(self): with self.subTest(needle=needle): self.assertNotIn(needle, header) + def test_backend_source_owns_restored_direct_ccu_runtime_glue(self): + source = BACKEND_SOURCE.read_text(encoding="utf-8") + for needle in [ + "#include \"ccu/tilexr_ccu_direct_runtime.h\"", + "#include \"ccu/tilexr_ccu_repository.h\"", + "TileXRCcuDirectRuntime", + "PrepareDirectCcuInstallAttempt", + "PrepareDirectCcuLowerLayerPlanCallback", + "TileXRCcuRunDirectInstallAttempt", + "TileXRCcuMakeRepositoryDeviceMemoryOps", + "DirectCcuThreadAllGather", + "g_directCcuAllGatherStates", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, source) + for fake_ready in [ + "options_ = options;\n initialized_ = true;\n return TILEXR_SUCCESS;", + "plan->ready = true;\n return TILEXR_SUCCESS;", + "return plan.ready ? TILEXR_SUCCESS", + ]: + with self.subTest(fake_ready=fake_ready): + self.assertNotIn(fake_ready, source) + if __name__ == "__main__": unittest.main() diff --git a/tests/comm/unit/test_tilexr_source_guards.cpp b/tests/comm/unit/test_tilexr_source_guards.cpp index dc66cdb4..4e69d275 100644 --- a/tests/comm/unit/test_tilexr_source_guards.cpp +++ b/tests/comm/unit/test_tilexr_source_guards.cpp @@ -403,29 +403,51 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() { const std::string commHeaderPath = "src/comm/tilexr_comm.h"; const std::string commSourcePath = "src/comm/tilexr_comm.cpp"; + const std::string ccuBackendHeaderPath = "src/comm/ccu/tilexr_ccu_backend.h"; + const std::string ccuBackendSourcePath = "src/comm/ccu/tilexr_ccu_backend.cpp"; const std::string ccuRuntimeHeaderPath = "src/comm/ccu/tilexr_ccu_direct_runtime.h"; const std::string ccuRuntimeSourcePath = "src/comm/ccu/tilexr_ccu_direct_runtime.cpp"; const auto commHeaderText = ReadFile(commHeaderPath); const auto commSourceText = ReadFile(commSourcePath); + const auto ccuBackendHeaderText = ReadFile(ccuBackendHeaderPath); + const auto ccuBackendSourceText = ReadFile(ccuBackendSourcePath); const auto ccuRuntimeHeaderText = ReadFile(ccuRuntimeHeaderPath); const auto ccuRuntimeSourceText = ReadFile(ccuRuntimeSourcePath); - CheckContains(commHeaderPath, commHeaderText, "PrepareDirectCcuInstallAttempt"); - CheckContains(commHeaderPath, commHeaderText, "FillDirectCcuLowerLayerPlanFromAllocation"); - CheckContains(commHeaderPath, commHeaderText, "PrepareDirectCcuLowerLayerPlanCallback"); - CheckContains(commHeaderPath, commHeaderText, "std::unique_ptr ccuDirectRuntime_"); - CheckContains(commSourcePath, commSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); - CheckContains(commSourcePath, commSourceText, "int TileXRComm::InitDirectCcuRuntime"); - CheckContains(commSourcePath, commSourceText, "int TileXRComm::PrepareDirectCcuInstallAttempt"); - CheckContains(commSourcePath, commSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); - CheckContains(commSourcePath, commSourceText, "TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)"); - CheckContains(commSourcePath, commSourceText, "next.lowerLayerPlan = nullptr"); + CheckContains(commHeaderPath, commHeaderText, "class TileXRCcuBackend;"); + CheckContains(commHeaderPath, commHeaderText, "std::unique_ptr ccuBackend_"); + for (const auto& forbiddenCommDetail : { + "tilexr_ccu_direct_runtime.h", + "tilexr_ccu_lower_layer_plan_builder.h", + "TileXRCcuDirectRuntime", + "directCcuBasicInfo_", + "directCcuLowerLayerPlan_", + "directCcuVerifiedEndpointRoutes_", + "PrepareDirectCcuInstallAttempt", + "PrepareDirectCcuLowerLayerPlanCallback", + }) { + CheckNotContains(commHeaderPath, commHeaderText, forbiddenCommDetail); + } + + CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "class TileXRCcuBackend"); + CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "TileXRSockExchange *exchange"); + CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "std::unique_ptr impl_"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_direct_runtime.h\""); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuBackend::Impl::Init"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); + CheckContains( + ccuBackendSourcePath, + ccuBackendSourceText, + "TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "next.lowerLayerPlan = nullptr"); CheckContains( - commSourcePath, - commSourceText, - "next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback"); - CheckContains(commSourcePath, commSourceText, "next.lowerLayerPlanUserData = this"); - CheckContains(commSourcePath, commSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); + ccuBackendSourcePath, + ccuBackendSourceText, + "next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "next.lowerLayerPlanUserData = this"); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "int CreateDriverAdapter("); CheckContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, "int TileXRCcuDirectRuntime::CreateDriverAdapter"); CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "TileXRCcuHccpLoader"); @@ -441,13 +463,13 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() } const auto initUdmaBegin = commSourceText.find("int TileXRComm::InitUDMA"); - const auto initDirectCcuBegin = commSourceText.find("int TileXRComm::InitDirectCcuRuntime"); - if (initUdmaBegin == std::string::npos || initDirectCcuBegin == std::string::npos || - initUdmaBegin >= initDirectCcuBegin) { + const auto initCcuBackendBegin = commSourceText.find("int TileXRComm::InitCcuBackend"); + if (initUdmaBegin == std::string::npos || initCcuBackendBegin == std::string::npos || + initUdmaBegin >= initCcuBackendBegin) { std::cerr << commSourcePath << ": cannot isolate InitUDMA body" << std::endl; ++g_failures; } else { - const auto initUdmaBody = commSourceText.substr(initUdmaBegin, initDirectCcuBegin - initUdmaBegin); + const auto initUdmaBody = commSourceText.substr(initUdmaBegin, initCcuBackendBegin - initUdmaBegin); CheckNotContains(commSourcePath, initUdmaBody, "RefreshDirectCcuBasicInfo"); CheckNotContains(commSourcePath, initUdmaBody, "ResetDirectCcuBasicInfo"); } @@ -480,6 +502,8 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() }) { CheckNotContains(commHeaderPath, commHeaderText, forbidden); CheckNotContains(commSourcePath, commSourceText, forbidden); + CheckNotContains(ccuBackendHeaderPath, ccuBackendHeaderText, forbidden); + CheckNotContains(ccuBackendSourcePath, ccuBackendSourceText, forbidden); CheckNotContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, forbidden); CheckNotContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, forbidden); } From 84857b03079ab221b90c491f138fa10151f353ea Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 12:41:23 +0800 Subject: [PATCH 10/41] fix: migrate CCU lower-layer source guards --- ...est_tilexr_ccu_lower_layer_plan_builder.py | 415 +++++++++--------- 1 file changed, 213 insertions(+), 202 deletions(-) diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 4f65ad4b..1d0c4cef 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -25,6 +25,8 @@ MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" COMM_HEADER_FILE = REPO_ROOT / "src" / "comm" / "tilexr_comm.h" COMM_SOURCE_FILE = REPO_ROOT / "src" / "comm" / "tilexr_comm.cpp" +CCU_BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" +CCU_BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" INCLUDE_DIR = REPO_ROOT / "src" / "include" COMM_DIR = REPO_ROOT / "src" / "comm" @@ -1820,32 +1822,39 @@ def test_plan_builder_surface_is_wired_into_tilexr_comm_without_udma_boundary(se def test_tilexr_comm_caches_direct_ccu_lower_layer_plan_from_ccu_runtime(self): comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - - self.assertIn('ccu/tilexr_ccu_lower_layer_plan_builder.h', comm_header) - self.assertIn('ccu/tilexr_ccu_direct_runtime.h', comm_header) - self.assertIn('ccu/tilexr_ccu_repository.h', comm_source) - self.assertIn("ConfigureDirectCcuLowerLayerTemplate", comm_header) - self.assertIn("ConfigureDirectCcuLowerLayerTemplateFromAllocation", comm_header) - self.assertIn("RefreshDirectCcuLowerLayerPlan", comm_header) - self.assertIn("PrepareDirectCcuLowerLayerTemplateFromAllocation", comm_header) - self.assertIn("HasDirectCcuLowerLayerPlan", comm_header) - self.assertIn("GetDirectCcuLowerLayerPlanStatus", comm_header) - self.assertIn("GetDirectCcuLowerLayerPlanReport", comm_header) - self.assertIn("ccuDirectRuntime_", comm_header) - - self.assertIn("int TileXRComm::ConfigureDirectCcuLowerLayerTemplate", comm_source) - self.assertIn("int TileXRComm::ConfigureDirectCcuLowerLayerTemplateFromAllocation", comm_source) - self.assertIn("int TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation", comm_source) - self.assertIn("int TileXRComm::RefreshDirectCcuLowerLayerPlan", comm_source) - self.assertIn("TileXRCcuBuildLowerLayerTransportTemplate", comm_source) - self.assertIn("const std::vector &remoteCcuBuffers", comm_source) - self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", comm_source) - self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr)", comm_source) - self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", comm_source) - self.assertIn("ccuDirectRuntime_->ExportLowerLayerTransportSnapshot", comm_source) - self.assertIn("RefreshDirectCcuLowerLayerPlan();", comm_source) - self.assertIn("direct CCU lower-layer template is not configured", comm_source) - self.assertIn("direct CCU lower-layer install plan cached", comm_source) + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + + for leaked in [ + 'ccu/tilexr_ccu_lower_layer_plan_builder.h', + 'ccu/tilexr_ccu_direct_runtime.h', + "ConfigureDirectCcuLowerLayerTemplate", + "ConfigureDirectCcuLowerLayerTemplateFromAllocation", + "RefreshDirectCcuLowerLayerPlan", + "PrepareDirectCcuLowerLayerTemplateFromAllocation", + "HasDirectCcuLowerLayerPlan", + "GetDirectCcuLowerLayerPlanStatus", + "GetDirectCcuLowerLayerPlanReport", + "ccuDirectRuntime_", + ]: + with self.subTest(leaked=leaked): + self.assertNotIn(leaked, comm_header) + + self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', backend_source) + self.assertIn('ccu/tilexr_ccu_direct_runtime.h', backend_source) + self.assertIn('ccu/tilexr_ccu_repository.h', backend_source) + self.assertIn("int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplate", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplateFromAllocation", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::RefreshDirectCcuLowerLayerPlan", backend_source) + self.assertIn("TileXRCcuBuildLowerLayerTransportTemplate", backend_source) + self.assertIn("const std::vector &remoteCcuBuffers", backend_source) + self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", backend_source) + self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr)", backend_source) + self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", backend_source) + self.assertIn("ccuDirectRuntime_->ExportLowerLayerTransportSnapshot", backend_source) + self.assertIn("RefreshDirectCcuLowerLayerPlan();", backend_source) + self.assertIn("direct CCU lower-layer template is not configured", backend_source) + self.assertIn("direct CCU lower-layer install plan cached", backend_source) register_body = comm_source[ comm_source.index("int TileXRComm::RegisterUDMAMemory"): @@ -1863,7 +1872,7 @@ def test_tilexr_comm_caches_direct_ccu_lower_layer_plan_from_ccu_runtime(self): init_udma_body = comm_source[ comm_source.index("int TileXRComm::InitUDMA"): - comm_source.index("int TileXRComm::InitDirectCcuRuntime") + comm_source.index("int TileXRComm::InitCcuBackend") ] self.assertNotIn("RefreshDirectCcuBasicInfo", init_udma_body) self.assertNotIn("ResetDirectCcuBasicInfo", init_udma_body) @@ -1884,7 +1893,7 @@ def test_tilexr_comm_caches_direct_ccu_lower_layer_plan_from_ccu_runtime(self): "libhcomm", "libhccl_v2", ]: - self.assertNotIn(forbidden, comm_header + "\n" + comm_source) + self.assertNotIn(forbidden, comm_header + "\n" + comm_source + "\n" + backend_source) def test_direct_ccu_runtime_owns_resource_window_boundary(self): runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") @@ -1892,6 +1901,8 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): specs_header = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.h").read_text(encoding="utf-8") comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") builder_header = BUILDER_HEADER.read_text(encoding="utf-8") self.assertIn("TileXRCcuLocalResourceWindowInfo", runtime_header) @@ -1917,58 +1928,66 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertIn("route.doorbellVa", runtime_source) self.assertIn("route.sqDepth", runtime_source) - self.assertIn("ConfigureDirectCcuLowerLayerTemplateFromAllocation(", comm_header) - self.assertIn("const std::vector &remoteCcuBuffers", comm_header) - self.assertIn("PrepareDirectCcuLowerLayerTemplateFromAllocation", comm_header) - self.assertIn("ExchangeDirectCcuRemoteNotifyCke", comm_header) - self.assertIn("DirectCcuAllGatherCallback", comm_header) - self.assertIn("DirectCcuThreadAllGather", comm_header) - self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer", comm_source) - self.assertIn("ccuDirectRuntime_->ExportLocalCcuRmaBuffer", comm_source) - self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", comm_source) - self.assertIn("options.allGather = &TileXRComm::DirectCcuAllGatherCallback", comm_source) - self.assertIn("options.allGatherUserData = this", comm_source) - self.assertIn("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke", comm_source) - self.assertIn("int TileXRComm::DirectCcuThreadAllGather", comm_source) - self.assertIn("DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this)", comm_source) - self.assertIn("comm->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf)", comm_source) - self.assertIn("InitThread", comm_source) - self.assertIn("ret = InitDirectCcuRuntime();", comm_source) - exchange_body = comm_source[ - comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): - comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + for leaked in [ + "ConfigureDirectCcuLowerLayerTemplateFromAllocation(", + "PrepareDirectCcuLowerLayerTemplateFromAllocation", + "ExchangeDirectCcuRemoteNotifyCke", + "DirectCcuAllGatherCallback", + "DirectCcuThreadAllGather", + "directCcuVerifiedEndpointRoutes_", + "directCcuLocalVerifiedEndpointRoute_", + ]: + with self.subTest(leaked=leaked): + self.assertNotIn(leaked, comm_header) + self.assertNotIn(leaked, backend_header) + + self.assertIn("const std::vector &remoteCcuBuffers", backend_source) + self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer", backend_source) + self.assertIn("ccuDirectRuntime_->ExportLocalCcuRmaBuffer", backend_source) + self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", backend_source) + self.assertIn("runtimeOptions.allGather = &TileXRCcuBackend::Impl::DirectCcuAllGatherCallback", backend_source) + self.assertIn("runtimeOptions.allGatherUserData = this", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::DirectCcuThreadAllGather", backend_source) + self.assertIn("DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this)", backend_source) + self.assertIn("backend->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf)", backend_source) + self.assertIn("TileXRComm::InitCcuBackend", comm_source) + self.assertIn("return ccuBackend_->Init(options);", comm_source) + exchange_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): + backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") ] self.assertNotIn("SelectDirectCcuRemoteBindingOverride", exchange_body) self.assertIn("peerLocalWaitCkeOffset", exchange_body) self.assertIn("peerResources.localWaitCkeStartId", exchange_body) self.assertIn("peerResources.localWaitCkeCount", exchange_body) self.assertNotIn("allocation.remoteNotifyCke.startId,\n routeIndex", exchange_body) - self.assertIn("allocation.localXn.startId", comm_source) + self.assertIn("allocation.localXn.startId", backend_source) self.assertIn("remoteXnStartId", exchange_body) self.assertIn("remoteXnCount", exchange_body) self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET", exchange_body) self.assertNotIn("TILEXR_CCU_XN_SLOT_BYTES", exchange_body) - self.assertIn("remoteXnId", comm_source) - self.assertIn("remoteNotifyCke", comm_source) - self.assertIn("templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId", comm_source) - self.assertIn("templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue", comm_source) - self.assertIn("templateSnapshot.msidToken.valid = true", comm_source) - self.assertIn("directCcuVerifiedEndpointRoutes_", comm_header) - self.assertIn("TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes", comm_source) - self.assertIn("directCcuLocalVerifiedEndpointRoute_", comm_header) - self.assertIn("TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute", comm_source) - self.assertIn("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute", comm_source) - self.assertIn("ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute", comm_source) + self.assertIn("remoteXnId", backend_source) + self.assertIn("remoteNotifyCke", backend_source) + self.assertIn("templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId", backend_source) + self.assertIn("templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue", backend_source) + self.assertIn("templateSnapshot.msidToken.valid = true", backend_source) + self.assertIn("directCcuVerifiedEndpointRoutes_", backend_source) + self.assertIn("TileXRCcuBackend::Impl::ConfigureDirectCcuVerifiedEndpointRoutes", backend_source) + self.assertIn("directCcuLocalVerifiedEndpointRoute_", backend_source) + self.assertIn("TileXRCcuBackend::Impl::ConfigureDirectCcuLocalVerifiedEndpointRoute", backend_source) + self.assertIn("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute", backend_source) + self.assertIn("ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute", backend_source) self.assertIn("TileXRCcuLocalEndpointRouteCollectorFn", runtime_header) self.assertIn("localEndpointRouteCollector", runtime_header) self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID", runtime_source) self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA", runtime_source) self.assertIn("direct CCU local endpoint route collected", runtime_source) self.assertIn("TileXRCcuOverlayVerifiedEndpointRoutes", builder_header) - self.assertGreaterEqual(comm_source.count("TileXRCcuOverlayVerifiedEndpointRoutes("), 3) - init_runtime_body = comm_source[ - comm_source.index("int TileXRComm::InitDirectCcuRuntime"): - comm_source.index("int TileXRComm::InitSDMA") + self.assertGreaterEqual(backend_source.count("TileXRCcuOverlayVerifiedEndpointRoutes("), 3) + init_runtime_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::Init("): + backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") ] self.assertIn("logicDevId", init_runtime_body) self.assertIn("devicePhyId", init_runtime_body) @@ -1988,9 +2007,9 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): runtime_source.index("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers"): runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot") ] - prepare_from_allocation_body = comm_source[ - comm_source.index("int TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation"): - comm_source.index("int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation") + prepare_from_allocation_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation"): + backend_source.index("int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation") ] export_snapshot_body = runtime_source[ runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot"): @@ -2012,10 +2031,10 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_memory_body) def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - exchange_body = comm_source[ - comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): - comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + exchange_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): + backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") ] compact_body = " ".join(exchange_body.split()) @@ -2042,10 +2061,10 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): compact_body) def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - exchange_body = comm_source[ - comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): - comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + exchange_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): + backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") ] compact_body = " ".join(exchange_body.split()) @@ -2061,10 +2080,10 @@ def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): self.assertIn("notifyCkeOwnerVerified &&", compact_body) def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - exchange_body = comm_source[ - comm_source.index("int TileXRComm::ExchangeDirectCcuRemoteNotifyCke"): - comm_source.index("int TileXRComm::DirectCcuAllGatherCallback") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + exchange_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): + backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") ] compact_body = " ".join(exchange_body.split()) @@ -2533,7 +2552,8 @@ def test_direct_ccu_runtime_collects_ra_ctx_endpoint_route_when_resource_window_ } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0xb0 || + buffers[0].remoteEid[0] != 0xbf || + buffers[0].remoteEid[15] != 0xb0 || buffers[0].tpn != 0x010203 || buffers[0].doorbellVa != 0x1122334455667788ULL || buffers[0].doorbellTokenId != 0x3456 || @@ -3209,7 +3229,8 @@ def test_direct_ccu_runtime_exports_configured_local_verified_endpoint_route(sel } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0x70 || + buffers[0].remoteEid[0] != 0x7f || + buffers[0].remoteEid[15] != 0x70 || buffers[0].tpn != localRoute.tpn || buffers[0].doorbellVa != localRoute.doorbellVa || buffers[0].doorbellTokenId != localRoute.doorbellTokenId || @@ -3329,7 +3350,8 @@ def test_direct_ccu_runtime_collects_local_verified_endpoint_route_before_exchan return 3; } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0x80 || + buffers[0].remoteEid[0] != 0x8f || + buffers[0].remoteEid[15] != 0x80 || buffers[0].tpn != 0x010203 || buffers[0].doorbellVa != 0x1122334455667788ULL || buffers[0].doorbellTokenId != 0x3456 || @@ -3521,8 +3543,8 @@ def test_direct_ccu_runtime_collects_ranked_env_local_verified_endpoint_route(se return 2; } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0xa0 || - buffers[0].remoteEid[15] != 0xaf || + buffers[0].remoteEid[0] != 0xaf || + buffers[0].remoteEid[15] != 0xa0 || buffers[0].tpn != 0x010203 || buffers[0].doorbellVa != 0x1122334455667788ULL || buffers[0].doorbellTokenId != 0x3456 || @@ -3552,22 +3574,29 @@ def test_direct_ccu_runtime_collects_ranked_env_local_verified_endpoint_route(se def test_tilexr_comm_caches_direct_ccu_basic_info_without_submit_readiness(self): comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - - self.assertIn("RefreshDirectCcuBasicInfo", comm_header) - self.assertIn("HasDirectCcuBasicInfo", comm_header) - self.assertIn("GetDirectCcuBasicInfoStatus", comm_header) - self.assertIn("GetDirectCcuBasicInfo", comm_header) - self.assertIn("GetDirectCcuBasicInfoReport", comm_header) - self.assertIn("directCcuBasicInfo_", comm_header) - self.assertIn("directCcuBasicInfoReport_", comm_header) - - self.assertIn("int TileXRComm::RefreshDirectCcuBasicInfo", comm_source) - self.assertIn("bool TileXRComm::HasDirectCcuBasicInfo", comm_source) - self.assertIn("ccuDirectRuntime_->QueryBasicInfo", comm_source) - self.assertIn("direct CCU basic info cached", comm_source) - self.assertIn("ResetDirectCcuBasicInfo", comm_source) - self.assertIn("ResetDirectCcuBasicInfo();", comm_source) - self.assertNotIn("udmaTransport_->" + "QueryCcuBasicInfo", comm_source) + backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + + for leaked in [ + "RefreshDirectCcuBasicInfo", + "HasDirectCcuBasicInfo", + "GetDirectCcuBasicInfoStatus", + "GetDirectCcuBasicInfo", + "GetDirectCcuBasicInfoReport", + "directCcuBasicInfo_", + "directCcuBasicInfoReport_", + ]: + with self.subTest(leaked=leaked): + self.assertNotIn(leaked, comm_header) + self.assertNotIn(leaked, backend_header) + + self.assertIn("int TileXRCcuBackend::Impl::RefreshDirectCcuBasicInfo", backend_source) + self.assertIn("bool TileXRCcuBackend::Impl::HasDirectCcuBasicInfo", backend_source) + self.assertIn("ccuDirectRuntime_->QueryBasicInfo", backend_source) + self.assertIn("direct CCU basic info cached", backend_source) + self.assertIn("ResetDirectCcuBasicInfo", backend_source) + self.assertIn("ResetDirectCcuBasicInfo();", backend_source) + self.assertNotIn("udmaTransport_->" + "QueryCcuBasicInfo", comm_source + "\n" + backend_source) for forbidden in [ "TileXRCcuPrepareSubmitTasks", @@ -3578,37 +3607,44 @@ def test_tilexr_comm_caches_direct_ccu_basic_info_without_submit_readiness(self) "libhcomm", "libhccl_v2", ]: - self.assertNotIn(forbidden, comm_header + "\n" + comm_source) + self.assertNotIn(forbidden, comm_header + "\n" + comm_source + "\n" + backend_source) def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self): comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") - self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', comm_header) - self.assertIn('ccu/tilexr_ccu_direct_runtime.h', comm_header) - self.assertIn("PrepareDirectCcuInstallAttempt", comm_header) - self.assertIn("FillDirectCcuLowerLayerPlanFromAllocation", comm_header) - self.assertIn("PrepareDirectCcuLowerLayerPlanCallback", comm_header) + self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', backend_source) + self.assertIn('ccu/tilexr_ccu_direct_runtime.h', backend_source) + for leaked in [ + "PrepareDirectCcuInstallAttempt", + "FillDirectCcuLowerLayerPlanFromAllocation", + "PrepareDirectCcuLowerLayerPlanCallback", + ]: + with self.subTest(leaked=leaked): + self.assertNotIn(leaked, comm_header) + self.assertNotIn(leaked, backend_header) self.assertIn("int CreateDriverAdapter(", runtime_header) - self.assertIn("int TileXRComm::PrepareDirectCcuInstallAttempt", comm_source) - self.assertIn("int TileXRComm::FillDirectCcuLowerLayerPlanFromAllocation", comm_source) - self.assertIn("int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback", comm_source) - self.assertIn("ccuDirectRuntime_->CreateDriverAdapter", comm_source) - self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)", comm_source) - self.assertIn("next.lowerLayerPlan = nullptr", comm_source) + self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation", backend_source) + self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback", backend_source) + self.assertIn("ccuDirectRuntime_->CreateDriverAdapter", backend_source) + self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)", backend_source) + self.assertIn("next.lowerLayerPlan = nullptr", backend_source) self.assertIn( - "next.prepareLowerLayerPlan = &TileXRComm::PrepareDirectCcuLowerLayerPlanCallback", - comm_source, + "next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback", + backend_source, ) - self.assertIn("next.lowerLayerPlanUserData = this", comm_source) - self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", comm_source) + self.assertIn("next.lowerLayerPlanUserData = this", backend_source) + self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", backend_source) self.assertIn("int TileXRCcuDirectRuntime::CreateDriverAdapter", runtime_source) - self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source) + self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source + "\n" + backend_source) - combined = comm_header + "\n" + comm_source + "\n" + runtime_header + "\n" + runtime_source + combined = comm_header + "\n" + comm_source + "\n" + backend_source + "\n" + runtime_header + "\n" + runtime_source for forbidden in [ "TileXRCcuPrepareSubmitTasks", "TileXRCcuSubmitTask", @@ -3621,20 +3657,20 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self self.assertNotIn(forbidden, combined) def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - self.assertIn("g_ccuDirectRuntimeUnavailableMessage", comm_source) - init_body = comm_source[ - comm_source.index("int TileXRComm::InitDirectCcuRuntime"): - comm_source.index("int TileXRComm::InitSDMA") + self.assertIn("g_ccuDirectRuntimeUnavailableMessage", backend_source) + init_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::Init("): + backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") ] - prepare_body = comm_source[ - comm_source.index("int TileXRComm::PrepareDirectCcuInstallAttempt"): - comm_source.index("int TileXRComm::RefreshDirectCcuLowerLayerPlan") + prepare_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"): + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt") ] self.assertIn("g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message", init_body) - self.assertIn("direct CCU runtime unavailable after process-level init failure", comm_source) + self.assertIn("direct CCU runtime unavailable after process-level init failure", backend_source) self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", prepare_body) self.assertLess( prepare_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), @@ -3642,15 +3678,15 @@ def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(se ) def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - init_body = comm_source[ - comm_source.index("int TileXRComm::InitDirectCcuRuntime"): - comm_source.index("int TileXRComm::InitSDMA") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + init_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::Init("): + backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") ] lock_pos = init_body.index("lock_guard lock(g_ccuDirectRuntimeMtx);") allocation_pos = init_body.index("ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime())") - runtime_init_pos = init_body.index("ccuDirectRuntime_->Init(options, &runtimeReport)") + runtime_init_pos = init_body.index("ccuDirectRuntime_->Init(runtimeOptions, &runtimeReport)") unavailable_set_pos = init_body.index("g_ccuDirectRuntimeUnavailable = true") self.assertLess(lock_pos, allocation_pos) @@ -3659,23 +3695,23 @@ def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): self.assertEqual(1, init_body.count("lock_guard lock(g_ccuDirectRuntimeMtx);")) def test_tilexr_comm_direct_ccu_prepare_can_select_install_die_for_diagnostics(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - prepare_body = comm_source[ - comm_source.index("int TileXRComm::PrepareDirectCcuInstallAttempt"): - comm_source.index("int TileXRComm::RefreshDirectCcuLowerLayerPlan") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + prepare_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"): + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt") ] - self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", comm_source) - self.assertIn("SelectDirectCcuInstallDieId", comm_source) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", backend_source) + self.assertIn("SelectDirectCcuInstallDieId", backend_source) self.assertIn("RefreshDirectCcuBasicInfo(installDieId)", prepare_body) self.assertIn("directCcuBasicInfo_.dieId != installDieId", prepare_body) self.assertNotIn("RefreshDirectCcuBasicInfo(0)", prepare_body) def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_process_init_failure(self): - comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") - thread_allgather_body = comm_source[ - comm_source.index("int TileXRComm::DirectCcuThreadAllGather"): - comm_source.index("int TileXRComm::PrepareDirectCcuLowerLayerPlanCallback") + backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + thread_allgather_body = backend_source[ + backend_source.index("int TileXRCcuBackend::Impl::DirectCcuThreadAllGather"): + backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback") ] self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", thread_allgather_body) @@ -3686,6 +3722,28 @@ def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_process_init_failu ) def test_tilexr_comm_direct_ccu_lower_layer_plan_api_is_header_visible(self): + comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") + for leaked in [ + "ConfigureDirectCcuLowerLayerTemplate", + "ConfigureDirectCcuLowerLayerTemplateFromAllocation", + "PrepareDirectCcuLowerLayerTemplateFromAllocation", + "RefreshDirectCcuLowerLayerPlan", + "HasDirectCcuLowerLayerPlan", + "GetDirectCcuLowerLayerPlanStatus", + "GetDirectCcuLowerLayerPlanReport", + "GetDirectCcuLowerLayerPlan", + "RefreshDirectCcuBasicInfo", + "HasDirectCcuBasicInfo", + "GetDirectCcuBasicInfoStatus", + "GetDirectCcuBasicInfo", + "GetDirectCcuBasicInfoReport", + "ConfigureDirectCcuVerifiedEndpointRoutes", + "ConfigureDirectCcuLocalVerifiedEndpointRoute", + "PrepareDirectCcuInstallAttempt", + ]: + with self.subTest(leaked=leaked): + self.assertNotIn(leaked, comm_header) + code = textwrap.dedent( r''' #include "tilexr_comm.h" @@ -3696,66 +3754,19 @@ def test_tilexr_comm_direct_ccu_lower_layer_plan_api_is_header_visible(self): int main() { - using ConfigureFn = int (TileXRComm::*)(const TileXRCcuLowerLayerTransportSnapshot&); - using ConfigureFromAllocationFn = int (TileXRComm::*)( - const TileXRCcuResourceAllocation&, - const std::vector&); - using PrepareFromAllocationFn = int (TileXRComm::*)(const TileXRCcuResourceAllocation&); - using RefreshFn = int (TileXRComm::*)(); - using HasFn = bool (TileXRComm::*)() const; - using StatusFn = int (TileXRComm::*)() const; - using ReportFn = const TileXRCcuLowerLayerPlanBuilderReport& (TileXRComm::*)() const; - using PlanFn = const TileXRCcuLowerLayerInstallPlan* (TileXRComm::*)() const; - using BasicRefreshFn = int (TileXRComm::*)(uint8_t); - using BasicHasFn = bool (TileXRComm::*)() const; - using BasicStatusFn = int (TileXRComm::*)() const; - using BasicInfoFn = const TileXRCcuBasicInfo* (TileXRComm::*)() const; - using BasicReportFn = const TileXRCcuDriverAdapterReport& (TileXRComm::*)() const; - using ConfigureVerifiedEndpointRoutesFn = int (TileXRComm::*)( - const std::vector&); - using ConfigureLocalVerifiedEndpointRouteFn = int (TileXRComm::*)( - const TileXRCcuLowerLayerTransportRoute&); - using InstallAttemptFn = int (TileXRComm::*)( - const TileXRCcuDirectInstallOptions&, - TileXRCcuDirectInstallAttempt*, - TileXRCcuDirectInstallReport*); - - ConfigureFn configure = &TileXRComm::ConfigureDirectCcuLowerLayerTemplate; - ConfigureFromAllocationFn configureFromAllocation = - &TileXRComm::ConfigureDirectCcuLowerLayerTemplateFromAllocation; - PrepareFromAllocationFn prepareFromAllocation = - &TileXRComm::PrepareDirectCcuLowerLayerTemplateFromAllocation; - RefreshFn refresh = &TileXRComm::RefreshDirectCcuLowerLayerPlan; - HasFn has = &TileXRComm::HasDirectCcuLowerLayerPlan; - StatusFn status = &TileXRComm::GetDirectCcuLowerLayerPlanStatus; - ReportFn report = &TileXRComm::GetDirectCcuLowerLayerPlanReport; - PlanFn plan = &TileXRComm::GetDirectCcuLowerLayerPlan; - BasicRefreshFn basicRefresh = &TileXRComm::RefreshDirectCcuBasicInfo; - BasicHasFn basicHas = &TileXRComm::HasDirectCcuBasicInfo; - BasicStatusFn basicStatus = &TileXRComm::GetDirectCcuBasicInfoStatus; - BasicInfoFn basicInfo = &TileXRComm::GetDirectCcuBasicInfo; - BasicReportFn basicReport = &TileXRComm::GetDirectCcuBasicInfoReport; - ConfigureVerifiedEndpointRoutesFn configureVerifiedEndpointRoutes = - &TileXRComm::ConfigureDirectCcuVerifiedEndpointRoutes; - ConfigureLocalVerifiedEndpointRouteFn configureLocalVerifiedEndpointRoute = - &TileXRComm::ConfigureDirectCcuLocalVerifiedEndpointRoute; - InstallAttemptFn installAttempt = &TileXRComm::PrepareDirectCcuInstallAttempt; - (void)configure; - (void)configureFromAllocation; - (void)prepareFromAllocation; - (void)refresh; - (void)has; - (void)status; - (void)report; - (void)plan; - (void)basicRefresh; - (void)basicHas; - (void)basicStatus; - (void)basicInfo; - (void)basicReport; - (void)configureVerifiedEndpointRoutes; - (void)configureLocalVerifiedEndpointRoute; - (void)installAttempt; + using InitFn = int (TileXRComm::*)(); + using GetterFn = TileXRCcuBackend* (TileXRComm::*)(); + using ConstGetterFn = const TileXRCcuBackend* (TileXRComm::*)() const; + using EnableForTestFn = int (TileXRComm::*)(); + + InitFn init = &TileXRComm::InitCcuBackend; + GetterFn getter = &TileXRComm::GetCcuBackendForCollectives; + ConstGetterFn constGetter = &TileXRComm::GetCcuBackendForCollectives; + EnableForTestFn enableForTest = &TileXRComm::EnableCcuBackendForTest; + (void)init; + (void)getter; + (void)constGetter; + (void)enableForTest; return 0; } ''' From 130ee975f6ce39506e846491a74ba849534d506a Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 13:44:49 +0800 Subject: [PATCH 11/41] refactor: split CCU backend internals --- src/comm/CMakeLists.txt | 6 + src/comm/ccu/tilexr_ccu_backend.cpp | 1173 +---------------- .../ccu/tilexr_ccu_collective_planner.cpp | 832 ++++++++++++ src/comm/ccu/tilexr_ccu_collective_planner.h | 101 ++ src/comm/ccu/tilexr_ccu_executor.cpp | 51 + src/comm/ccu/tilexr_ccu_executor.h | 30 + src/comm/ccu/tilexr_ccu_runtime_session.cpp | 339 +++++ src/comm/ccu/tilexr_ccu_runtime_session.h | 68 + tests/ccu/test_tilexr_ccu_backend_boundary.py | 49 +- ...est_tilexr_ccu_lower_layer_plan_builder.py | 241 ++-- 10 files changed, 1666 insertions(+), 1224 deletions(-) create mode 100644 src/comm/ccu/tilexr_ccu_collective_planner.cpp create mode 100644 src/comm/ccu/tilexr_ccu_collective_planner.h create mode 100644 src/comm/ccu/tilexr_ccu_executor.cpp create mode 100644 src/comm/ccu/tilexr_ccu_executor.h create mode 100644 src/comm/ccu/tilexr_ccu_runtime_session.cpp create mode 100644 src/comm/ccu/tilexr_ccu_runtime_session.h diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index 648d383c..4b161a4e 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -110,6 +110,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_barrier_program.cpp ccu/tilexr_ccu_backend.h ccu/tilexr_ccu_backend.cpp + ccu/tilexr_ccu_collective_planner.h + ccu/tilexr_ccu_collective_planner.cpp ccu/tilexr_ccu_direct_orchestrator.h ccu/tilexr_ccu_direct_orchestrator.cpp ccu/tilexr_ccu_direct_runtime.h @@ -143,6 +145,10 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_resource_allocator.cpp ccu/tilexr_ccu_runtime.h ccu/tilexr_ccu_runtime.cpp + ccu/tilexr_ccu_runtime_session.h + ccu/tilexr_ccu_runtime_session.cpp + ccu/tilexr_ccu_executor.h + ccu/tilexr_ccu_executor.cpp ccu/tilexr_ccu_specs.h ccu/tilexr_ccu_specs.cpp tools/socket/tilexr_sock_exchange.h diff --git a/src/comm/ccu/tilexr_ccu_backend.cpp b/src/comm/ccu/tilexr_ccu_backend.cpp index a1efe86c..3c3afbde 100644 --- a/src/comm/ccu/tilexr_ccu_backend.cpp +++ b/src/comm/ccu/tilexr_ccu_backend.cpp @@ -3,1152 +3,115 @@ */ #include "ccu/tilexr_ccu_backend.h" -#include "ccu/tilexr_ccu_direct_orchestrator.h" -#include "ccu/tilexr_ccu_direct_runtime.h" -#include "ccu/tilexr_ccu_memory_program.h" -#include "ccu/tilexr_ccu_repository.h" -#include "tilexr_log.h" -#include "tools/socket/tilexr_sock_exchange.h" +#include "ccu/tilexr_ccu_collective_planner.h" +#include "ccu/tilexr_ccu_executor.h" +#include "ccu/tilexr_ccu_runtime_session.h" -#include -#include -#include -#include -#include -#include -#include #include -#include -#include - -#include "runtime/dev.h" -#include "runtime/mem.h" -#include "runtime/rts/rts_device.h" - -using namespace std; -using namespace chrono; namespace TileXR { -constexpr int TILEXR_INIT_TIMEOUT = 600; -constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; - -struct TileXRThreadAllGatherState { - std::vector data[TILEXR_MAX_RANK_SIZE]; - uint64_t arrivals = 0; - uint64_t departures = 0; - size_t bytes = 0; -}; -static map g_directCcuAllGatherStates; -static std::mutex g_mtx; -static std::mutex g_ccuDirectRuntimeMtx; -static bool g_ccuDirectRuntimeUnavailable = false; -static std::string g_ccuDirectRuntimeUnavailableMessage; - -uint8_t SelectDirectCcuInstallDieId() -{ - const char *text = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); - if (text == nullptr || text[0] == '\0') { - return 0; - } - char *end = nullptr; - const unsigned long parsed = std::strtoul(text, &end, 0); - if (end == text || *end != '\0' || parsed > 1UL) { - return 0; - } - return static_cast(parsed); -} - -uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) -{ - if (peerRouteCount == 0) { - return 0; - } - return static_cast(peerLocalIndex) + - static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); -} - -uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) -{ - return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); -} - -uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) -{ - if (syncRouteCount == 0) { - return 0; - } - return syncRouteCount; -} - -uint16_t SelectDirectCcuChannelBoundRemoteXnId( - uint16_t remoteXnStartId, - size_t peerLocalIndex, - uint32_t syncIndex, - size_t peerRouteCount) -{ - return static_cast( - static_cast(remoteXnStartId) + - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); -} - -uint16_t SelectDirectCcuRemoteNotifyCkeId(uint16_t remoteNotifyCkeStartId, size_t routeIndex) -{ - return static_cast(static_cast(remoteNotifyCkeStartId) + routeIndex); -} - -std::string ProcessDirectCcuRuntimeUnavailableMessage() -{ - lock_guard lock(g_ccuDirectRuntimeMtx); - if (!g_ccuDirectRuntimeUnavailable) { - return {}; - } - return g_ccuDirectRuntimeUnavailableMessage.empty() ? - "direct CCU runtime unavailable after process-level init failure" : - "direct CCU runtime unavailable after process-level init failure: " + - g_ccuDirectRuntimeUnavailableMessage; -} - -struct DirectCcuMemoryCopyEndpoint { - uint64_t sourceAddr = 0; - uint64_t sourceToken = 0; - uint64_t destinationAddr = 0; - uint64_t destinationToken = 0; - uint64_t bytes = 0; - uint32_t rank = 0; - uint32_t valid = 0; -}; - -int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) -{ - if (addr == 0 || bytes == 0 || packedToken == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - *packedToken = 0; - rtMemUbTokenInfo info {}; - info.va = addr; - info.size = bytes; - const rtError_t ret = rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info); - if (ret != RT_ERROR_NONE) { - return TILEXR_ERROR_MKIRT; - } - constexpr uint32_t tokenIdRightShift = 8U; - const uint32_t tokenId = info.tokenId >> tokenIdRightShift; - *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); - return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; -} - -int BuildDirectCcuLocalMemoryCopyEndpoint( - uint32_t rank, - uint64_t sourceAddr, - uint64_t destinationAddr, - uint64_t bytes, - DirectCcuMemoryCopyEndpoint *endpoint) -{ - if (endpoint == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - *endpoint = DirectCcuMemoryCopyEndpoint {}; - endpoint->rank = rank; - endpoint->bytes = bytes; - endpoint->sourceAddr = sourceAddr; - endpoint->destinationAddr = destinationAddr; - int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); - if (ret != TILEXR_SUCCESS) { - return ret; - } - ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); - if (ret != TILEXR_SUCCESS) { - return ret; - } - endpoint->valid = 1; - return TILEXR_SUCCESS; -} - class TileXRCcuBackend::Impl { public: + Impl(); int Init(const TileXRCcuBackendOptions &options); void Shutdown(); bool Available() const; - - int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); - bool HasDirectCcuBasicInfo() const; - int GetDirectCcuBasicInfoStatus() const; - const TileXRCcuBasicInfo *GetDirectCcuBasicInfo() const; - const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; - int ConfigureDirectCcuLowerLayerTemplate(const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot); - int ConfigureDirectCcuVerifiedEndpointRoutes( - const std::vector &verifiedRoutes); - int ConfigureDirectCcuLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route); - int ConfigureDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation, - const std::vector &remoteCcuBuffers); - int PrepareDirectCcuLowerLayerTemplateFromAllocation(const TileXRCcuResourceAllocation &allocation); - int PrepareDirectCcuInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report); - int PrepareDirectCcuMemoryCopyInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - uint64_t localSourceAddr, - uint64_t localDestinationAddr, - uint64_t bytes, - uint32_t peerRank, - TileXRCcuMemoryCopyDirection direction, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report); - int RefreshDirectCcuLowerLayerPlan(); - bool HasDirectCcuLowerLayerPlan() const; - int GetDirectCcuLowerLayerPlanStatus() const; - const TileXRCcuLowerLayerPlanBuilderReport &GetDirectCcuLowerLayerPlanReport() const; - const TileXRCcuLowerLayerInstallPlan *GetDirectCcuLowerLayerPlan() const; - int ReadDirectCcuInstructionsForDebug( - uint8_t dieId, - uint16_t instructionStartId, - void *instructions, - uint32_t instructionCount, - uint32_t instructionBytes, - TileXRCcuDriverAdapterReport *report); + bool Supports(const TileXRCcuCollectiveRequest &request) const; + int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); + int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); +#ifdef TILEXR_CCU_TESTING + bool RuntimeInitializedForTest() const; +#endif private: - void ResetDirectCcuBasicInfo(); - void ResetDirectCcuLowerLayerPlan(); - int FillDirectCcuLowerLayerPlanFromAllocation( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report); - int ExchangeDirectCcuRemoteNotifyCke( - const TileXRCcuResourceAllocation &allocation, - std::vector *remoteCcuBuffers, - TileXRCcuLowerLayerPlanBuilderReport *report); - static int PrepareDirectCcuLowerLayerPlanCallback( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report, - void *userData); - static int DirectCcuAllGatherCallback(const void *sendBuf, size_t sendBytes, void *recvBuf, void *userData); - int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); + int EnsureInternals(); - TileXRCcuBackendOptions options_ = {}; - int rank_ = 0; - int rankSize_ = 0; - int devId_ = 0; - std::string uid_ = {}; - TileXRSockExchange *socketExchange_ = nullptr; - bool initialized_ = false; - std::unique_ptr ccuDirectRuntime_; - bool directCcuBasicInfoValid_ = false; - int directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - TileXRCcuBasicInfo directCcuBasicInfo_ = {}; - TileXRCcuDriverAdapterReport directCcuBasicInfoReport_ = {}; - bool directCcuLowerLayerTemplateConfigured_ = false; - bool directCcuLowerLayerPlanValid_ = false; - int directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerTemplate_ = {}; - TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerSnapshot_ = {}; - TileXRCcuLowerLayerInstallPlan directCcuLowerLayerPlan_ = {}; - TileXRCcuLowerLayerPlanBuilderReport directCcuLowerLayerPlanReport_ = {}; - std::vector directCcuVerifiedEndpointRoutes_ = {}; - TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; - bool directCcuLocalVerifiedEndpointRouteValid_ = false; - uint64_t directCcuThreadAllGatherRound_ = 0; + std::unique_ptr runtimeSession_; + std::unique_ptr planner_; + std::unique_ptr executor_; }; -void TileXRCcuBackend::Impl::Shutdown() -{ - initialized_ = false; - ResetDirectCcuBasicInfo(); - ResetDirectCcuLowerLayerPlan(); - directCcuLowerLayerTemplateConfigured_ = false; - directCcuLowerLayerTemplate_ = TileXRCcuLowerLayerTransportSnapshot {}; - directCcuVerifiedEndpointRoutes_.clear(); - directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; - directCcuLocalVerifiedEndpointRouteValid_ = false; - directCcuThreadAllGatherRound_ = 0; - if (ccuDirectRuntime_ != nullptr) { - ccuDirectRuntime_->Shutdown(); - ccuDirectRuntime_.reset(); - } - options_ = TileXRCcuBackendOptions {}; - socketExchange_ = nullptr; -} - -bool TileXRCcuBackend::Impl::Available() const -{ - return initialized_ && ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable(); -} - -int TileXRCcuBackend::Impl::Init(const TileXRCcuBackendOptions &options) +TileXRCcuBackend::Impl::Impl() + : runtimeSession_(new (std::nothrow) TileXRCcuRuntimeSession()), + planner_(new (std::nothrow) TileXRCcuCollectivePlanner()), + executor_(new (std::nothrow) TileXRCcuExecutor()) { - Shutdown(); - options_ = options; - rank_ = options.rank; - rankSize_ = options.rankSize; - devId_ = options.devId; - uid_ = options.uid; - socketExchange_ = options.exchange; - if (rankSize_ <= 1) { - TILEXR_LOG(INFO) << "direct CCU runtime skipped for single-rank communicator"; - return TILEXR_SUCCESS; - } - - lock_guard lock(g_ccuDirectRuntimeMtx); - if (g_ccuDirectRuntimeUnavailable) { - TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; - return TILEXR_SUCCESS; - } - - ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime()); - if (ccuDirectRuntime_ == nullptr) { - TILEXR_LOG(WARN) << "TileXRCcuDirectRuntime allocation failed, direct CCU disabled"; - return TILEXR_SUCCESS; - } - - TileXRCcuDirectRuntimeOptions runtimeOptions {}; - runtimeOptions.rank = rank_; - runtimeOptions.rankSize = rankSize_; - runtimeOptions.devId = devId_; - runtimeOptions.allGather = &TileXRCcuBackend::Impl::DirectCcuAllGatherCallback; - runtimeOptions.allGatherUserData = this; - TileXRCcuDirectRuntimeReport runtimeReport; - const int ret = ccuDirectRuntime_->Init(runtimeOptions, &runtimeReport); - if (ret != TILEXR_SUCCESS || !ccuDirectRuntime_->IsAvailable()) { - TILEXR_LOG(WARN) << "TileXR direct CCU runtime init failed: " << ret - << ", logicDevId " << runtimeReport.logicDevId - << ", devicePhyId " << runtimeReport.devicePhyId - << ", hdcType " << runtimeReport.hdcType - << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) - << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) - << ", " << runtimeReport.message << ", direct CCU disabled"; - g_ccuDirectRuntimeUnavailable = true; - g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; - ResetDirectCcuBasicInfo(); - ccuDirectRuntime_.reset(); - return TILEXR_SUCCESS; - } - - const int ccuInfoRet = RefreshDirectCcuBasicInfo(0); - if (ccuInfoRet != TILEXR_SUCCESS && ccuInfoRet != TILEXR_ERROR_NOT_FOUND) { - TILEXR_LOG(WARN) << "direct CCU basic info refresh failed after runtime init: " << ccuInfoRet - << ", " << directCcuBasicInfoReport_.message; - } - - TILEXR_LOG(INFO) << "InitDirectCcuRuntime success, rank " << rank_ << "/" << rankSize_ - << " logicDevId " << runtimeReport.logicDevId - << " devicePhyId " << runtimeReport.devicePhyId - << " hdcType " << runtimeReport.hdcType - << " raInitialized " << (runtimeReport.raInitialized ? 1 : 0) - << " ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0); - initialized_ = true; - return TILEXR_SUCCESS; -} - -void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo() -{ - directCcuBasicInfoValid_ = false; - directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - directCcuBasicInfo_ = TileXRCcuBasicInfo {}; - directCcuBasicInfoReport_ = TileXRCcuDriverAdapterReport {}; -} - -void TileXRCcuBackend::Impl::ResetDirectCcuLowerLayerPlan() -{ - directCcuLowerLayerPlanValid_ = false; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - directCcuLowerLayerSnapshot_ = TileXRCcuLowerLayerTransportSnapshot {}; - directCcuLowerLayerPlan_ = TileXRCcuLowerLayerInstallPlan {}; - directCcuLowerLayerPlanReport_ = TileXRCcuLowerLayerPlanBuilderReport {}; -} - -int TileXRCcuBackend::Impl::RefreshDirectCcuBasicInfo(uint8_t dieId) -{ - ResetDirectCcuBasicInfo(); - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuBasicInfoReport_.message = "direct CCU runtime is unavailable for basic info"; - directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuBasicInfoStatus_; - } - - TileXRCcuBasicInfo basicInfo; - TileXRCcuDriverAdapterReport report; - const int ret = ccuDirectRuntime_->QueryBasicInfo(dieId, &basicInfo, &report); - directCcuBasicInfoReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuBasicInfoStatus_ = ret; - return directCcuBasicInfoStatus_; - } - - directCcuBasicInfo_ = basicInfo; - directCcuBasicInfoReport_.message = "direct CCU basic info cached"; - directCcuBasicInfoValid_ = true; - directCcuBasicInfoStatus_ = TILEXR_SUCCESS; - return TILEXR_SUCCESS; -} - -bool TileXRCcuBackend::Impl::HasDirectCcuBasicInfo() const -{ - return directCcuBasicInfoValid_; -} - -int TileXRCcuBackend::Impl::GetDirectCcuBasicInfoStatus() const -{ - return directCcuBasicInfoStatus_; -} - -const TileXRCcuBasicInfo *TileXRCcuBackend::Impl::GetDirectCcuBasicInfo() const -{ - return directCcuBasicInfoValid_ ? &directCcuBasicInfo_ : nullptr; -} - -const TileXRCcuDriverAdapterReport &TileXRCcuBackend::Impl::GetDirectCcuBasicInfoReport() const -{ - return directCcuBasicInfoReport_; -} - -int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplate( - const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot) -{ - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); -} - -int TileXRCcuBackend::Impl::ConfigureDirectCcuVerifiedEndpointRoutes( - const std::vector &verifiedRoutes) -{ - TileXRCcuLowerLayerTransportSnapshot validationSnapshot; - validationSnapshot.routes = verifiedRoutes; - TileXRCcuLowerLayerPlanBuilderReport report; - int ret = TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &validationSnapshot, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return ret; - } - - directCcuVerifiedEndpointRoutes_ = verifiedRoutes; - if (directCcuLowerLayerTemplateConfigured_) { - return RefreshDirectCcuLowerLayerPlan(); - } - return TILEXR_SUCCESS; -} - -int TileXRCcuBackend::Impl::ConfigureDirectCcuLocalVerifiedEndpointRoute( - const TileXRCcuLowerLayerTransportRoute &route) -{ - TileXRCcuLowerLayerTransportSnapshot validationSnapshot; - validationSnapshot.routes.push_back(route); - TileXRCcuLowerLayerPlanBuilderReport report; - std::vector routes {route}; - int ret = TileXRCcuOverlayVerifiedEndpointRoutes(routes, &validationSnapshot, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; - directCcuLocalVerifiedEndpointRouteValid_ = false; - directCcuLowerLayerPlanStatus_ = ret; - return ret; - } - - directCcuLocalVerifiedEndpointRoute_ = route; - directCcuLocalVerifiedEndpointRouteValid_ = true; - if (ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable()) { - return ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(route); - } - return TILEXR_SUCCESS; -} - -int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation, - const std::vector &remoteCcuBuffers) -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuBasicInfoValid_) { - directCcuLowerLayerPlanReport_.message = - "direct CCU basic info is unavailable for lower-layer transport template"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot templateSnapshot; - TileXRCcuLowerLayerPlanBuilderReport report; - int ret = TileXRCcuBuildLowerLayerTransportTemplate( - directCcuBasicInfo_, - allocation, - remoteCcuBuffers, - &templateSnapshot, - &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &templateSnapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); -} - -int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation( - const TileXRCcuResourceAllocation &allocation) -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuBasicInfoValid_) { - directCcuLowerLayerPlanReport_.message = - "direct CCU basic info is unavailable for lower-layer transport template"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuLowerLayerPlanReport_.message = - "direct CCU runtime is unavailable for resource window registration"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - int ret = ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to register direct CCU resource window"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLocalResourceWindowInfo localCcuResourceWindow; - ret = ccuDirectRuntime_->ExportLocalCcuRmaBuffer(&localCcuResourceWindow); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU local resource window token"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - if (directCcuLocalVerifiedEndpointRouteValid_) { - ret = ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(directCcuLocalVerifiedEndpointRoute_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to configure direct CCU local verified endpoint route"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - } else { - TileXRCcuDirectRuntimeReport endpointRouteReport; - ret = ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute(&endpointRouteReport); - if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { - TILEXR_LOG(WARN) << "direct CCU local endpoint route collection failed closed: " - << ret << ", " << endpointRouteReport.message; - } - } - - std::vector remoteCcuBuffers; - ret = ccuDirectRuntime_->ExportRemoteCcuRmaBuffers(&remoteCcuBuffers); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU peer resource window tokens"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = ExchangeDirectCcuRemoteNotifyCke(allocation, &remoteCcuBuffers, &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot templateSnapshot; - TileXRCcuLowerLayerPlanBuilderReport report; - ret = TileXRCcuBuildLowerLayerTransportTemplate( - directCcuBasicInfo_, - allocation, - remoteCcuBuffers, - &templateSnapshot, - &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - templateSnapshot.msidToken.dieId = directCcuBasicInfo_.dieId; - templateSnapshot.msidToken.msId = directCcuBasicInfo_.msId; - templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId; - templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue; - templateSnapshot.msidToken.valid = true; - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &templateSnapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerTemplate_ = templateSnapshot; - directCcuLowerLayerTemplateConfigured_ = true; - return RefreshDirectCcuLowerLayerPlan(); } -int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report) +int TileXRCcuBackend::Impl::EnsureInternals() { - if (plan == nullptr || report == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; + if (runtimeSession_ == nullptr) { + runtimeSession_.reset(new (std::nothrow) TileXRCcuRuntimeSession()); } - int ret = PrepareDirectCcuLowerLayerTemplateFromAllocation(allocation); - if (ret != TILEXR_SUCCESS) { - *report = directCcuLowerLayerPlanReport_; - return ret; + if (planner_ == nullptr) { + planner_.reset(new (std::nothrow) TileXRCcuCollectivePlanner()); } - if (!directCcuLowerLayerPlanValid_) { - *report = directCcuLowerLayerPlanReport_; - return TILEXR_ERROR_NOT_FOUND; + if (executor_ == nullptr) { + executor_.reset(new (std::nothrow) TileXRCcuExecutor()); } - *plan = directCcuLowerLayerPlan_; - *report = directCcuLowerLayerPlanReport_; - return TILEXR_SUCCESS; + return runtimeSession_ == nullptr || planner_ == nullptr || executor_ == nullptr ? + TILEXR_ERROR_INTERNAL : + TILEXR_SUCCESS; } -int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke( - const TileXRCcuResourceAllocation &allocation, - std::vector *remoteCcuBuffers, - TileXRCcuLowerLayerPlanBuilderReport *report) +int TileXRCcuBackend::Impl::Init(const TileXRCcuBackendOptions &options) { - if (remoteCcuBuffers == nullptr) { - if (report != nullptr) { - report->message = "missing direct CCU remote notify CKE exchange inputs"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (rankSize_ <= 1 || rank_ < 0 || rank_ >= rankSize_) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - const size_t peerRouteCount = static_cast(rankSize_ - 1); - const size_t syncRouteCount = allocation.remoteXn.num; - if (allocation.localXn.num == 0 || - allocation.localWaitCke.num == 0 || - allocation.remoteNotifyCke.num == 0 || - allocation.remoteXn.num < static_cast(rankSize_ - 1) || - allocation.localWaitCke.num < allocation.remoteXn.num || - allocation.remoteNotifyCke.num < allocation.remoteXn.num || - allocation.channels.num == 0 || - remoteCcuBuffers->size() != peerRouteCount) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - struct PeerResourceExchange { - uint16_t localXnStartId; - uint16_t localXnCount; - uint16_t remoteXnStartId; - uint16_t remoteXnCount; - uint16_t localWaitCkeStartId; - uint16_t localWaitCkeCount; - uint16_t remoteNotifyCkeStartId; - uint16_t remoteNotifyCkeCount; - uint16_t channelStartId; - uint16_t channelCount; - }; - PeerResourceExchange local { - allocation.localXn.startId, - allocation.localXn.num, - allocation.remoteXn.startId, - DirectCcuRemoteXnProofSpan(allocation.remoteXn.num), - allocation.localWaitCke.startId, - allocation.localWaitCke.num, - allocation.remoteNotifyCke.startId, - allocation.remoteNotifyCke.num, - allocation.channels.startId, - allocation.channels.num, - }; - std::vector all(rankSize_); - const int ret = DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this); + const int ret = EnsureInternals(); if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to exchange direct CCU peer XN/CKE resources"; - } return ret; } - - std::vector peerRanks; - peerRanks.reserve(peerRouteCount); - for (int peer = 0; peer < rankSize_; ++peer) { - if (peer != rank_) { - peerRanks.push_back(peer); - } - } - if (peerRanks.size() != peerRouteCount) { - if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; + if (planner_ != nullptr) { + planner_->Reset(); } - std::vector peerCcuBuffers = *remoteCcuBuffers; - remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); - - size_t routeIndex = 0; - for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { - const size_t peerBufferIndex = syncIndex % peerRouteCount; - const int peer = peerRanks[peerBufferIndex]; - const PeerResourceExchange &peerResources = all[peer]; - const size_t peerLocalIndex = static_cast(rank_ < peer ? rank_ : rank_ - 1); - const uint32_t peerLocalXnOffset = - SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t selectedRemoteXnOffset = - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t peerLocalWaitCkeOffset = routeIndex; - if (peerResources.localXnCount == 0 || - peerResources.remoteXnCount == 0 || - peerResources.localWaitCkeCount == 0 || - peerResources.channelCount == 0 || - peerLocalXnOffset >= peerResources.localXnCount || - selectedRemoteXnOffset >= peerResources.remoteXnCount || - peerLocalIndex >= peerResources.channelCount || - peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { - if (report != nullptr) { - report->message = "peer direct CCU local XN/CKE resources are incomplete"; - } - return TILEXR_ERROR_NOT_FOUND; - } - uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( - peerResources.remoteXnStartId, - peerLocalIndex, - syncIndex, - peerRouteCount); - const uint16_t peerLocalXnId = - static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); - uint16_t remoteNotifyCke = - static_cast(static_cast(peerResources.localWaitCkeStartId) + - peerLocalWaitCkeOffset); - (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; - (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; - (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; - const bool peerLocalXnOwnerVerified = - static_cast(peerLocalXnId) >= peerResources.localXnStartId && - static_cast(peerLocalXnId) < - static_cast(peerResources.localXnStartId) + peerResources.localXnCount; - const bool notifyCkeOwnerVerified = - static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && - static_cast(remoteNotifyCke) < - static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; - const bool localChannelOwnerVerified = - allocation.channels.num != 0 && - peerLocalXnOwnerVerified && - static_cast(channelBoundRemoteXnId) >= peerResources.remoteXnStartId && - static_cast(channelBoundRemoteXnId) < - static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && - routeIndex < allocation.channels.num && - peerResources.channelStartId != 0 && - peerLocalIndex < peerResources.channelCount; - const bool transportResourceExchangeVerified = - notifyCkeOwnerVerified && - allocation.localWaitCke.num != 0 && - routeIndex < allocation.localWaitCke.num && - peerLocalWaitCkeOffset < peerResources.localWaitCkeCount; - (*remoteCcuBuffers)[routeIndex].channelResourceOwnerVerified = localChannelOwnerVerified; - (*remoteCcuBuffers)[routeIndex].transportResourceExchangeVerified = transportResourceExchangeVerified; - ++routeIndex; - } - return TILEXR_SUCCESS; + return runtimeSession_->Init(options); } -int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback( - const void *sendBuf, - size_t sendBytes, - void *recvBuf, - void *userData) +void TileXRCcuBackend::Impl::Shutdown() { - auto *backend = static_cast(userData); - if (backend == nullptr || sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0) { - return TILEXR_ERROR_PARA_CHECK_FAIL; + if (planner_ != nullptr) { + planner_->Reset(); } - if (backend->socketExchange_ == nullptr) { - return backend->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf); + if (runtimeSession_ != nullptr) { + runtimeSession_->Shutdown(); } - return backend->socketExchange_->AllGather( - static_cast(sendBuf), - sendBytes, - static_cast(recvBuf)); } -int TileXRCcuBackend::Impl::DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) +bool TileXRCcuBackend::Impl::Available() const { - if (sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0 || rank_ < 0 || - rank_ >= rankSize_ || rankSize_ <= 0 || uid_.empty()) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - const uint64_t round = directCcuThreadAllGatherRound_++; - const std::string key = uid_ + ":ccu:" + std::to_string(round); - auto start = high_resolution_clock::now(); - for (;;) { - { - lock_guard lock(g_mtx); - auto &state = g_directCcuAllGatherStates[key]; - if (state.bytes == 0) { - state.bytes = sendBytes; - } else if (state.bytes != sendBytes) { - g_directCcuAllGatherStates.erase(key); - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - if (state.data[rank_].empty()) { - state.data[rank_].resize(sendBytes); - std::memcpy(state.data[rank_].data(), sendBuf, sendBytes); - ++state.arrivals; - } - if (state.arrivals == static_cast(rankSize_)) { - auto *output = static_cast(recvBuf); - for (int i = 0; i < rankSize_; ++i) { - std::memcpy(output + static_cast(i) * sendBytes, state.data[i].data(), sendBytes); - } - ++state.departures; - if (state.departures == static_cast(rankSize_)) { - g_directCcuAllGatherStates.erase(key); - } - return TILEXR_SUCCESS; - } - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - lock_guard lock(g_mtx); - g_directCcuAllGatherStates.erase(key); - TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ - << " uid " << uid_ << " round " << round << ", " - << processUnavailableMessage; - return TILEXR_ERROR_NOT_FOUND; - } - this_thread::sleep_for(1ms); - auto elapsed = duration_cast(high_resolution_clock::now() - start); - if (elapsed.count() > TILEXR_INIT_TIMEOUT) { - lock_guard lock(g_mtx); - g_directCcuAllGatherStates.erase(key); - TILEXR_LOG(ERROR) << "direct CCU thread allgather timeout rank " << rank_ << "/" << rankSize_ - << " uid " << uid_ << " round " << round; - return TILEXR_ERROR_TIMEOUT; - } - } + return runtimeSession_ != nullptr && runtimeSession_->Available(); } -int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback( - const TileXRCcuResourceAllocation &allocation, - TileXRCcuLowerLayerInstallPlan *plan, - TileXRCcuLowerLayerPlanBuilderReport *report, - void *userData) +bool TileXRCcuBackend::Impl::Supports(const TileXRCcuCollectiveRequest &request) const { - auto *backend = static_cast(userData); - if (backend == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - return backend->FillDirectCcuLowerLayerPlanFromAllocation(allocation, plan, report); + return runtimeSession_ != nullptr && planner_ != nullptr && planner_->Supports(*runtimeSession_, request); } -int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report) +int TileXRCcuBackend::Impl::PrepareCollective( + const TileXRCcuCollectiveRequest &request, + TileXRCcuCollectivePlan *plan) { - if (!initialized_) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = "TileXRCcuBackend is not initialized for direct CCU install attempt"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = processUnavailableMessage; - } - return TILEXR_ERROR_NOT_FOUND; - } - const uint8_t installDieId = SelectDirectCcuInstallDieId(); - if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { - const int ret = RefreshDirectCcuBasicInfo(installDieId); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = directCcuBasicInfoReport_.message; - } - return ret; - } + if (runtimeSession_ == nullptr || planner_ == nullptr) { + return TILEXR_ERROR_INTERNAL; } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = "direct CCU runtime is unavailable for install attempt"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - TileXRCcuDriverAdapter adapter; - TileXRCcuDriverAdapterReport adapterReport; - int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport{}; - report->message = adapterReport.message; - } - return ret; - } - - TileXRCcuDirectInstallOptions next = options; - next.basicInfo = &directCcuBasicInfo_; - next.offlineOnly = false; - next.driverAdapter = &adapter; - next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); - next.repositoryMemoryUserData = nullptr; - next.lowerLayerPlan = nullptr; - next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback; - next.lowerLayerPlanUserData = this; - if (next.provider.empty()) { - next.provider = "tilexr-comm-direct-ccu"; - } - - return TileXRCcuRunDirectInstallAttempt(next, attempt, report); + return planner_->PrepareCollective(*runtimeSession_, request, plan); } -int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt( - const TileXRCcuDirectInstallOptions &options, - uint64_t localSourceAddr, - uint64_t localDestinationAddr, - uint64_t bytes, - uint32_t peerRank, - TileXRCcuMemoryCopyDirection direction, - TileXRCcuDirectInstallAttempt *attempt, - TileXRCcuDirectInstallReport *report) +int TileXRCcuBackend::Impl::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream) { - if (!initialized_) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "TileXRCcuBackend is not initialized for direct CCU memory copy install attempt"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - if (localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || - peerRank >= static_cast(rankSize_) || peerRank == static_cast(rank_)) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "invalid direct CCU memory copy endpoint"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = processUnavailableMessage; - } - return TILEXR_ERROR_NOT_FOUND; + if (runtimeSession_ == nullptr || executor_ == nullptr) { + return TILEXR_ERROR_INTERNAL; } - const uint8_t installDieId = SelectDirectCcuInstallDieId(); - if (!directCcuBasicInfoValid_ || directCcuBasicInfo_.dieId != installDieId) { - const int ret = RefreshDirectCcuBasicInfo(installDieId); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = directCcuBasicInfoReport_.message; - } - return ret; - } - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "direct CCU runtime is unavailable for memory copy install attempt"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - DirectCcuMemoryCopyEndpoint localEndpoint; - int ret = BuildDirectCcuLocalMemoryCopyEndpoint( - static_cast(rank_), - localSourceAddr, - localDestinationAddr, - bytes, - &localEndpoint); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to query direct CCU memory copy local buffer token"; - } - return ret; - } - - std::vector allEndpoints(static_cast(rankSize_)); - ret = DirectCcuAllGatherCallback( - &localEndpoint, - sizeof(localEndpoint), - allEndpoints.data(), - this); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to exchange direct CCU memory copy peer endpoints"; - } - return ret; - } - const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; - if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "invalid direct CCU memory copy peer endpoint"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - - TileXRCcuDirectMemoryCopySpec memoryCopy; - memoryCopy.direction = direction; - memoryCopy.lengthBytes = bytes; - if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { - memoryCopy.localAddr = localEndpoint.destinationAddr; - memoryCopy.localToken = localEndpoint.destinationToken; - memoryCopy.remoteAddr = peerEndpoint.sourceAddr; - memoryCopy.remoteToken = peerEndpoint.sourceToken; - } else { - memoryCopy.localAddr = localEndpoint.sourceAddr; - memoryCopy.localToken = localEndpoint.sourceToken; - memoryCopy.remoteAddr = peerEndpoint.destinationAddr; - memoryCopy.remoteToken = peerEndpoint.destinationToken; - } - - TileXRCcuDriverAdapter adapter; - TileXRCcuDriverAdapterReport adapterReport; - ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, &adapterReport); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = adapterReport.message; - } - return ret; - } - - TileXRCcuDirectInstallOptions next = options; - next.basicInfo = &directCcuBasicInfo_; - next.offlineOnly = false; - next.driverAdapter = &adapter; - next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); - next.repositoryMemoryUserData = nullptr; - next.lowerLayerPlan = nullptr; - next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback; - next.lowerLayerPlanUserData = this; - next.sqeArgCount = 0; - next.syncResourceCount = 1; - next.syncInstructionCount = std::max( - next.syncInstructionCount, - TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); - next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; - if (next.provider.empty()) { - next.provider = "tilexr-comm-direct-ccu-memory-copy"; - } - - return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); -} - -int TileXRCcuBackend::Impl::RefreshDirectCcuLowerLayerPlan() -{ - ResetDirectCcuLowerLayerPlan(); - if (!directCcuLowerLayerTemplateConfigured_) { - directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer template is not configured"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - directCcuLowerLayerPlanReport_.message = "direct CCU runtime is unavailable for lower-layer planning"; - directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerTransportSnapshot snapshot; - int ret = ccuDirectRuntime_->ExportLowerLayerTransportSnapshot(directCcuLowerLayerTemplate_, &snapshot); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanReport_.message = "failed to export direct CCU lower-layer transport snapshot"; - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - ret = TileXRCcuOverlayVerifiedEndpointRoutes( - directCcuVerifiedEndpointRoutes_, - &snapshot, - &directCcuLowerLayerPlanReport_); - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - TileXRCcuLowerLayerInstallPlan plan; - TileXRCcuLowerLayerPlanBuilderReport report; - ret = TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report); - directCcuLowerLayerPlanReport_ = report; - if (ret != TILEXR_SUCCESS) { - directCcuLowerLayerPlanStatus_ = ret; - return directCcuLowerLayerPlanStatus_; - } - - directCcuLowerLayerSnapshot_ = snapshot; - directCcuLowerLayerPlan_ = plan; - directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer install plan cached"; - directCcuLowerLayerPlanValid_ = true; - directCcuLowerLayerPlanStatus_ = TILEXR_SUCCESS; - return TILEXR_SUCCESS; -} - -bool TileXRCcuBackend::Impl::HasDirectCcuLowerLayerPlan() const -{ - return directCcuLowerLayerPlanValid_; + return executor_->Submit(*runtimeSession_, plan, stream); } -int TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlanStatus() const -{ - return directCcuLowerLayerPlanStatus_; -} - -const TileXRCcuLowerLayerPlanBuilderReport &TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlanReport() const -{ - return directCcuLowerLayerPlanReport_; -} - -const TileXRCcuLowerLayerInstallPlan *TileXRCcuBackend::Impl::GetDirectCcuLowerLayerPlan() const +#ifdef TILEXR_CCU_TESTING +bool TileXRCcuBackend::Impl::RuntimeInitializedForTest() const { - return directCcuLowerLayerPlanValid_ ? &directCcuLowerLayerPlan_ : nullptr; + return Available(); } +#endif -int TileXRCcuBackend::Impl::ReadDirectCcuInstructionsForDebug( - uint8_t dieId, - uint16_t instructionStartId, - void *instructions, - uint32_t instructionCount, - uint32_t instructionBytes, - TileXRCcuDriverAdapterReport *report) -{ - if (report != nullptr) { - *report = TileXRCcuDriverAdapterReport{}; - } - if (!initialized_) { - if (report != nullptr) { - report->message = "TileXRCcuBackend is not initialized for direct CCU instruction readback"; - } - return TILEXR_ERROR_NOT_INITIALIZED; - } - if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { - if (report != nullptr) { - report->message = "direct CCU runtime is unavailable for instruction readback"; - } - return TILEXR_ERROR_NOT_FOUND; - } - - TileXRCcuDriverAdapter adapter; - int ret = ccuDirectRuntime_->CreateDriverAdapter(&adapter, report); - if (ret != TILEXR_SUCCESS) { - return ret; - } - return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); -} TileXRCcuBackend::TileXRCcuBackend() : impl_(new (std::nothrow) Impl()) { } @@ -1183,35 +146,29 @@ bool TileXRCcuBackend::Available() const bool TileXRCcuBackend::Supports(const TileXRCcuCollectiveRequest &request) const { - (void)request; - return false; + return impl_ != nullptr && impl_->Supports(request); } int TileXRCcuBackend::PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan) { - if (plan == nullptr) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - *plan = TileXRCcuCollectivePlan {}; - if (!Available()) { - return TILEXR_ERROR_NOT_INITIALIZED; + if (impl_ == nullptr) { + return TILEXR_ERROR_INTERNAL; } - (void)request; - return TILEXR_ERROR_NOT_SUPPORT; + return impl_->PrepareCollective(request, plan); } -int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream) +int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream) { - if (!Available()) { - return TILEXR_ERROR_NOT_INITIALIZED; + if (impl_ == nullptr) { + return TILEXR_ERROR_INTERNAL; } - return plan.ready ? TILEXR_ERROR_NOT_SUPPORT : TILEXR_ERROR_PARA_CHECK_FAIL; + return impl_->SubmitCollective(plan, stream); } #ifdef TILEXR_CCU_TESTING bool TileXRCcuBackend::RuntimeInitializedForTest() const { - return Available(); + return impl_ != nullptr && impl_->RuntimeInitializedForTest(); } #endif diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp new file mode 100644 index 00000000..80bbe4f2 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -0,0 +1,832 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#include "ccu/tilexr_ccu_collective_planner.h" + +#include "ccu/tilexr_ccu_repository.h" +#include "ccu/tilexr_ccu_runtime_session.h" +#include "tilexr_log.h" + +#include +#include +#include +#include + +#include "runtime/dev.h" +#include "runtime/mem.h" +#include "runtime/rts/rts_device.h" + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; + +uint8_t SelectDirectCcuInstallDieId() +{ + const char *text = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); + if (text == nullptr || text[0] == '\0') { + return 0; + } + char *end = nullptr; + const unsigned long parsed = std::strtoul(text, &end, 0); + if (end == text || *end != '\0' || parsed > 1UL) { + return 0; + } + return static_cast(parsed); +} + +uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +{ + if (peerRouteCount == 0) { + return 0; + } + return static_cast(peerLocalIndex) + + static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); +} + +uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +{ + return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); +} + +uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) +{ + if (syncRouteCount == 0) { + return 0; + } + return syncRouteCount; +} + +uint16_t SelectDirectCcuChannelBoundRemoteXnId( + uint16_t remoteXnStartId, + size_t peerLocalIndex, + uint32_t syncIndex, + size_t peerRouteCount) +{ + return static_cast( + static_cast(remoteXnStartId) + + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); +} + +struct DirectCcuMemoryCopyEndpoint { + uint64_t sourceAddr = 0; + uint64_t sourceToken = 0; + uint64_t destinationAddr = 0; + uint64_t destinationToken = 0; + uint64_t bytes = 0; + uint32_t rank = 0; + uint32_t valid = 0; +}; + +int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) +{ + if (addr == 0 || bytes == 0 || packedToken == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *packedToken = 0; + rtMemUbTokenInfo info {}; + info.va = addr; + info.size = bytes; + const rtError_t ret = rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info); + if (ret != RT_ERROR_NONE) { + return TILEXR_ERROR_MKIRT; + } + constexpr uint32_t tokenIdRightShift = 8U; + const uint32_t tokenId = info.tokenId >> tokenIdRightShift; + *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); + return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; +} + +int BuildDirectCcuLocalMemoryCopyEndpoint( + uint32_t rank, + uint64_t sourceAddr, + uint64_t destinationAddr, + uint64_t bytes, + DirectCcuMemoryCopyEndpoint *endpoint) +{ + if (endpoint == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *endpoint = DirectCcuMemoryCopyEndpoint {}; + endpoint->rank = rank; + endpoint->bytes = bytes; + endpoint->sourceAddr = sourceAddr; + endpoint->destinationAddr = destinationAddr; + int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); + if (ret != TILEXR_SUCCESS) { + return ret; + } + endpoint->valid = 1; + return TILEXR_SUCCESS; +} + +void TileXRCcuCollectivePlanner::Reset() +{ + ResetDirectCcuLowerLayerPlan(); + directCcuLowerLayerTemplateConfigured_ = false; + directCcuLowerLayerTemplate_ = TileXRCcuLowerLayerTransportSnapshot {}; + directCcuVerifiedEndpointRoutes_.clear(); + directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; + directCcuLocalVerifiedEndpointRouteValid_ = false; +} + +bool TileXRCcuCollectivePlanner::Supports( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request) const +{ + (void)session; + (void)request; + return false; +} + +int TileXRCcuCollectivePlanner::PrepareCollective( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request, + TileXRCcuCollectivePlan *plan) const +{ + if (plan == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *plan = TileXRCcuCollectivePlan {}; + if (!session.Available()) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + (void)request; + return TILEXR_ERROR_NOT_SUPPORT; +} + +void TileXRCcuCollectivePlanner::ResetDirectCcuLowerLayerPlan() +{ + directCcuLowerLayerPlanValid_ = false; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuLowerLayerSnapshot_ = TileXRCcuLowerLayerTransportSnapshot {}; + directCcuLowerLayerPlan_ = TileXRCcuLowerLayerInstallPlan {}; + directCcuLowerLayerPlanReport_ = TileXRCcuLowerLayerPlanBuilderReport {}; +} + +int TileXRCcuCollectivePlanner::ConfigureDirectCcuLowerLayerTemplate( + TileXRCcuRuntimeSession &session, + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot) +{ + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(session); +} + +int TileXRCcuCollectivePlanner::ConfigureDirectCcuVerifiedEndpointRoutes( + TileXRCcuRuntimeSession &session, + const std::vector &verifiedRoutes) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes = verifiedRoutes; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuVerifiedEndpointRoutes_ = verifiedRoutes; + if (directCcuLowerLayerTemplateConfigured_) { + return RefreshDirectCcuLowerLayerPlan(session); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuCollectivePlanner::ConfigureDirectCcuLocalVerifiedEndpointRoute( + TileXRCcuRuntimeSession &session, + const TileXRCcuLowerLayerTransportRoute &route) +{ + TileXRCcuLowerLayerTransportSnapshot validationSnapshot; + validationSnapshot.routes.push_back(route); + TileXRCcuLowerLayerPlanBuilderReport report; + std::vector routes {route}; + int ret = TileXRCcuOverlayVerifiedEndpointRoutes(routes, &validationSnapshot, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLocalVerifiedEndpointRoute_ = TileXRCcuLowerLayerTransportRoute {}; + directCcuLocalVerifiedEndpointRouteValid_ = false; + directCcuLowerLayerPlanStatus_ = ret; + return ret; + } + + directCcuLocalVerifiedEndpointRoute_ = route; + directCcuLocalVerifiedEndpointRouteValid_ = true; + if (session.Available()) { + return session.ConfigureLocalVerifiedEndpointRoute(route); + } + return TILEXR_SUCCESS; +} + +int TileXRCcuCollectivePlanner::ConfigureDirectCcuLowerLayerTemplateFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers) +{ + ResetDirectCcuLowerLayerPlan(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + int ret = TileXRCcuBuildLowerLayerTransportTemplate( + *basicInfo, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(session); +} + +int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerTemplateFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation) +{ + ResetDirectCcuLowerLayerPlan(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr) { + directCcuLowerLayerPlanReport_.message = + "direct CCU basic info is unavailable for lower-layer transport template"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (!session.Available()) { + directCcuLowerLayerPlanReport_.message = + "direct CCU runtime is unavailable for resource window registration"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to register direct CCU resource window"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLocalResourceWindowInfo localCcuResourceWindow; + ret = session.ExportLocalCcuRmaBuffer(&localCcuResourceWindow); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU local resource window token"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + if (directCcuLocalVerifiedEndpointRouteValid_) { + ret = session.ConfigureLocalVerifiedEndpointRoute(directCcuLocalVerifiedEndpointRoute_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to configure direct CCU local verified endpoint route"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + } else { + TileXRCcuDirectRuntimeReport endpointRouteReport; + ret = session.RefreshLocalVerifiedEndpointRoute(&endpointRouteReport); + if (ret != TILEXR_SUCCESS && ret != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU local endpoint route collection failed closed: " + << ret << ", " << endpointRouteReport.message; + } + } + + std::vector remoteCcuBuffers; + ret = session.ExportRemoteCcuRmaBuffers(&remoteCcuBuffers); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU peer resource window tokens"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = ExchangeDirectCcuRemoteNotifyCke(session, allocation, &remoteCcuBuffers, &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot templateSnapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerTransportTemplate( + *basicInfo, + allocation, + remoteCcuBuffers, + &templateSnapshot, + &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + templateSnapshot.msidToken.dieId = basicInfo->dieId; + templateSnapshot.msidToken.msId = basicInfo->msId; + templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId; + templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue; + templateSnapshot.msidToken.valid = true; + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &templateSnapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerTemplate_ = templateSnapshot; + directCcuLowerLayerTemplateConfigured_ = true; + return RefreshDirectCcuLowerLayerPlan(session); +} + +int TileXRCcuCollectivePlanner::FillDirectCcuLowerLayerPlanFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (plan == nullptr || report == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + int ret = PrepareDirectCcuLowerLayerTemplateFromAllocation(session, allocation); + if (ret != TILEXR_SUCCESS) { + *report = directCcuLowerLayerPlanReport_; + return ret; + } + if (!directCcuLowerLayerPlanValid_) { + *report = directCcuLowerLayerPlanReport_; + return TILEXR_ERROR_NOT_FOUND; + } + *plan = directCcuLowerLayerPlan_; + *report = directCcuLowerLayerPlanReport_; + return TILEXR_SUCCESS; +} + +int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report) +{ + if (remoteCcuBuffers == nullptr) { + if (report != nullptr) { + report->message = "missing direct CCU remote notify CKE exchange inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize <= 1 || rank < 0 || rank >= rankSize) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const size_t peerRouteCount = static_cast(rankSize - 1); + const size_t syncRouteCount = allocation.remoteXn.num; + if (allocation.localXn.num == 0 || + allocation.localWaitCke.num == 0 || + allocation.remoteNotifyCke.num == 0 || + allocation.remoteXn.num < static_cast(rankSize - 1) || + allocation.localWaitCke.num < allocation.remoteXn.num || + allocation.remoteNotifyCke.num < allocation.remoteXn.num || + allocation.channels.num == 0 || + remoteCcuBuffers->size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + struct PeerResourceExchange { + uint16_t localXnStartId; + uint16_t localXnCount; + uint16_t remoteXnStartId; + uint16_t remoteXnCount; + uint16_t localWaitCkeStartId; + uint16_t localWaitCkeCount; + uint16_t remoteNotifyCkeStartId; + uint16_t remoteNotifyCkeCount; + uint16_t channelStartId; + uint16_t channelCount; + }; + PeerResourceExchange local { + allocation.localXn.startId, + allocation.localXn.num, + allocation.remoteXn.startId, + DirectCcuRemoteXnProofSpan(allocation.remoteXn.num), + allocation.localWaitCke.startId, + allocation.localWaitCke.num, + allocation.remoteNotifyCke.startId, + allocation.remoteNotifyCke.num, + allocation.channels.startId, + allocation.channels.num, + }; + std::vector all(rankSize); + const int ret = session.AllGather(&local, sizeof(local), all.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to exchange direct CCU peer XN/CKE resources"; + } + return ret; + } + + std::vector peerRanks; + peerRanks.reserve(peerRouteCount); + for (int peer = 0; peer < rankSize; ++peer) { + if (peer != rank) { + peerRanks.push_back(peer); + } + } + if (peerRanks.size() != peerRouteCount) { + if (report != nullptr) { + report->message = "invalid direct CCU peer XN/CKE exchange shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector peerCcuBuffers = *remoteCcuBuffers; + remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); + + size_t routeIndex = 0; + for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { + const size_t peerBufferIndex = syncIndex % peerRouteCount; + const int peer = peerRanks[peerBufferIndex]; + const PeerResourceExchange &peerResources = all[peer]; + const size_t peerLocalIndex = static_cast(rank < peer ? rank : rank - 1); + const uint32_t peerLocalXnOffset = + SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t selectedRemoteXnOffset = + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const uint32_t peerLocalWaitCkeOffset = routeIndex; + if (peerResources.localXnCount == 0 || + peerResources.remoteXnCount == 0 || + peerResources.localWaitCkeCount == 0 || + peerResources.channelCount == 0 || + peerLocalXnOffset >= peerResources.localXnCount || + selectedRemoteXnOffset >= peerResources.remoteXnCount || + peerLocalIndex >= peerResources.channelCount || + peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { + if (report != nullptr) { + report->message = "peer direct CCU local XN/CKE resources are incomplete"; + } + return TILEXR_ERROR_NOT_FOUND; + } + uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( + peerResources.remoteXnStartId, + peerLocalIndex, + syncIndex, + peerRouteCount); + const uint16_t peerLocalXnId = + static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); + uint16_t remoteNotifyCke = + static_cast(static_cast(peerResources.localWaitCkeStartId) + + peerLocalWaitCkeOffset); + (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; + (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; + (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; + const bool peerLocalXnOwnerVerified = + static_cast(peerLocalXnId) >= peerResources.localXnStartId && + static_cast(peerLocalXnId) < + static_cast(peerResources.localXnStartId) + peerResources.localXnCount; + const bool notifyCkeOwnerVerified = + static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && + static_cast(remoteNotifyCke) < + static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; + const bool localChannelOwnerVerified = + allocation.channels.num != 0 && + peerLocalXnOwnerVerified && + static_cast(channelBoundRemoteXnId) >= peerResources.remoteXnStartId && + static_cast(channelBoundRemoteXnId) < + static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && + routeIndex < allocation.channels.num && + peerResources.channelStartId != 0 && + peerLocalIndex < peerResources.channelCount; + const bool transportResourceExchangeVerified = + notifyCkeOwnerVerified && + allocation.localWaitCke.num != 0 && + routeIndex < allocation.localWaitCke.num && + peerLocalWaitCkeOffset < peerResources.localWaitCkeCount; + (*remoteCcuBuffers)[routeIndex].channelResourceOwnerVerified = localChannelOwnerVerified; + (*remoteCcuBuffers)[routeIndex].transportResourceExchangeVerified = transportResourceExchangeVerified; + ++routeIndex; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData) +{ + auto *context = static_cast(userData); + if (context == nullptr || context->planner == nullptr || context->session == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return context->planner->FillDirectCcuLowerLayerPlanFromAllocation( + *context->session, + allocation, + plan, + report); +} + +int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "TileXRCcuBackend is not initialized for direct CCU install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const std::string processUnavailableMessage = + TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = "direct CCU runtime is unavailable for install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + int ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport{}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu"; + } + + return TileXRCcuRunDirectInstallAttempt(next, attempt, report); +} + +int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU memory copy install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string processUnavailableMessage = + TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = processUnavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for memory copy install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + int ret = BuildDirectCcuLocalMemoryCopyEndpoint( + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU memory copy local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU memory copy peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU memory copy peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuDirectMemoryCopySpec memoryCopy; + memoryCopy.direction = direction; + memoryCopy.lengthBytes = bytes; + if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { + memoryCopy.localAddr = localEndpoint.destinationAddr; + memoryCopy.localToken = localEndpoint.destinationToken; + memoryCopy.remoteAddr = peerEndpoint.sourceAddr; + memoryCopy.remoteToken = peerEndpoint.sourceToken; + } else { + memoryCopy.localAddr = localEndpoint.sourceAddr; + memoryCopy.localToken = localEndpoint.sourceToken; + memoryCopy.remoteAddr = peerEndpoint.destinationAddr; + memoryCopy.remoteToken = peerEndpoint.destinationToken; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 1; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-memory-copy"; + } + + return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); +} + +int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session) +{ + ResetDirectCcuLowerLayerPlan(); + if (!directCcuLowerLayerTemplateConfigured_) { + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer template is not configured"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + if (!session.Available()) { + directCcuLowerLayerPlanReport_.message = "direct CCU runtime is unavailable for lower-layer planning"; + directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + int ret = session.ExportLowerLayerTransportSnapshot(directCcuLowerLayerTemplate_, &snapshot); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanReport_.message = "failed to export direct CCU lower-layer transport snapshot"; + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + ret = TileXRCcuOverlayVerifiedEndpointRoutes( + directCcuVerifiedEndpointRoutes_, + &snapshot, + &directCcuLowerLayerPlanReport_); + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + TileXRCcuLowerLayerInstallPlan plan; + TileXRCcuLowerLayerPlanBuilderReport report; + ret = TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report); + directCcuLowerLayerPlanReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuLowerLayerPlanStatus_ = ret; + return directCcuLowerLayerPlanStatus_; + } + + directCcuLowerLayerSnapshot_ = snapshot; + directCcuLowerLayerPlan_ = plan; + directCcuLowerLayerPlanReport_.message = "direct CCU lower-layer install plan cached"; + directCcuLowerLayerPlanValid_ = true; + directCcuLowerLayerPlanStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRCcuCollectivePlanner::HasDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_; +} + +int TileXRCcuCollectivePlanner::GetDirectCcuLowerLayerPlanStatus() const +{ + return directCcuLowerLayerPlanStatus_; +} + +const TileXRCcuLowerLayerPlanBuilderReport &TileXRCcuCollectivePlanner::GetDirectCcuLowerLayerPlanReport() const +{ + return directCcuLowerLayerPlanReport_; +} + +const TileXRCcuLowerLayerInstallPlan *TileXRCcuCollectivePlanner::GetDirectCcuLowerLayerPlan() const +{ + return directCcuLowerLayerPlanValid_ ? &directCcuLowerLayerPlan_ : nullptr; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h new file mode 100644 index 00000000..dfb55d92 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#ifndef TILEXR_CCU_COLLECTIVE_PLANNER_H +#define TILEXR_CCU_COLLECTIVE_PLANNER_H + +#include +#include + +#include "ccu/tilexr_ccu_backend.h" +#include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_memory_program.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession; + +class TileXRCcuCollectivePlanner { +public: + void Reset(); + bool Supports(const TileXRCcuRuntimeSession &session, const TileXRCcuCollectiveRequest &request) const; + int PrepareCollective( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectiveRequest &request, + TileXRCcuCollectivePlan *plan) const; + + int ConfigureDirectCcuLowerLayerTemplate( + TileXRCcuRuntimeSession &session, + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot); + int ConfigureDirectCcuVerifiedEndpointRoutes( + TileXRCcuRuntimeSession &session, + const std::vector &verifiedRoutes); + int ConfigureDirectCcuLocalVerifiedEndpointRoute( + TileXRCcuRuntimeSession &session, + const TileXRCcuLowerLayerTransportRoute &route); + int ConfigureDirectCcuLowerLayerTemplateFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + const std::vector &remoteCcuBuffers); + int PrepareDirectCcuLowerLayerTemplateFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation); + int PrepareDirectCcuInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuMemoryCopyInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuMemoryCopyDirection direction, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session); + bool HasDirectCcuLowerLayerPlan() const; + int GetDirectCcuLowerLayerPlanStatus() const; + const TileXRCcuLowerLayerPlanBuilderReport &GetDirectCcuLowerLayerPlanReport() const; + const TileXRCcuLowerLayerInstallPlan *GetDirectCcuLowerLayerPlan() const; + +private: + struct LowerLayerPlanCallbackContext { + TileXRCcuCollectivePlanner *planner = nullptr; + TileXRCcuRuntimeSession *session = nullptr; + }; + + void ResetDirectCcuLowerLayerPlan(); + int FillDirectCcuLowerLayerPlanFromAllocation( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report); + int ExchangeDirectCcuRemoteNotifyCke( + TileXRCcuRuntimeSession &session, + const TileXRCcuResourceAllocation &allocation, + std::vector *remoteCcuBuffers, + TileXRCcuLowerLayerPlanBuilderReport *report); + static int PrepareDirectCcuLowerLayerPlanCallback( + const TileXRCcuResourceAllocation &allocation, + TileXRCcuLowerLayerInstallPlan *plan, + TileXRCcuLowerLayerPlanBuilderReport *report, + void *userData); + + bool directCcuLowerLayerTemplateConfigured_ = false; + bool directCcuLowerLayerPlanValid_ = false; + int directCcuLowerLayerPlanStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerTemplate_ = {}; + TileXRCcuLowerLayerTransportSnapshot directCcuLowerLayerSnapshot_ = {}; + TileXRCcuLowerLayerInstallPlan directCcuLowerLayerPlan_ = {}; + TileXRCcuLowerLayerPlanBuilderReport directCcuLowerLayerPlanReport_ = {}; + std::vector directCcuVerifiedEndpointRoutes_ = {}; + TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; + bool directCcuLocalVerifiedEndpointRouteValid_ = false; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_COLLECTIVE_PLANNER_H diff --git a/src/comm/ccu/tilexr_ccu_executor.cpp b/src/comm/ccu/tilexr_ccu_executor.cpp new file mode 100644 index 00000000..46ac5686 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_executor.cpp @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#include "ccu/tilexr_ccu_executor.h" + +#include "ccu/tilexr_ccu_runtime_session.h" + +namespace TileXR { + +int TileXRCcuExecutor::Submit( + const TileXRCcuRuntimeSession &session, + const TileXRCcuCollectivePlan &plan, + aclrtStream) const +{ + if (!session.Available()) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + return plan.ready ? TILEXR_ERROR_NOT_SUPPORT : TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int TileXRCcuExecutor::ReadDirectCcuInstructionsForDebug( + TileXRCcuRuntimeSession &session, + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report) const +{ + if (report != nullptr) { + *report = TileXRCcuDriverAdapterReport{}; + } + if (!session.Available()) { + if (report != nullptr) { + report->message = "TileXRCcuBackend is not initialized for direct CCU instruction readback"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + + TileXRCcuDriverAdapter adapter; + int ret = session.CreateDriverAdapter(&adapter, report); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr && report->message.empty()) { + report->message = "direct CCU runtime is unavailable for instruction readback"; + } + return ret; + } + return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_executor.h b/src/comm/ccu/tilexr_ccu_executor.h new file mode 100644 index 00000000..19f8f257 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_executor.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#ifndef TILEXR_CCU_EXECUTOR_H +#define TILEXR_CCU_EXECUTOR_H + +#include "acl/acl_base.h" +#include "ccu/tilexr_ccu_backend.h" +#include "ccu/tilexr_ccu_driver_adapter.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession; + +class TileXRCcuExecutor { +public: + int Submit(const TileXRCcuRuntimeSession &session, const TileXRCcuCollectivePlan &plan, aclrtStream stream) const; + int ReadDirectCcuInstructionsForDebug( + TileXRCcuRuntimeSession &session, + uint8_t dieId, + uint16_t instructionStartId, + void *instructions, + uint32_t instructionCount, + uint32_t instructionBytes, + TileXRCcuDriverAdapterReport *report) const; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_EXECUTOR_H diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.cpp b/src/comm/ccu/tilexr_ccu_runtime_session.cpp new file mode 100644 index 00000000..d6c66746 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_runtime_session.cpp @@ -0,0 +1,339 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#include "ccu/tilexr_ccu_runtime_session.h" + +#include "tilexr_log.h" +#include "tools/socket/tilexr_sock_exchange.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std; +using namespace chrono; + +namespace TileXR { + +constexpr int TILEXR_INIT_TIMEOUT = 600; + +struct TileXRThreadAllGatherState { + std::vector data[TILEXR_MAX_RANK_SIZE]; + uint64_t arrivals = 0; + uint64_t departures = 0; + size_t bytes = 0; +}; +static map g_directCcuAllGatherStates; +static std::mutex g_mtx; +static std::mutex g_ccuDirectRuntimeMtx; +static bool g_ccuDirectRuntimeUnavailable = false; +static std::string g_ccuDirectRuntimeUnavailableMessage; + +std::string TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage() +{ + lock_guard lock(g_ccuDirectRuntimeMtx); + if (!g_ccuDirectRuntimeUnavailable) { + return {}; + } + return g_ccuDirectRuntimeUnavailableMessage.empty() ? + "direct CCU runtime unavailable after process-level init failure" : + "direct CCU runtime unavailable after process-level init failure: " + + g_ccuDirectRuntimeUnavailableMessage; +} + +void TileXRCcuRuntimeSession::Shutdown() +{ + initialized_ = false; + ResetDirectCcuBasicInfo(); + directCcuThreadAllGatherRound_ = 0; + if (ccuDirectRuntime_ != nullptr) { + ccuDirectRuntime_->Shutdown(); + ccuDirectRuntime_.reset(); + } + options_ = TileXRCcuBackendOptions {}; + socketExchange_ = nullptr; +} + +bool TileXRCcuRuntimeSession::Available() const +{ + return initialized_ && ccuDirectRuntime_ != nullptr && ccuDirectRuntime_->IsAvailable(); +} + +int TileXRCcuRuntimeSession::Rank() const +{ + return rank_; +} + +int TileXRCcuRuntimeSession::RankSize() const +{ + return rankSize_; +} + +int TileXRCcuRuntimeSession::Init(const TileXRCcuBackendOptions &options) +{ + Shutdown(); + options_ = options; + rank_ = options.rank; + rankSize_ = options.rankSize; + devId_ = options.devId; + uid_ = options.uid; + socketExchange_ = options.exchange; + if (rankSize_ <= 1) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped for single-rank communicator"; + return TILEXR_SUCCESS; + } + + lock_guard lock(g_ccuDirectRuntimeMtx); + if (g_ccuDirectRuntimeUnavailable) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; + return TILEXR_SUCCESS; + } + + ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime()); + if (ccuDirectRuntime_ == nullptr) { + TILEXR_LOG(WARN) << "TileXRCcuDirectRuntime allocation failed, direct CCU disabled"; + return TILEXR_SUCCESS; + } + + TileXRCcuDirectRuntimeOptions runtimeOptions {}; + runtimeOptions.rank = rank_; + runtimeOptions.rankSize = rankSize_; + runtimeOptions.devId = devId_; + runtimeOptions.allGather = &TileXRCcuRuntimeSession::DirectCcuAllGatherCallback; + runtimeOptions.allGatherUserData = this; + TileXRCcuDirectRuntimeReport runtimeReport; + const int ret = ccuDirectRuntime_->Init(runtimeOptions, &runtimeReport); + if (ret != TILEXR_SUCCESS || !ccuDirectRuntime_->IsAvailable()) { + TILEXR_LOG(WARN) << "TileXR direct CCU runtime init failed: " << ret + << ", logicDevId " << runtimeReport.logicDevId + << ", devicePhyId " << runtimeReport.devicePhyId + << ", hdcType " << runtimeReport.hdcType + << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) + << ", " << runtimeReport.message << ", direct CCU disabled"; + g_ccuDirectRuntimeUnavailable = true; + g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; + ResetDirectCcuBasicInfo(); + ccuDirectRuntime_.reset(); + return TILEXR_SUCCESS; + } + + const int ccuInfoRet = RefreshDirectCcuBasicInfo(0); + if (ccuInfoRet != TILEXR_SUCCESS && ccuInfoRet != TILEXR_ERROR_NOT_FOUND) { + TILEXR_LOG(WARN) << "direct CCU basic info refresh failed after runtime init: " << ccuInfoRet + << ", " << directCcuBasicInfoReport_.message; + } + + TILEXR_LOG(INFO) << "InitDirectCcuRuntime success, rank " << rank_ << "/" << rankSize_ + << " logicDevId " << runtimeReport.logicDevId + << " devicePhyId " << runtimeReport.devicePhyId + << " hdcType " << runtimeReport.hdcType + << " raInitialized " << (runtimeReport.raInitialized ? 1 : 0) + << " ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0); + initialized_ = true; + return TILEXR_SUCCESS; +} + +void TileXRCcuRuntimeSession::ResetDirectCcuBasicInfo() +{ + directCcuBasicInfoValid_ = false; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + directCcuBasicInfo_ = TileXRCcuBasicInfo {}; + directCcuBasicInfoReport_ = TileXRCcuDriverAdapterReport {}; +} + +int TileXRCcuRuntimeSession::RefreshDirectCcuBasicInfo(uint8_t dieId) +{ + ResetDirectCcuBasicInfo(); + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + directCcuBasicInfoReport_.message = "direct CCU runtime is unavailable for basic info"; + directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + return directCcuBasicInfoStatus_; + } + + TileXRCcuBasicInfo basicInfo; + TileXRCcuDriverAdapterReport report; + const int ret = ccuDirectRuntime_->QueryBasicInfo(dieId, &basicInfo, &report); + directCcuBasicInfoReport_ = report; + if (ret != TILEXR_SUCCESS) { + directCcuBasicInfoStatus_ = ret; + return directCcuBasicInfoStatus_; + } + + directCcuBasicInfo_ = basicInfo; + directCcuBasicInfoReport_.message = "direct CCU basic info cached"; + directCcuBasicInfoValid_ = true; + directCcuBasicInfoStatus_ = TILEXR_SUCCESS; + return TILEXR_SUCCESS; +} + +bool TileXRCcuRuntimeSession::HasDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_; +} + +int TileXRCcuRuntimeSession::GetDirectCcuBasicInfoStatus() const +{ + return directCcuBasicInfoStatus_; +} + +const TileXRCcuBasicInfo *TileXRCcuRuntimeSession::GetDirectCcuBasicInfo() const +{ + return directCcuBasicInfoValid_ ? &directCcuBasicInfo_ : nullptr; +} + +const TileXRCcuDriverAdapterReport &TileXRCcuRuntimeSession::GetDirectCcuBasicInfoReport() const +{ + return directCcuBasicInfoReport_; +} + +int TileXRCcuRuntimeSession::RegisterCcuResourceRmaBuffer(uint64_t resourceAddr) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(resourceAddr); +} + +int TileXRCcuRuntimeSession::ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo *info) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->ExportLocalCcuRmaBuffer(info); +} + +int TileXRCcuRuntimeSession::ExportRemoteCcuRmaBuffers(std::vector *buffers) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->ExportRemoteCcuRmaBuffers(buffers); +} + +int TileXRCcuRuntimeSession::ExportLowerLayerTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot, + TileXRCcuLowerLayerTransportSnapshot *snapshot) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->ExportLowerLayerTransportSnapshot(templateSnapshot, snapshot); +} + +int TileXRCcuRuntimeSession::ConfigureLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute(route); +} + +int TileXRCcuRuntimeSession::RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRuntimeReport *report) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute(report); +} + +int TileXRCcuRuntimeSession::CreateDriverAdapter( + TileXRCcuDriverAdapter *adapter, + TileXRCcuDriverAdapterReport *report) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + if (report != nullptr) { + report->message = "direct CCU runtime is unavailable"; + } + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->CreateDriverAdapter(adapter, report); +} + +int TileXRCcuRuntimeSession::AllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) +{ + return DirectCcuAllGatherCallback(sendBuf, sendBytes, recvBuf, this); +} + +int TileXRCcuRuntimeSession::DirectCcuAllGatherCallback( + const void *sendBuf, + size_t sendBytes, + void *recvBuf, + void *userData) +{ + auto *session = static_cast(userData); + if (session == nullptr || sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (session->socketExchange_ == nullptr) { + return session->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf); + } + return session->socketExchange_->AllGather( + static_cast(sendBuf), + sendBytes, + static_cast(recvBuf)); +} + +int TileXRCcuRuntimeSession::DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf) +{ + if (sendBuf == nullptr || recvBuf == nullptr || sendBytes == 0 || rank_ < 0 || + rank_ >= rankSize_ || rankSize_ <= 0 || uid_.empty()) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint64_t round = directCcuThreadAllGatherRound_++; + const std::string key = uid_ + ":ccu:" + std::to_string(round); + auto start = high_resolution_clock::now(); + for (;;) { + { + lock_guard lock(g_mtx); + auto &state = g_directCcuAllGatherStates[key]; + if (state.bytes == 0) { + state.bytes = sendBytes; + } else if (state.bytes != sendBytes) { + g_directCcuAllGatherStates.erase(key); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (state.data[rank_].empty()) { + state.data[rank_].resize(sendBytes); + std::memcpy(state.data[rank_].data(), sendBuf, sendBytes); + ++state.arrivals; + } + if (state.arrivals == static_cast(rankSize_)) { + auto *output = static_cast(recvBuf); + for (int i = 0; i < rankSize_; ++i) { + std::memcpy(output + static_cast(i) * sendBytes, state.data[i].data(), sendBytes); + } + ++state.departures; + if (state.departures == static_cast(rankSize_)) { + g_directCcuAllGatherStates.erase(key); + } + return TILEXR_SUCCESS; + } + } + const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); + if (!processUnavailableMessage.empty()) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round << ", " + << processUnavailableMessage; + return TILEXR_ERROR_NOT_FOUND; + } + this_thread::sleep_for(1ms); + auto elapsed = duration_cast(high_resolution_clock::now() - start); + if (elapsed.count() > TILEXR_INIT_TIMEOUT) { + lock_guard lock(g_mtx); + g_directCcuAllGatherStates.erase(key); + TILEXR_LOG(ERROR) << "direct CCU thread allgather timeout rank " << rank_ << "/" << rankSize_ + << " uid " << uid_ << " round " << round; + return TILEXR_ERROR_TIMEOUT; + } + } +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.h b/src/comm/ccu/tilexr_ccu_runtime_session.h new file mode 100644 index 00000000..d0010864 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_runtime_session.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2026 TileXR Project + */ +#ifndef TILEXR_CCU_RUNTIME_SESSION_H +#define TILEXR_CCU_RUNTIME_SESSION_H + +#include +#include +#include +#include +#include + +#include "ccu/tilexr_ccu_backend.h" +#include "ccu/tilexr_ccu_direct_runtime.h" + +namespace TileXR { + +class TileXRCcuRuntimeSession { +public: + int Init(const TileXRCcuBackendOptions &options); + void Shutdown(); + bool Available() const; + + int Rank() const; + int RankSize() const; + int AllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); + + int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); + bool HasDirectCcuBasicInfo() const; + int GetDirectCcuBasicInfoStatus() const; + const TileXRCcuBasicInfo *GetDirectCcuBasicInfo() const; + const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; + + int RegisterCcuResourceRmaBuffer(uint64_t resourceAddr); + int ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo *info); + int ExportRemoteCcuRmaBuffers(std::vector *buffers); + int ExportLowerLayerTransportSnapshot( + const TileXRCcuLowerLayerTransportSnapshot &templateSnapshot, + TileXRCcuLowerLayerTransportSnapshot *snapshot); + int ConfigureLocalVerifiedEndpointRoute(const TileXRCcuLowerLayerTransportRoute &route); + int RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRuntimeReport *report); + int CreateDriverAdapter(TileXRCcuDriverAdapter *adapter, TileXRCcuDriverAdapterReport *report); + + static std::string ProcessDirectCcuRuntimeUnavailableMessage(); + +private: + void ResetDirectCcuBasicInfo(); + static int DirectCcuAllGatherCallback(const void *sendBuf, size_t sendBytes, void *recvBuf, void *userData); + int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); + + TileXRCcuBackendOptions options_ = {}; + int rank_ = 0; + int rankSize_ = 0; + int devId_ = 0; + std::string uid_ = {}; + TileXRSockExchange *socketExchange_ = nullptr; + bool initialized_ = false; + std::unique_ptr ccuDirectRuntime_; + bool directCcuBasicInfoValid_ = false; + int directCcuBasicInfoStatus_ = TILEXR_ERROR_NOT_FOUND; + TileXRCcuBasicInfo directCcuBasicInfo_ = {}; + TileXRCcuDriverAdapterReport directCcuBasicInfoReport_ = {}; + uint64_t directCcuThreadAllGatherRound_ = 0; +}; + +} // namespace TileXR + +#endif // TILEXR_CCU_RUNTIME_SESSION_H diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 914f8252..ca6ec0e0 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -11,6 +11,12 @@ COMM_HEADER = REPO_ROOT / "src" / "comm" / "tilexr_comm.h" BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" +RUNTIME_SESSION_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.h" +RUNTIME_SESSION_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.cpp" +PLANNER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.h" +PLANNER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp" +EXECUTOR_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.h" +EXECUTOR_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.cpp" class TileXRCcuBackendBoundaryTest(unittest.TestCase): @@ -18,6 +24,18 @@ def test_backend_files_exist(self): self.assertTrue(BACKEND_HEADER.exists()) self.assertTrue(BACKEND_SOURCE.exists()) + def test_backend_internals_are_split(self): + for path in [ + RUNTIME_SESSION_HEADER, + RUNTIME_SESSION_SOURCE, + PLANNER_HEADER, + PLANNER_SOURCE, + EXECUTOR_HEADER, + EXECUTOR_SOURCE, + ]: + with self.subTest(path=path.name): + self.assertTrue(path.exists()) + def test_tilexr_comm_header_owns_only_opaque_backend(self): header = COMM_HEADER.read_text(encoding="utf-8") self.assertIn("class TileXRCcuBackend;", header) @@ -49,28 +67,47 @@ def test_backend_header_exposes_facade_not_public_c_api(self): with self.subTest(needle=needle): self.assertNotIn(needle, header) - def test_backend_source_owns_restored_direct_ccu_runtime_glue(self): + def test_split_sources_own_restored_direct_ccu_runtime_glue(self): source = BACKEND_SOURCE.read_text(encoding="utf-8") + runtime_header = RUNTIME_SESSION_HEADER.read_text(encoding="utf-8") + runtime = RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + executor = EXECUTOR_SOURCE.read_text(encoding="utf-8") + self.assertIn("#include \"ccu/tilexr_ccu_runtime_session.h\"", source) + self.assertIn("#include \"ccu/tilexr_ccu_collective_planner.h\"", source) + self.assertIn("#include \"ccu/tilexr_ccu_executor.h\"", source) + self.assertNotIn("#include \"ccu/tilexr_ccu_direct_runtime.h\"", source) + self.assertNotIn("#include \"ccu/tilexr_ccu_repository.h\"", source) + self.assertNotIn("TileXRCcuDirectRuntime", source) + for needle in [ "#include \"ccu/tilexr_ccu_direct_runtime.h\"", - "#include \"ccu/tilexr_ccu_repository.h\"", "TileXRCcuDirectRuntime", + "DirectCcuThreadAllGather", + "g_directCcuAllGatherStates", + ]: + with self.subTest(needle=needle): + self.assertIn(needle, runtime_header + "\n" + runtime) + + for needle in [ + "#include \"ccu/tilexr_ccu_repository.h\"", "PrepareDirectCcuInstallAttempt", "PrepareDirectCcuLowerLayerPlanCallback", "TileXRCcuRunDirectInstallAttempt", "TileXRCcuMakeRepositoryDeviceMemoryOps", - "DirectCcuThreadAllGather", - "g_directCcuAllGatherStates", + "PrepareDirectCcuMemoryCopyInstallAttempt", ]: with self.subTest(needle=needle): - self.assertIn(needle, source) + self.assertIn(needle, planner) + + self.assertIn("ReadDirectCcuInstructionsForDebug", executor) for fake_ready in [ "options_ = options;\n initialized_ = true;\n return TILEXR_SUCCESS;", "plan->ready = true;\n return TILEXR_SUCCESS;", "return plan.ready ? TILEXR_SUCCESS", ]: with self.subTest(fake_ready=fake_ready): - self.assertNotIn(fake_ready, source) + self.assertNotIn(fake_ready, source + "\n" + runtime + "\n" + planner + "\n" + executor) if __name__ == "__main__": diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 1d0c4cef..5516af42 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -27,6 +27,12 @@ COMM_SOURCE_FILE = REPO_ROOT / "src" / "comm" / "tilexr_comm.cpp" CCU_BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" CCU_BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" +CCU_RUNTIME_SESSION_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.h" +CCU_RUNTIME_SESSION_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.cpp" +CCU_PLANNER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.h" +CCU_PLANNER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp" +CCU_EXECUTOR_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.h" +CCU_EXECUTOR_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_executor.cpp" COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" INCLUDE_DIR = REPO_ROOT / "src" / "include" COMM_DIR = REPO_ROOT / "src" / "comm" @@ -1823,6 +1829,9 @@ def test_tilexr_comm_caches_direct_ccu_lower_layer_plan_from_ccu_runtime(self): comm_header = COMM_HEADER_FILE.read_text(encoding="utf-8") comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + planner_header = CCU_PLANNER_HEADER.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") for leaked in [ 'ccu/tilexr_ccu_lower_layer_plan_builder.h', @@ -1839,22 +1848,22 @@ def test_tilexr_comm_caches_direct_ccu_lower_layer_plan_from_ccu_runtime(self): with self.subTest(leaked=leaked): self.assertNotIn(leaked, comm_header) - self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', backend_source) - self.assertIn('ccu/tilexr_ccu_direct_runtime.h', backend_source) - self.assertIn('ccu/tilexr_ccu_repository.h', backend_source) - self.assertIn("int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplate", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::ConfigureDirectCcuLowerLayerTemplateFromAllocation", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::RefreshDirectCcuLowerLayerPlan", backend_source) - self.assertIn("TileXRCcuBuildLowerLayerTransportTemplate", backend_source) - self.assertIn("const std::vector &remoteCcuBuffers", backend_source) - self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", backend_source) - self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(directCcuBasicInfo_.resourceAddr)", backend_source) - self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", backend_source) - self.assertIn("ccuDirectRuntime_->ExportLowerLayerTransportSnapshot", backend_source) - self.assertIn("RefreshDirectCcuLowerLayerPlan();", backend_source) - self.assertIn("direct CCU lower-layer template is not configured", backend_source) - self.assertIn("direct CCU lower-layer install plan cached", backend_source) + self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', planner_header + "\n" + planner_source) + self.assertIn('ccu/tilexr_ccu_direct_runtime.h', CCU_RUNTIME_SESSION_HEADER.read_text(encoding="utf-8")) + self.assertIn('ccu/tilexr_ccu_repository.h', planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::ConfigureDirectCcuLowerLayerTemplate", planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::ConfigureDirectCcuLowerLayerTemplateFromAllocation", planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerTemplateFromAllocation", planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan", planner_source) + self.assertIn("TileXRCcuBuildLowerLayerTransportTemplate", planner_source) + self.assertIn("const std::vector &remoteCcuBuffers", planner_source) + self.assertIn("TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot", planner_source) + self.assertIn("session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr)", planner_source) + self.assertIn("session.ExportRemoteCcuRmaBuffers", planner_source) + self.assertIn("ccuDirectRuntime_->ExportLowerLayerTransportSnapshot", runtime_session_source) + self.assertIn("RefreshDirectCcuLowerLayerPlan(session);", planner_source) + self.assertIn("direct CCU lower-layer template is not configured", planner_source) + self.assertIn("direct CCU lower-layer install plan cached", planner_source) register_body = comm_source[ comm_source.index("int TileXRComm::RegisterUDMAMemory"): @@ -1903,6 +1912,10 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") builder_header = BUILDER_HEADER.read_text(encoding="utf-8") self.assertIn("TileXRCcuLocalResourceWindowInfo", runtime_header) @@ -1941,53 +1954,53 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn(leaked, comm_header) self.assertNotIn(leaked, backend_header) - self.assertIn("const std::vector &remoteCcuBuffers", backend_source) - self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer", backend_source) - self.assertIn("ccuDirectRuntime_->ExportLocalCcuRmaBuffer", backend_source) - self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", backend_source) - self.assertIn("runtimeOptions.allGather = &TileXRCcuBackend::Impl::DirectCcuAllGatherCallback", backend_source) - self.assertIn("runtimeOptions.allGatherUserData = this", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::DirectCcuThreadAllGather", backend_source) - self.assertIn("DirectCcuAllGatherCallback(&local, sizeof(local), all.data(), this)", backend_source) - self.assertIn("backend->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf)", backend_source) + self.assertIn("const std::vector &remoteCcuBuffers", planner_source) + self.assertIn("ccuDirectRuntime_->RegisterCcuResourceRmaBuffer", runtime_session_source) + self.assertIn("ccuDirectRuntime_->ExportLocalCcuRmaBuffer", runtime_session_source) + self.assertIn("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers", runtime_session_source) + self.assertIn("runtimeOptions.allGather = &TileXRCcuRuntimeSession::DirectCcuAllGatherCallback", runtime_session_source) + self.assertIn("runtimeOptions.allGatherUserData = this", runtime_session_source) + self.assertIn("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke", planner_source) + self.assertIn("int TileXRCcuRuntimeSession::DirectCcuThreadAllGather", runtime_session_source) + self.assertIn("session.AllGather(&local, sizeof(local), all.data())", planner_source) + self.assertIn("session->DirectCcuThreadAllGather(sendBuf, sendBytes, recvBuf)", runtime_session_source) self.assertIn("TileXRComm::InitCcuBackend", comm_source) self.assertIn("return ccuBackend_->Init(options);", comm_source) - exchange_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): - backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") + exchange_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback") ] self.assertNotIn("SelectDirectCcuRemoteBindingOverride", exchange_body) self.assertIn("peerLocalWaitCkeOffset", exchange_body) self.assertIn("peerResources.localWaitCkeStartId", exchange_body) self.assertIn("peerResources.localWaitCkeCount", exchange_body) self.assertNotIn("allocation.remoteNotifyCke.startId,\n routeIndex", exchange_body) - self.assertIn("allocation.localXn.startId", backend_source) + self.assertIn("allocation.localXn.startId", planner_source) self.assertIn("remoteXnStartId", exchange_body) self.assertIn("remoteXnCount", exchange_body) self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET", exchange_body) self.assertNotIn("TILEXR_CCU_XN_SLOT_BYTES", exchange_body) - self.assertIn("remoteXnId", backend_source) - self.assertIn("remoteNotifyCke", backend_source) - self.assertIn("templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId", backend_source) - self.assertIn("templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue", backend_source) - self.assertIn("templateSnapshot.msidToken.valid = true", backend_source) - self.assertIn("directCcuVerifiedEndpointRoutes_", backend_source) - self.assertIn("TileXRCcuBackend::Impl::ConfigureDirectCcuVerifiedEndpointRoutes", backend_source) - self.assertIn("directCcuLocalVerifiedEndpointRoute_", backend_source) - self.assertIn("TileXRCcuBackend::Impl::ConfigureDirectCcuLocalVerifiedEndpointRoute", backend_source) - self.assertIn("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute", backend_source) - self.assertIn("ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute", backend_source) + self.assertIn("remoteXnId", planner_source) + self.assertIn("remoteNotifyCke", planner_source) + self.assertIn("templateSnapshot.msidToken.tokenId = localCcuResourceWindow.tokenId", planner_source) + self.assertIn("templateSnapshot.msidToken.tokenValue = localCcuResourceWindow.tokenValue", planner_source) + self.assertIn("templateSnapshot.msidToken.valid = true", planner_source) + self.assertIn("directCcuVerifiedEndpointRoutes_", planner_source) + self.assertIn("TileXRCcuCollectivePlanner::ConfigureDirectCcuVerifiedEndpointRoutes", planner_source) + self.assertIn("directCcuLocalVerifiedEndpointRoute_", planner_source) + self.assertIn("TileXRCcuCollectivePlanner::ConfigureDirectCcuLocalVerifiedEndpointRoute", planner_source) + self.assertIn("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute", runtime_session_source) + self.assertIn("ccuDirectRuntime_->RefreshLocalVerifiedEndpointRoute", runtime_session_source) self.assertIn("TileXRCcuLocalEndpointRouteCollectorFn", runtime_header) self.assertIn("localEndpointRouteCollector", runtime_header) self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_EID", runtime_source) self.assertIn("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_DOORBELL_VA", runtime_source) self.assertIn("direct CCU local endpoint route collected", runtime_source) self.assertIn("TileXRCcuOverlayVerifiedEndpointRoutes", builder_header) - self.assertGreaterEqual(backend_source.count("TileXRCcuOverlayVerifiedEndpointRoutes("), 3) - init_runtime_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::Init("): - backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") + self.assertGreaterEqual(planner_source.count("TileXRCcuOverlayVerifiedEndpointRoutes("), 3) + init_runtime_body = runtime_session_source[ + runtime_session_source.index("int TileXRCcuRuntimeSession::Init("): + runtime_session_source.index("void TileXRCcuRuntimeSession::ResetDirectCcuBasicInfo") ] self.assertIn("logicDevId", init_runtime_body) self.assertIn("devicePhyId", init_runtime_body) @@ -2007,9 +2020,9 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): runtime_source.index("int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers"): runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot") ] - prepare_from_allocation_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerTemplateFromAllocation"): - backend_source.index("int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation") + prepare_from_allocation_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerTemplateFromAllocation"): + planner_source.index("int TileXRCcuCollectivePlanner::FillDirectCcuLowerLayerPlanFromAllocation") ] export_snapshot_body = runtime_source[ runtime_source.index("int TileXRCcuDirectRuntime::ExportLowerLayerTransportSnapshot"): @@ -2019,8 +2032,8 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn("buffers->clear();\n return TILEXR_ERROR_NOT_FOUND;", export_remote_body) self.assertNotIn("*snapshot = TileXRCcuLowerLayerTransportSnapshot{};\n return TILEXR_ERROR_NOT_FOUND;", export_snapshot_body) self.assertLess( - prepare_from_allocation_body.index("ccuDirectRuntime_->ConfigureLocalVerifiedEndpointRoute"), - prepare_from_allocation_body.index("ccuDirectRuntime_->ExportRemoteCcuRmaBuffers"), + prepare_from_allocation_body.index("session.ConfigureLocalVerifiedEndpointRoute"), + prepare_from_allocation_body.index("session.ExportRemoteCcuRmaBuffers"), ) register_memory_body = comm_source[ @@ -2031,10 +2044,10 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_memory_body) def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - exchange_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): - backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") + exchange_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback") ] compact_body = " ".join(exchange_body.split()) @@ -2061,10 +2074,10 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): compact_body) def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - exchange_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): - backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") + exchange_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback") ] compact_body = " ".join(exchange_body.split()) @@ -2080,17 +2093,17 @@ def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): self.assertIn("notifyCkeOwnerVerified &&", compact_body) def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - exchange_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::ExchangeDirectCcuRemoteNotifyCke"): - backend_source.index("int TileXRCcuBackend::Impl::DirectCcuAllGatherCallback") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") + exchange_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback") ] compact_body = " ".join(exchange_body.split()) - self.assertIn("const size_t peerRouteCount = static_cast(rankSize_ - 1)", compact_body) + self.assertIn("const size_t peerRouteCount = static_cast(rankSize - 1)", compact_body) self.assertIn("const size_t syncRouteCount = allocation.remoteXn.num", compact_body) - self.assertIn("allocation.remoteXn.num < static_cast(rankSize_ - 1)", compact_body) - self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize_ - 1)", compact_body) + self.assertIn("allocation.remoteXn.num < static_cast(rankSize - 1)", compact_body) + self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize - 1)", compact_body) self.assertIn("std::vector peerCcuBuffers = *remoteCcuBuffers", compact_body) self.assertIn("remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{})", compact_body) self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) @@ -3576,6 +3589,7 @@ def test_tilexr_comm_caches_direct_ccu_basic_info_without_submit_readiness(self) comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") for leaked in [ "RefreshDirectCcuBasicInfo", @@ -3590,13 +3604,13 @@ def test_tilexr_comm_caches_direct_ccu_basic_info_without_submit_readiness(self) self.assertNotIn(leaked, comm_header) self.assertNotIn(leaked, backend_header) - self.assertIn("int TileXRCcuBackend::Impl::RefreshDirectCcuBasicInfo", backend_source) - self.assertIn("bool TileXRCcuBackend::Impl::HasDirectCcuBasicInfo", backend_source) - self.assertIn("ccuDirectRuntime_->QueryBasicInfo", backend_source) - self.assertIn("direct CCU basic info cached", backend_source) - self.assertIn("ResetDirectCcuBasicInfo", backend_source) - self.assertIn("ResetDirectCcuBasicInfo();", backend_source) - self.assertNotIn("udmaTransport_->" + "QueryCcuBasicInfo", comm_source + "\n" + backend_source) + self.assertIn("int TileXRCcuRuntimeSession::RefreshDirectCcuBasicInfo", runtime_session_source) + self.assertIn("bool TileXRCcuRuntimeSession::HasDirectCcuBasicInfo", runtime_session_source) + self.assertIn("ccuDirectRuntime_->QueryBasicInfo", runtime_session_source) + self.assertIn("direct CCU basic info cached", runtime_session_source) + self.assertIn("ResetDirectCcuBasicInfo", runtime_session_source) + self.assertIn("ResetDirectCcuBasicInfo();", runtime_session_source) + self.assertNotIn("udmaTransport_->" + "QueryCcuBasicInfo", comm_source + "\n" + backend_source + "\n" + runtime_session_source) for forbidden in [ "TileXRCcuPrepareSubmitTasks", @@ -3614,11 +3628,14 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self comm_source = COMM_SOURCE_FILE.read_text(encoding="utf-8") backend_header = CCU_BACKEND_HEADER.read_text(encoding="utf-8") backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + planner_header = CCU_PLANNER_HEADER.read_text(encoding="utf-8") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") runtime_header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") runtime_source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") - self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', backend_source) - self.assertIn('ccu/tilexr_ccu_direct_runtime.h', backend_source) + self.assertIn('ccu/tilexr_ccu_direct_orchestrator.h', planner_header + "\n" + planner_source) + self.assertIn('ccu/tilexr_ccu_direct_runtime.h', CCU_RUNTIME_SESSION_HEADER.read_text(encoding="utf-8")) for leaked in [ "PrepareDirectCcuInstallAttempt", "FillDirectCcuLowerLayerPlanFromAllocation", @@ -3629,22 +3646,25 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self self.assertNotIn(leaked, backend_header) self.assertIn("int CreateDriverAdapter(", runtime_header) - self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::FillDirectCcuLowerLayerPlanFromAllocation", backend_source) - self.assertIn("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback", backend_source) - self.assertIn("ccuDirectRuntime_->CreateDriverAdapter", backend_source) - self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)", backend_source) - self.assertIn("next.lowerLayerPlan = nullptr", backend_source) + self.assertIn("int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt", planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::FillDirectCcuLowerLayerPlanFromAllocation", planner_source) + self.assertIn("int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback", planner_source) + self.assertIn("ccuDirectRuntime_->CreateDriverAdapter", runtime_session_source) + self.assertIn("TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)", planner_source) + self.assertIn("next.lowerLayerPlan = nullptr", planner_source) self.assertIn( - "next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback", - backend_source, + "next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback", + planner_source, ) - self.assertIn("next.lowerLayerPlanUserData = this", backend_source) - self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", backend_source) + self.assertIn("next.lowerLayerPlanUserData = &callbackContext", planner_source) + self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", planner_source) self.assertIn("int TileXRCcuDirectRuntime::CreateDriverAdapter", runtime_source) - self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source + "\n" + backend_source) + self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source + "\n" + backend_source + "\n" + runtime_session_source) - combined = comm_header + "\n" + comm_source + "\n" + backend_source + "\n" + runtime_header + "\n" + runtime_source + combined = ( + comm_header + "\n" + comm_source + "\n" + backend_source + "\n" + + planner_source + "\n" + runtime_session_source + "\n" + runtime_header + "\n" + runtime_source + ) for forbidden in [ "TileXRCcuPrepareSubmitTasks", "TileXRCcuSubmitTask", @@ -3657,20 +3677,21 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self self.assertNotIn(forbidden, combined) def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") - self.assertIn("g_ccuDirectRuntimeUnavailableMessage", backend_source) - init_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::Init("): - backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") + self.assertIn("g_ccuDirectRuntimeUnavailableMessage", runtime_session_source) + init_body = runtime_session_source[ + runtime_session_source.index("int TileXRCcuRuntimeSession::Init("): + runtime_session_source.index("void TileXRCcuRuntimeSession::ResetDirectCcuBasicInfo") ] - prepare_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"): - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt") + prepare_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt") ] self.assertIn("g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message", init_body) - self.assertIn("direct CCU runtime unavailable after process-level init failure", backend_source) + self.assertIn("direct CCU runtime unavailable after process-level init failure", runtime_session_source) self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", prepare_body) self.assertLess( prepare_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), @@ -3678,10 +3699,10 @@ def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(se ) def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - init_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::Init("): - backend_source.index("void TileXRCcuBackend::Impl::ResetDirectCcuBasicInfo") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + init_body = runtime_session_source[ + runtime_session_source.index("int TileXRCcuRuntimeSession::Init("): + runtime_session_source.index("void TileXRCcuRuntimeSession::ResetDirectCcuBasicInfo") ] lock_pos = init_body.index("lock_guard lock(g_ccuDirectRuntimeMtx);") @@ -3695,23 +3716,23 @@ def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): self.assertEqual(1, init_body.count("lock_guard lock(g_ccuDirectRuntimeMtx);")) def test_tilexr_comm_direct_ccu_prepare_can_select_install_die_for_diagnostics(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - prepare_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"): - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuMemoryCopyInstallAttempt") + planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") + prepare_body = planner_source[ + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt"): + planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt") ] - self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", backend_source) - self.assertIn("SelectDirectCcuInstallDieId", backend_source) + self.assertIn("TILEXR_CCU_DIRECT_INSTALL_DIE_ID", planner_source) + self.assertIn("SelectDirectCcuInstallDieId", planner_source) self.assertIn("RefreshDirectCcuBasicInfo(installDieId)", prepare_body) - self.assertIn("directCcuBasicInfo_.dieId != installDieId", prepare_body) + self.assertIn("basicInfo->dieId != installDieId", prepare_body) self.assertNotIn("RefreshDirectCcuBasicInfo(0)", prepare_body) def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_process_init_failure(self): - backend_source = CCU_BACKEND_SOURCE.read_text(encoding="utf-8") - thread_allgather_body = backend_source[ - backend_source.index("int TileXRCcuBackend::Impl::DirectCcuThreadAllGather"): - backend_source.index("int TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback") + runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") + thread_allgather_body = runtime_session_source[ + runtime_session_source.index("int TileXRCcuRuntimeSession::DirectCcuThreadAllGather"): + runtime_session_source.index("} // namespace TileXR") ] self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", thread_allgather_body) From b4de63967223f204cdb10f4676ed9a37c0aaf661 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 14:07:57 +0800 Subject: [PATCH 12/41] fix: update CCU split source guards --- tests/comm/unit/test_tilexr_source_guards.cpp | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/tests/comm/unit/test_tilexr_source_guards.cpp b/tests/comm/unit/test_tilexr_source_guards.cpp index 4e69d275..90b34666 100644 --- a/tests/comm/unit/test_tilexr_source_guards.cpp +++ b/tests/comm/unit/test_tilexr_source_guards.cpp @@ -405,12 +405,22 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() const std::string commSourcePath = "src/comm/tilexr_comm.cpp"; const std::string ccuBackendHeaderPath = "src/comm/ccu/tilexr_ccu_backend.h"; const std::string ccuBackendSourcePath = "src/comm/ccu/tilexr_ccu_backend.cpp"; + const std::string ccuRuntimeSessionHeaderPath = "src/comm/ccu/tilexr_ccu_runtime_session.h"; + const std::string ccuRuntimeSessionSourcePath = "src/comm/ccu/tilexr_ccu_runtime_session.cpp"; + const std::string ccuPlannerHeaderPath = "src/comm/ccu/tilexr_ccu_collective_planner.h"; + const std::string ccuPlannerSourcePath = "src/comm/ccu/tilexr_ccu_collective_planner.cpp"; + const std::string ccuExecutorSourcePath = "src/comm/ccu/tilexr_ccu_executor.cpp"; const std::string ccuRuntimeHeaderPath = "src/comm/ccu/tilexr_ccu_direct_runtime.h"; const std::string ccuRuntimeSourcePath = "src/comm/ccu/tilexr_ccu_direct_runtime.cpp"; const auto commHeaderText = ReadFile(commHeaderPath); const auto commSourceText = ReadFile(commSourcePath); const auto ccuBackendHeaderText = ReadFile(ccuBackendHeaderPath); const auto ccuBackendSourceText = ReadFile(ccuBackendSourcePath); + const auto ccuRuntimeSessionHeaderText = ReadFile(ccuRuntimeSessionHeaderPath); + const auto ccuRuntimeSessionSourceText = ReadFile(ccuRuntimeSessionSourcePath); + const auto ccuPlannerHeaderText = ReadFile(ccuPlannerHeaderPath); + const auto ccuPlannerSourceText = ReadFile(ccuPlannerSourcePath); + const auto ccuExecutorSourceText = ReadFile(ccuExecutorSourcePath); const auto ccuRuntimeHeaderText = ReadFile(ccuRuntimeHeaderPath); const auto ccuRuntimeSourceText = ReadFile(ccuRuntimeSourcePath); @@ -432,22 +442,32 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "class TileXRCcuBackend"); CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "TileXRSockExchange *exchange"); CheckContains(ccuBackendHeaderPath, ccuBackendHeaderText, "std::unique_ptr impl_"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_direct_runtime.h\""); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_runtime_session.h\""); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_collective_planner.h\""); + CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_executor.h\""); + CheckNotContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_direct_runtime.h\""); + CheckNotContains(ccuBackendSourcePath, ccuBackendSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckNotContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuDirectRuntime"); CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuBackend::Impl::Init"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuBackend::Impl::PrepareDirectCcuInstallAttempt"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); + CheckContains(ccuRuntimeSessionHeaderPath, ccuRuntimeSessionHeaderText, "TileXRCcuDirectRuntime"); + CheckContains(ccuRuntimeSessionSourcePath, ccuRuntimeSessionSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); + CheckContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, "class TileXRCcuRuntimeSession"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback"); CheckContains( - ccuBackendSourcePath, - ccuBackendSourceText, + ccuPlannerSourcePath, + ccuPlannerSourceText, "TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode)"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "next.lowerLayerPlan = nullptr"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "next.lowerLayerPlan = nullptr"); CheckContains( - ccuBackendSourcePath, - ccuBackendSourceText, - "next.prepareLowerLayerPlan = &TileXRCcuBackend::Impl::PrepareDirectCcuLowerLayerPlanCallback"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "next.lowerLayerPlanUserData = this"); - CheckContains(ccuBackendSourcePath, ccuBackendSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); + ccuPlannerSourcePath, + ccuPlannerSourceText, + "next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "LowerLayerPlanCallbackContext callbackContext"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "next.lowerLayerPlanUserData = &callbackContext"); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); + CheckContains(ccuExecutorSourcePath, ccuExecutorSourceText, "ReadDirectCcuInstructionsForDebug"); CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "int CreateDriverAdapter("); CheckContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, "int TileXRCcuDirectRuntime::CreateDriverAdapter"); CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "TileXRCcuHccpLoader"); @@ -504,6 +524,11 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() CheckNotContains(commSourcePath, commSourceText, forbidden); CheckNotContains(ccuBackendHeaderPath, ccuBackendHeaderText, forbidden); CheckNotContains(ccuBackendSourcePath, ccuBackendSourceText, forbidden); + CheckNotContains(ccuRuntimeSessionHeaderPath, ccuRuntimeSessionHeaderText, forbidden); + CheckNotContains(ccuRuntimeSessionSourcePath, ccuRuntimeSessionSourceText, forbidden); + CheckNotContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, forbidden); + CheckNotContains(ccuPlannerSourcePath, ccuPlannerSourceText, forbidden); + CheckNotContains(ccuExecutorSourcePath, ccuExecutorSourceText, forbidden); CheckNotContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, forbidden); CheckNotContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, forbidden); } From fbb446891be1153bba90667f572e24d536966db8 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 14:28:22 +0800 Subject: [PATCH 13/41] fix: guard CCU diagnostic internals --- .../ccu/tilexr_ccu_collective_planner.cpp | 12 ++++++++++- src/comm/ccu/tilexr_ccu_collective_planner.h | 4 ++++ src/comm/ccu/tilexr_ccu_executor.cpp | 2 ++ src/comm/ccu/tilexr_ccu_executor.h | 4 ++++ tests/ccu/test_tilexr_ccu_backend_boundary.py | 20 +++++++++++++++++-- ...est_tilexr_ccu_lower_layer_plan_builder.py | 8 ++++++++ tests/comm/unit/test_tilexr_source_guards.cpp | 13 ++++++++++++ 7 files changed, 60 insertions(+), 3 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 80bbe4f2..311eec0d 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -3,22 +3,28 @@ */ #include "ccu/tilexr_ccu_collective_planner.h" +#ifdef TILEXR_CCU_TESTING +#include "ccu/tilexr_ccu_memory_program.h" +#endif #include "ccu/tilexr_ccu_repository.h" #include "ccu/tilexr_ccu_runtime_session.h" #include "tilexr_log.h" -#include #include #include #include +#ifdef TILEXR_CCU_TESTING #include "runtime/dev.h" #include "runtime/mem.h" #include "runtime/rts/rts_device.h" +#endif namespace TileXR { +#ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; +#endif uint8_t SelectDirectCcuInstallDieId() { @@ -67,6 +73,7 @@ uint16_t SelectDirectCcuChannelBoundRemoteXnId( SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); } +#ifdef TILEXR_CCU_TESTING struct DirectCcuMemoryCopyEndpoint { uint64_t sourceAddr = 0; uint64_t sourceToken = 0; @@ -122,6 +129,7 @@ int BuildDirectCcuLocalMemoryCopyEndpoint( endpoint->valid = 1; return TILEXR_SUCCESS; } +#endif void TileXRCcuCollectivePlanner::Reset() { @@ -619,6 +627,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt( return TileXRCcuRunDirectInstallAttempt(next, attempt, report); } +#ifdef TILEXR_CCU_TESTING int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, @@ -761,6 +770,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); } +#endif int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session) { diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index dfb55d92..27888e5f 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -9,7 +9,9 @@ #include "ccu/tilexr_ccu_backend.h" #include "ccu/tilexr_ccu_direct_orchestrator.h" +#ifdef TILEXR_CCU_TESTING #include "ccu/tilexr_ccu_memory_program.h" +#endif namespace TileXR { @@ -45,6 +47,7 @@ class TileXRCcuCollectivePlanner { const TileXRCcuDirectInstallOptions &options, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); +#ifdef TILEXR_CCU_TESTING int PrepareDirectCcuMemoryCopyInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, @@ -55,6 +58,7 @@ class TileXRCcuCollectivePlanner { TileXRCcuMemoryCopyDirection direction, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); +#endif int RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session); bool HasDirectCcuLowerLayerPlan() const; int GetDirectCcuLowerLayerPlanStatus() const; diff --git a/src/comm/ccu/tilexr_ccu_executor.cpp b/src/comm/ccu/tilexr_ccu_executor.cpp index 46ac5686..fe463881 100644 --- a/src/comm/ccu/tilexr_ccu_executor.cpp +++ b/src/comm/ccu/tilexr_ccu_executor.cpp @@ -18,6 +18,7 @@ int TileXRCcuExecutor::Submit( return plan.ready ? TILEXR_ERROR_NOT_SUPPORT : TILEXR_ERROR_PARA_CHECK_FAIL; } +#ifdef TILEXR_CCU_TESTING int TileXRCcuExecutor::ReadDirectCcuInstructionsForDebug( TileXRCcuRuntimeSession &session, uint8_t dieId, @@ -47,5 +48,6 @@ int TileXRCcuExecutor::ReadDirectCcuInstructionsForDebug( } return adapter.ReadInstructions(dieId, instructionStartId, instructions, instructionCount, instructionBytes, report); } +#endif } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_executor.h b/src/comm/ccu/tilexr_ccu_executor.h index 19f8f257..100f594d 100644 --- a/src/comm/ccu/tilexr_ccu_executor.h +++ b/src/comm/ccu/tilexr_ccu_executor.h @@ -6,7 +6,9 @@ #include "acl/acl_base.h" #include "ccu/tilexr_ccu_backend.h" +#ifdef TILEXR_CCU_TESTING #include "ccu/tilexr_ccu_driver_adapter.h" +#endif namespace TileXR { @@ -15,6 +17,7 @@ class TileXRCcuRuntimeSession; class TileXRCcuExecutor { public: int Submit(const TileXRCcuRuntimeSession &session, const TileXRCcuCollectivePlan &plan, aclrtStream stream) const; +#ifdef TILEXR_CCU_TESTING int ReadDirectCcuInstructionsForDebug( TileXRCcuRuntimeSession &session, uint8_t dieId, @@ -23,6 +26,7 @@ class TileXRCcuExecutor { uint32_t instructionCount, uint32_t instructionBytes, TileXRCcuDriverAdapterReport *report) const; +#endif }; } // namespace TileXR diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index ca6ec0e0..1985c58b 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -95,12 +95,28 @@ def test_split_sources_own_restored_direct_ccu_runtime_glue(self): "PrepareDirectCcuLowerLayerPlanCallback", "TileXRCcuRunDirectInstallAttempt", "TileXRCcuMakeRepositoryDeviceMemoryOps", - "PrepareDirectCcuMemoryCopyInstallAttempt", ]: with self.subTest(needle=needle): self.assertIn(needle, planner) - self.assertIn("ReadDirectCcuInstructionsForDebug", executor) + planner_header = PLANNER_HEADER.read_text(encoding="utf-8") + executor_header = EXECUTOR_HEADER.read_text(encoding="utf-8") + self.assertRegex( + planner_header, + r"(?s)#ifdef TILEXR_CCU_TESTING.*PrepareDirectCcuMemoryCopyInstallAttempt.*#endif", + ) + self.assertRegex( + planner, + r"(?s)#ifdef TILEXR_CCU_TESTING.*PrepareDirectCcuMemoryCopyInstallAttempt.*#endif", + ) + self.assertRegex( + executor_header, + r"(?s)#ifdef TILEXR_CCU_TESTING.*ReadDirectCcuInstructionsForDebug.*#endif", + ) + self.assertRegex( + executor, + r"(?s)#ifdef TILEXR_CCU_TESTING.*ReadDirectCcuInstructionsForDebug.*#endif", + ) for fake_ready in [ "options_ = options;\n initialized_ = true;\n return TILEXR_SUCCESS;", "plan->ready = true;\n return TILEXR_SUCCESS;", diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 5516af42..41049bf5 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -3658,6 +3658,14 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self ) self.assertIn("next.lowerLayerPlanUserData = &callbackContext", planner_source) self.assertIn("TileXRCcuRunDirectInstallAttempt(next, attempt, report)", planner_source) + self.assertRegex( + planner_header, + r"(?s)#ifdef TILEXR_CCU_TESTING.*PrepareDirectCcuMemoryCopyInstallAttempt.*#endif", + ) + self.assertRegex( + planner_source, + r"(?s)#ifdef TILEXR_CCU_TESTING.*PrepareDirectCcuMemoryCopyInstallAttempt.*#endif", + ) self.assertIn("int TileXRCcuDirectRuntime::CreateDriverAdapter", runtime_source) self.assertNotIn("udmaTransport_->" + "CreateCcuDriverAdapter", comm_source + "\n" + backend_source + "\n" + runtime_session_source) diff --git a/tests/comm/unit/test_tilexr_source_guards.cpp b/tests/comm/unit/test_tilexr_source_guards.cpp index 90b34666..84f9896b 100644 --- a/tests/comm/unit/test_tilexr_source_guards.cpp +++ b/tests/comm/unit/test_tilexr_source_guards.cpp @@ -409,6 +409,7 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() const std::string ccuRuntimeSessionSourcePath = "src/comm/ccu/tilexr_ccu_runtime_session.cpp"; const std::string ccuPlannerHeaderPath = "src/comm/ccu/tilexr_ccu_collective_planner.h"; const std::string ccuPlannerSourcePath = "src/comm/ccu/tilexr_ccu_collective_planner.cpp"; + const std::string ccuExecutorHeaderPath = "src/comm/ccu/tilexr_ccu_executor.h"; const std::string ccuExecutorSourcePath = "src/comm/ccu/tilexr_ccu_executor.cpp"; const std::string ccuRuntimeHeaderPath = "src/comm/ccu/tilexr_ccu_direct_runtime.h"; const std::string ccuRuntimeSourcePath = "src/comm/ccu/tilexr_ccu_direct_runtime.cpp"; @@ -420,6 +421,7 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() const auto ccuRuntimeSessionSourceText = ReadFile(ccuRuntimeSessionSourcePath); const auto ccuPlannerHeaderText = ReadFile(ccuPlannerHeaderPath); const auto ccuPlannerSourceText = ReadFile(ccuPlannerSourcePath); + const auto ccuExecutorHeaderText = ReadFile(ccuExecutorHeaderPath); const auto ccuExecutorSourceText = ReadFile(ccuExecutorSourcePath); const auto ccuRuntimeHeaderText = ReadFile(ccuRuntimeHeaderPath); const auto ccuRuntimeSourceText = ReadFile(ccuRuntimeSourcePath); @@ -452,8 +454,15 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() CheckContains(ccuRuntimeSessionHeaderPath, ccuRuntimeSessionHeaderText, "TileXRCcuDirectRuntime"); CheckContains(ccuRuntimeSessionSourcePath, ccuRuntimeSessionSourceText, "ccuDirectRuntime_->CreateDriverAdapter"); CheckContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, "class TileXRCcuRuntimeSession"); + CheckContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, "#ifdef TILEXR_CCU_TESTING"); + CheckContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, "PrepareDirectCcuMemoryCopyInstallAttempt"); CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "#include \"ccu/tilexr_ccu_repository.h\""); + CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "#ifdef TILEXR_CCU_TESTING"); CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt"); + CheckContains( + ccuPlannerSourcePath, + ccuPlannerSourceText, + "TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt"); CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback"); CheckContains( ccuPlannerSourcePath, @@ -467,6 +476,9 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "LowerLayerPlanCallbackContext callbackContext"); CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "next.lowerLayerPlanUserData = &callbackContext"); CheckContains(ccuPlannerSourcePath, ccuPlannerSourceText, "TileXRCcuRunDirectInstallAttempt(next, attempt, report)"); + CheckContains(ccuExecutorHeaderPath, ccuExecutorHeaderText, "#ifdef TILEXR_CCU_TESTING"); + CheckContains(ccuExecutorHeaderPath, ccuExecutorHeaderText, "ReadDirectCcuInstructionsForDebug"); + CheckContains(ccuExecutorSourcePath, ccuExecutorSourceText, "#ifdef TILEXR_CCU_TESTING"); CheckContains(ccuExecutorSourcePath, ccuExecutorSourceText, "ReadDirectCcuInstructionsForDebug"); CheckContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, "int CreateDriverAdapter("); CheckContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, "int TileXRCcuDirectRuntime::CreateDriverAdapter"); @@ -528,6 +540,7 @@ void TestCommDirectCcuInstallAttemptDoesNotSubmit() CheckNotContains(ccuRuntimeSessionSourcePath, ccuRuntimeSessionSourceText, forbidden); CheckNotContains(ccuPlannerHeaderPath, ccuPlannerHeaderText, forbidden); CheckNotContains(ccuPlannerSourcePath, ccuPlannerSourceText, forbidden); + CheckNotContains(ccuExecutorHeaderPath, ccuExecutorHeaderText, forbidden); CheckNotContains(ccuExecutorSourcePath, ccuExecutorSourceText, forbidden); CheckNotContains(ccuRuntimeHeaderPath, ccuRuntimeHeaderText, forbidden); CheckNotContains(ccuRuntimeSourcePath, ccuRuntimeSourceText, forbidden); From 59e84d046229d5ecca6126249d42b644d1ab6283 Mon Sep 17 00:00:00 2001 From: Kur0x Date: Tue, 7 Jul 2026 14:36:55 +0800 Subject: [PATCH 14/41] feat: connect CCU collective backend dispatch --- src/collectives/host/collective_backend.cpp | 13 ++++++++++--- src/comm/tilexr_comm.cpp | 5 +++++ src/comm/tilexr_comm.h | 1 + .../unit/test_tilexr_collective_backend_options.cpp | 11 +++++++++-- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/collectives/host/collective_backend.cpp b/src/collectives/host/collective_backend.cpp index 5b9cff3e..0896a66a 100644 --- a/src/collectives/host/collective_backend.cpp +++ b/src/collectives/host/collective_backend.cpp @@ -26,11 +26,18 @@ int DispatchAiv(const CollectiveRequest &request) int DispatchUdma(const CollectiveRequest &request) { - (void)request; - if (!g_testState.enabled || !g_testState.udmaInitialized) { + if (g_testState.enabled) { + if (!g_testState.udmaInitialized) { + return TileXR::TILEXR_ERROR_NOT_INITIALIZED; + } + return g_testState.udmaSupported ? g_testState.udmaReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; + } + + auto *comm = static_cast(request.comm); + if (comm == nullptr || !comm->IsUdmaAvailableForCollectives()) { return TileXR::TILEXR_ERROR_NOT_INITIALIZED; } - return g_testState.udmaSupported ? g_testState.udmaReturn : TileXR::TILEXR_ERROR_NOT_SUPPORT; + return TileXR::TILEXR_ERROR_NOT_SUPPORT; } int DispatchCcu(const CollectiveRequest &request) diff --git a/src/comm/tilexr_comm.cpp b/src/comm/tilexr_comm.cpp index 82fac246..c41e9ef7 100644 --- a/src/comm/tilexr_comm.cpp +++ b/src/comm/tilexr_comm.cpp @@ -469,6 +469,11 @@ const TileXRUDMARegistry* TileXRComm::GetUDMARegistryHost() const return UDMARegistryValid(&udmaRegistry_, rankSize_) ? &udmaRegistry_ : nullptr; } +bool TileXRComm::IsUdmaAvailableForCollectives() const +{ + return udmaTransport_ != nullptr && udmaTransport_->IsAvailable(); +} + int TileXRComm::InitCommon() { // enable peer device diff --git a/src/comm/tilexr_comm.h b/src/comm/tilexr_comm.h index 83d2c6ce..6b05e259 100644 --- a/src/comm/tilexr_comm.h +++ b/src/comm/tilexr_comm.h @@ -49,6 +49,7 @@ class TileXRComm { int UnregisterUDMAMemory(TileXRUDMAMemHandle handle); GM_ADDR GetUDMARegistryPtr() const; const TileXRUDMARegistry* GetUDMARegistryHost() const; + bool IsUdmaAvailableForCollectives() const; int InitCcuBackend(); TileXRCcuBackend *GetCcuBackendForCollectives(); const TileXRCcuBackend *GetCcuBackendForCollectives() const; diff --git a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp index f86346dd..8a220d64 100644 --- a/tests/collectives/unit/test_tilexr_collective_backend_options.cpp +++ b/tests/collectives/unit/test_tilexr_collective_backend_options.cpp @@ -106,13 +106,20 @@ int CheckBackendDispatch() return 7; } + state.ccuSupported = true; + state.ccuReturn = TileXR::TILEXR_SUCCESS; + SetBackendTestState(state); + if (DispatchCollective(request, TILEXR_COLLECTIVE_BACKEND_CCU) != TileXR::TILEXR_SUCCESS) { + return 8; + } + options.backend = TILEXR_COLLECTIVE_BACKEND_UDMA; state.udmaInitialized = false; state.udmaSupported = false; SetBackendTestState(state); if (TileXRAllGatherEx(&sendValue, &recvValue, 1, TileXR::TILEXR_DATA_TYPE_INT32, request.comm, nullptr, &options) != TileXR::TILEXR_ERROR_NOT_INITIALIZED) { - return 8; + return 9; } options.backend = TILEXR_COLLECTIVE_BACKEND_CCU; @@ -121,7 +128,7 @@ int CheckBackendDispatch() SetBackendTestState(state); if (TileXRAllGatherEx(&sendValue, &recvValue, 1, TileXR::TILEXR_DATA_TYPE_INT32, request.comm, nullptr, &options) != TileXR::TILEXR_ERROR_NOT_SUPPORT) { - return 9; + return 10; } ResetBackendTestState(); From 7267da23ced7168a0212979c308941c6e7379692 Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Wed, 8 Jul 2026 17:21:43 +0800 Subject: [PATCH 15/41] fix(ccu): apply P2P direct copy smoke defaults - Apply direct-CCU-only and resource window defaults for P2P CCU copy smoke runs. - Forward RA context registration mode and lower-layer WQE defaults to rank processes. - Accept skipped non-receiver ranks while requiring at least one passing receiver result. --- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 52 +++++++++++++++++++ .../test_tilexr_ccu_direct_smoke_runner.py | 23 ++++++++ 2 files changed, 75 insertions(+) diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index d48e1dbd..cd55c1a2 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -80,6 +80,42 @@ default_sync_instruction_count() esac } +apply_p2p_ccu_copy_defaults() +{ + if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" != "1" ]; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK="${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK:-0}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-143}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_GSA_START="${TILEXR_CCU_PROBE_GSA_START:-510}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_BARRIER_MODE="${TILEXR_CCU_DIRECT_BARRIER_MODE:-sync_cke}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" +} + +apply_p2p_ccu_copy_defaults + if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" for diagnostic_var in \ @@ -88,6 +124,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW \ TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE \ TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE \ TILEXR_CCU_DIRECT_INSTALL_ORDER \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do @@ -289,6 +326,9 @@ fi if [ "${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE=${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE}") fi +if [ "${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE}") +fi if [ "${TILEXR_CCU_DIRECT_INSTALL_ORDER:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_INSTALL_ORDER=${TILEXR_CCU_DIRECT_INSTALL_ORDER}") fi @@ -594,16 +634,28 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then echo "ERROR: direct CCU P2P CCU-copy check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1" >&2 exit 11 fi + p2p_passed_count=0 for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy" "${log}"; then echo "ERROR: direct CCU P2P CCU-copy result missing in ${log}" >&2 exit 12 fi + if grep -q "tilexr_ccu_direct_smoke p2pCcuCopy .*passed=1" "${log}"; then + p2p_passed_count=$((p2p_passed_count + 1)) + continue + fi + if grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped" "${log}"; then + continue + fi if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy .*passed=1" "${log}"; then echo "ERROR: direct CCU P2P CCU-copy check failed in ${log}" >&2 exit 13 fi done + if [ "${p2p_passed_count}" -lt 1 ]; then + echo "ERROR: direct CCU P2P CCU-copy produced no passing receiver result" >&2 + exit 13 + fi fi echo "tilexr_ccu_direct_smoke_runner success workDir=${work_dir}" diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 94c31cad..85c5be50 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -123,6 +123,29 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): with self.subTest(needle=needle): self.assertLess(gate, source.index(needle)) + def test_runner_p2p_mode_applies_direct_ccu_resource_defaults(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("apply_p2p_ccu_copy_defaults", source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK="${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK:-0}"', source) + self.assertIn('TILEXR_CCU_PROBE_GSA_START="${TILEXR_CCU_PROBE_GSA_START:-510}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}"', source) + self.assertIn('TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}"', source) + self.assertIn('TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}"', source) + self.assertIn('TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}"', source) + self.assertIn('TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}"', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE}")', source) + self.assertIn("p2p_passed_count=0", source) + self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) + self.assertIn("direct CCU P2P CCU-copy produced no passing receiver result", source) + def test_runner_default_run_skips_without_hardware(self): with tempfile.TemporaryDirectory() as temp_dir: env = os.environ.copy() From 8b5a414526feabc2c977437006c47799adbe8048 Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Wed, 8 Jul 2026 17:50:51 +0800 Subject: [PATCH 16/41] fix(ccu): adapt smoke probe to internal backend --- src/comm/CMakeLists.txt | 1 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 253 +++++++++++++----- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 1 + .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 46 ++-- 4 files changed, 214 insertions(+), 87 deletions(-) diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index 4b161a4e..ca75df16 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -170,6 +170,7 @@ set_target_properties(tile-comm PROPERTIES SKIP_BUILD_RPATH TRUE INSTALL_RPATH "" SKIP_INSTALL_RPATH TRUE) +target_compile_definitions(tile-comm PRIVATE TILEXR_CCU_TESTING=1) if(TILEXR_HAVE_PTO_SDMA) target_compile_definitions(tile-comm PRIVATE TILEXR_HAVE_PTO_SDMA=1) diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 232d46da..0e1f899b 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -11,6 +11,10 @@ #include "acl/acl_rt.h" #include "tilexr_api.h" #include "tilexr_types.h" +#include "ccu/tilexr_ccu_collective_planner.h" +#include "ccu/tilexr_ccu_executor.h" +#include "ccu/tilexr_ccu_runtime_session.h" +#include "tools/socket/tilexr_sock_exchange.h" #include "runtime/dev.h" #include @@ -21,6 +25,8 @@ #include #include #include +#include +#include #include #include #include @@ -28,6 +34,26 @@ namespace { +struct DirectCcuSmokeContext { + std::unique_ptr exchange; + TileXR::TileXRCcuRuntimeSession session; + TileXR::TileXRCcuCollectivePlanner planner; + TileXR::TileXRCcuExecutor executor; +}; + +using TileXRDirectCcuPrepareOptions = TileXR::TileXRCcuDirectInstallOptions; +using TileXRDirectCcuPrepareReport = TileXR::TileXRCcuDirectInstallReport; +using TileXRDirectCcuSubmitReport = TileXR::TileXRCcuDirectSubmitReport; +using TileXRDirectCcuPreparedTasksPtr = TileXR::TileXRCcuDirectInstallAttempt*; +using TileXRDirectCcuTaskInfo = TileXR::TileXRCcuTask; +using TileXRDirectCcuInstructionReadbackReport = TileXR::TileXRCcuDriverAdapterReport; + +constexpr uint32_t TILEXR_DIRECT_CCU_SQE_ARGS_LEN = TileXR::TILEXR_CCU_SQE_ARGS_LEN; + +struct TileXRDirectCcuInstructionWords { + uint32_t words[4] = {}; +}; + constexpr const char* kEnableEnv = "TILEXR_CCU_DIRECT_SMOKE_ENABLE"; constexpr const char* kThreadModeEnv = "TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE"; constexpr const char* kDirectCcuOnlyInitEnv = "TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT"; @@ -231,62 +257,62 @@ bool SyncXnLoadPostOnlyBarrierMode() return value != nullptr && std::string(value) == "sync_xn_load_post_only"; } -uint32_t RepositoryInstallWindowFromEnv() +TileXR::TileXRCcuRepositoryInstallWindow RepositoryInstallWindowFromEnv() { const char* value = std::getenv(kRepositoryInstallWindowEnv); if (value == nullptr || value[0] == '\0') { - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION; + return TileXR::TileXRCcuRepositoryInstallWindow::Mission; } const std::string text(value); if (text == "full_repository" || text == "full" || text == "1") { - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_FULL_REPOSITORY; + return TileXR::TileXRCcuRepositoryInstallWindow::FullRepository; } - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_WINDOW_MISSION; + return TileXR::TileXRCcuRepositoryInstallWindow::Mission; } -uint32_t RepositoryInstallDataLenModeFromEnv() +TileXR::TileXRCcuRepositoryInstallDataLenMode RepositoryInstallDataLenModeFromEnv() { const char* value = std::getenv(kRepositoryInstallDataLenModeEnv); if (value == nullptr || value[0] == '\0') { - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES; + return TileXR::TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; } const std::string text(value); if (text == "descriptor_bytes" || text == "descriptor" || text == "1") { - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_DESCRIPTOR_BYTES; + return TileXR::TileXRCcuRepositoryInstallDataLenMode::DescriptorBytes; } - return TILEXR_DIRECT_CCU_REPOSITORY_INSTALL_DATA_LEN_INSTRUCTION_BYTES; + return TileXR::TileXRCcuRepositoryInstallDataLenMode::InstructionBytes; } -uint32_t RepositoryMemoryAllocModeFromEnv() +TileXR::TileXRCcuRepositoryMemoryAllocMode RepositoryMemoryAllocModeFromEnv() { const char* value = std::getenv(kRepositoryMemoryAllocModeEnv); if (value == nullptr || value[0] == '\0') { - return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL; + return TileXR::TileXRCcuRepositoryMemoryAllocMode::Acl; } const std::string text(value); if (text == "acl_module3" || text == "acl_hccl_module" || text == "module3" || text == "1") { - return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL_MODULE3; + return TileXR::TileXRCcuRepositoryMemoryAllocMode::AclModule3; } if (text == "rt_hbm" || text == "rt" || text == "runtime_hbm" || text == "2") { - return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM; + return TileXR::TileXRCcuRepositoryMemoryAllocMode::RtHbm; } - return TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_ACL; + return TileXR::TileXRCcuRepositoryMemoryAllocMode::Acl; } -uint32_t InstallOrderFromEnv() +TileXR::TileXRCcuInstallOrder InstallOrderFromEnv() { const char* value = std::getenv(kInstallOrderEnv); if (value == nullptr || value[0] == '\0') { - return TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST; + return TileXR::TileXRCcuInstallOrder::InstallLowerLayerFirst; } const std::string text(value); if (text == "lower_layer_first" || text == "install_lower_layer_first" || text == "1") { - return TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST; + return TileXR::TileXRCcuInstallOrder::InstallLowerLayerFirst; } if (text == "repository_first" || text == "repo_first" || text == "0") { - return TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST; + return TileXR::TileXRCcuInstallOrder::RepositoryFirst; } - return TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST; + return TileXR::TileXRCcuInstallOrder::RepositoryFirst; } uint32_t DefaultSyncInstructionCount(uint32_t syncResourceCount) @@ -501,12 +527,33 @@ int DeviceFromEnv(int rank) return DeviceFromList(std::getenv("TILEXR_TEST_DEVICES"), rank, firstDevice + rank); } -int InitCommForDirectCcuSmoke(int commDomain, int rankSize, int rank, TileXRCommPtr* raw) +int InitCommForDirectCcuSmoke(int commDomain, int rankSize, int rank, int device, DirectCcuSmokeContext* context) { - if (EnvFlag(kDirectCcuOnlyInitEnv)) { - return TileXRCommInitRankDirectCcuWithDomain(commDomain, rankSize, rank, raw); + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - return TileXRCommInitRankWithDomain(commDomain, rankSize, rank, raw); + + context->exchange.reset(new (std::nothrow) TileXR::TileXRSockExchange(rank, rankSize, commDomain)); + if (context->exchange == nullptr) { + return TileXR::TILEXR_ERROR_INTERNAL; + } + + TileXR::TileXRCcuBackendOptions options {}; + options.rank = rank; + options.rankSize = rankSize; + options.devId = device; + options.uid = "tilexr-direct-smoke-probe"; + options.exchange = context->exchange.get(); + const int ret = context->session.Init(options); + if (ret == TileXR::TILEXR_SUCCESS) { + std::cout << "tilexr_ccu_direct_smoke internalDirectCcuInit" + << " rank=" << rank + << " rankSize=" << rankSize + << " device=" << device + << " directOnly=" << (EnvFlag(kDirectCcuOnlyInitEnv) ? 1 : 0) + << std::endl; + } + return ret; } TileXRDirectCcuPrepareOptions MakePrepareOptions(int rank, int rankSize, int device) @@ -540,8 +587,8 @@ TileXRDirectCcuPrepareOptions MakePrepareOptions(int rank, int rankSize, int dev RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_NOTIFY_CKE_START", kRemoteNotifyCkeStartEnv, 0)); options.remoteNotifyCkeCount = static_cast( RankEnvInt("TILEXR_CCU_PROBE_RANK", rank, "_REMOTE_NOTIFY_CKE_COUNT", kRemoteNotifyCkeCountEnv, 0)); - options.repositoryInstallWindow = RepositoryInstallWindowFromEnv(); - options.repositoryInstallDataLenMode = RepositoryInstallDataLenModeFromEnv(); + options.repositoryInstallOptions.window = RepositoryInstallWindowFromEnv(); + options.repositoryInstallOptions.dataLenMode = RepositoryInstallDataLenModeFromEnv(); options.repositoryMemoryAllocMode = RepositoryMemoryAllocModeFromEnv(); options.installOrder = InstallOrderFromEnv(); options.deviceId = static_cast(device); @@ -584,6 +631,76 @@ void PrintSubmitReport( << std::endl; } +int TileXRDirectCcuGetPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t taskIndex, + TileXRDirectCcuTaskInfo* task) +{ + if (prepared == nullptr || task == nullptr || taskIndex >= prepared->submitTasks.size()) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + *task = prepared->submitTasks[taskIndex]; + return TileXR::TILEXR_SUCCESS; +} + +int TileXRDirectCcuSubmitPrepared( + TileXRDirectCcuPreparedTasksPtr prepared, + void* stream, + TileXRDirectCcuSubmitReport* report) +{ + if (prepared == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TileXR::TileXRCcuSubmitPreparedTasks(prepared->submitTasks, stream, nullptr, nullptr, report); +} + +int TileXRDirectCcuSubmitPreparedTask( + TileXRDirectCcuPreparedTasksPtr prepared, + uint32_t taskIndex, + void* stream, + TileXRDirectCcuSubmitReport* report) +{ + if (prepared == nullptr || taskIndex >= prepared->submitTasks.size()) { + if (report != nullptr) { + *report = TileXRDirectCcuSubmitReport {}; + report->taskCount = prepared == nullptr ? 0U : static_cast(prepared->submitTasks.size()); + report->message = "selected direct CCU submit task is missing"; + } + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector selected {prepared->submitTasks[taskIndex]}; + return TileXR::TileXRCcuSubmitPreparedTasks(selected, stream, nullptr, nullptr, report); +} + +int TileXRDirectCcuDestroyPrepared(TileXRDirectCcuPreparedTasksPtr prepared) +{ + if (prepared == nullptr) { + return TileXR::TILEXR_SUCCESS; + } + return TileXR::TileXRCcuReleaseDirectInstallAttemptResources(*prepared); +} + +int TileXRCommReadDirectCcuInstructions( + DirectCcuSmokeContext* context, + uint8_t dieId, + uint16_t instructionStartId, + uint32_t instructionCount, + TileXRDirectCcuInstructionWords* instructions, + TileXRDirectCcuInstructionReadbackReport* report) +{ + if (context == nullptr || instructions == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + return context->executor.ReadDirectCcuInstructionsForDebug( + context->session, + dieId, + instructionStartId, + instructions, + instructionCount, + sizeof(TileXRDirectCcuInstructionWords), + report); +} + int SubmitPreparedWithSelector( TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount, @@ -600,10 +717,7 @@ int SubmitPreparedWithSelector( if (!selectFirst && !selectSecond) { if (report != nullptr) { *report = TileXRDirectCcuSubmitReport {}; - std::strncpy( - report->message, - "invalid TILEXR_CCU_DIRECT_SMOKE_SUBMIT_TASK_SELECTOR", - sizeof(report->message) - 1U); + report->message = "invalid TILEXR_CCU_DIRECT_SMOKE_SUBMIT_TASK_SELECTOR"; } return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -618,7 +732,7 @@ int SubmitPreparedWithSelector( if (report != nullptr) { *report = TileXRDirectCcuSubmitReport {}; report->taskCount = taskCount; - std::strncpy(report->message, "selected direct CCU submit task is missing", sizeof(report->message) - 1U); + report->message = "selected direct CCU submit task is missing"; } return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -669,10 +783,10 @@ void PrintConfig( << " localWaitCkeCount=" << options.localWaitCkeCount << " remoteNotifyCkeStartId=" << options.remoteNotifyCkeStartId << " remoteNotifyCkeCount=" << options.remoteNotifyCkeCount - << " repositoryInstallWindow=" << options.repositoryInstallWindow - << " repositoryInstallDataLenMode=" << options.repositoryInstallDataLenMode - << " repositoryMemoryAllocMode=" << options.repositoryMemoryAllocMode - << " installOrder=" << options.installOrder + << " repositoryInstallWindow=" << static_cast(options.repositoryInstallOptions.window) + << " repositoryInstallDataLenMode=" << static_cast(options.repositoryInstallOptions.dataLenMode) + << " repositoryMemoryAllocMode=" << static_cast(options.repositoryMemoryAllocMode) + << " installOrder=" << static_cast(options.installOrder) << " barrierMode=\"" << (std::getenv(kBarrierModeEnv) == nullptr ? "" : std::getenv(kBarrierModeEnv)) << "\"" << " resourceWindowTokenId=\"" @@ -684,7 +798,7 @@ void PrintConfig( << " resourceWindowTokenValue=\"" << (std::getenv(kResourceWindowTokenValueEnv) == nullptr ? "" : std::getenv(kResourceWindowTokenValueEnv)) << "\"" - << " provider=\"" << (options.provider == nullptr ? "" : options.provider) << "\"" + << " provider=\"" << options.provider << "\"" << std::endl; } @@ -717,15 +831,15 @@ void PrintPreparedTasks(TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskC std::cout << std::endl; } -void PrintInstructionReadback(TileXRCommPtr raw, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) +void PrintInstructionReadback(DirectCcuSmokeContext* context, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) { if (!EnvFlag(kReadbackInstructionsEnv)) { return; } - if (raw == nullptr) { + if (context == nullptr) { std::cout << "tilexr_ccu_direct_smoke instructionReadback ret=" << TileXR::TILEXR_ERROR_PARA_CHECK_FAIL - << " message=\"missing TileXRComm\"" + << " message=\"missing direct CCU smoke context\"" << std::endl; return; } @@ -747,7 +861,7 @@ void PrintInstructionReadback(TileXRCommPtr raw, TileXRDirectCcuPreparedTasksPtr std::vector readback(readCount); TileXRDirectCcuInstructionReadbackReport report; const int readRet = TileXRCommReadDirectCcuInstructions( - raw, + context, static_cast(task.dieId), static_cast(task.instStartId), readCount, @@ -963,7 +1077,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) } } -int RunPreparedSmokeForRank(TileXRCommPtr raw, int rank, int rankSize, int device) +int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); const int peer = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; @@ -977,25 +1091,26 @@ int RunPreparedSmokeForRank(TileXRCommPtr raw, int rank, int rankSize, int devic } PrintConfig(options, rankSize); - TileXRDirectCcuPreparedTasksPtr prepared = nullptr; + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; TileXRDirectCcuPrepareReport installReport; int ret = p2pCcuCopyEnabled && p2pCcuCopy.initRet != ACL_SUCCESS ? p2pCcuCopy.initRet : p2pCcuCopyEnabled ? - [&]() { - TileXRDirectCcuMemoryCopyPrepareOptions copyOptions {}; - copyOptions.prepare = options; - copyOptions.localSourceAddr = reinterpret_cast(p2pCcuCopy.source.ptr); - copyOptions.localDestinationAddr = reinterpret_cast(p2pCcuCopy.destination.ptr); - copyOptions.bytes = p2pCcuCopy.bytes; - copyOptions.peerRank = static_cast(peer); - copyOptions.direction = TILEXR_DIRECT_CCU_MEMORY_COPY_REMOTE_TO_LOCAL; - return TileXRCommPrepareDirectCcuMemoryCopy(raw, ©Options, &prepared, &installReport); - }() : - TileXRCommPrepareDirectCcu(raw, &options, &prepared, &installReport); + context->planner.PrepareDirectCcuMemoryCopyInstallAttempt( + context->session, + options, + reinterpret_cast(p2pCcuCopy.source.ptr), + reinterpret_cast(p2pCcuCopy.destination.ptr), + p2pCcuCopy.bytes, + static_cast(peer), + TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, + prepared, + &installReport) : + context->planner.PrepareDirectCcuInstallAttempt(context->session, options, prepared, &installReport); PrintInstallReport("tilexr_ccu_direct_smoke prepare", ret, installReport); PrintPreparedTasks(prepared, installReport.submitTaskCount); - PrintInstructionReadback(raw, prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); int finalRet = 0; const bool submitRequested = EnvFlag(kSubmitEnv); @@ -1127,7 +1242,18 @@ int RunThreadModeSmoke(int rankSize) rankResults[rank] = 14; return; } - rankResults[rank] = RunPreparedSmokeForRank(comms[rank], rank, rankSize, devices[rank]); + DirectCcuSmokeContext context; + const int initRet = InitCommForDirectCcuSmoke(0, rankSize, rank, devices[rank], &context); + if (initRet != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke threadMode direct CCU init ret=" + << initRet + << " rank=" << rank + << " device=" << devices[rank] + << std::endl; + rankResults[rank] = 5; + return; + } + rankResults[rank] = RunPreparedSmokeForRank(&context, rank, rankSize, devices[rank]); }); } for (auto& rankThread : rankThreads) { @@ -1215,17 +1341,16 @@ int main() return 4; } - TileXRCommPtr raw = nullptr; - ret = InitCommForDirectCcuSmoke(commDomain, rankSize, rank, &raw); - if (ret != TileXR::TILEXR_SUCCESS || raw == nullptr) { - std::cerr << "tilexr_ccu_direct_smoke comm init ret=" << ret - << " raw=" << raw << std::endl; + DirectCcuSmokeContext context; + ret = InitCommForDirectCcuSmoke(commDomain, rankSize, rank, device, &context); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_direct_smoke direct CCU context init ret=" << ret << std::endl; aclrtResetDevice(device); aclFinalize(); return 5; } - int finalRet = RunPreparedSmokeForRank(raw, rank, rankSize, device); + int finalRet = RunPreparedSmokeForRank(&context, rank, rankSize, device); if (ShouldFastExitAfterPrepareFailure(finalRet)) { std::cout << "tilexr_ccu_direct_smoke fastExitOnPrepareFailure=1" << " ret=" << finalRet @@ -1244,13 +1369,9 @@ int main() std::fflush(stderr); std::_Exit(finalRet); } - TraceLifecycle("before TileXRCommDestroy"); - const int destroyRet = TileXRCommDestroy(raw); - TraceLifecycle("after TileXRCommDestroy"); - if (destroyRet != TileXR::TILEXR_SUCCESS) { - std::cerr << "tilexr_ccu_direct_smoke destroy ret=" << destroyRet << std::endl; - finalRet = finalRet == 0 ? 10 : finalRet; - } + TraceLifecycle("before DirectCcuSmokeContext shutdown"); + context.session.Shutdown(); + TraceLifecycle("after DirectCcuSmokeContext shutdown"); TraceLifecycle("before aclrtResetDevice"); aclrtResetDevice(device); TraceLifecycle("after aclrtResetDevice"); diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index cd55c1a2..5afcac6b 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -231,6 +231,7 @@ driver_lib_dir="${ASCEND_DRIVER_PATH}/lib64/driver" probe_bin="${work_dir}/ccu_tilexr_direct_smoke_probe" c++ -std=c++14 \ + -DTILEXR_CCU_TESTING=1 \ -I "${repo_root}/src/include" \ -I "${repo_root}/src/comm" \ -I "${cann_root}/pkg_inc" \ diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 29f9f6fd..7c8279bc 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -118,6 +118,7 @@ def compile_probe(self): compile_cmd = [ compiler, "-std=c++14", + "-DTILEXR_CCU_TESTING=1", "-I", str(INCLUDE_DIR), "-I", @@ -180,7 +181,7 @@ def test_source_exists_and_defines_default_safe_env_guards(self): self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE", source) - self.assertIn("TILEXR_DIRECT_CCU_REPOSITORY_MEMORY_ALLOC_RT_HBM", source) + self.assertIn("TileXRCcuRepositoryMemoryAllocMode::RtHbm", source) self.assertIn('text == "rt_hbm"', source) self.assertIn("TILEXR_CCU_DIRECT_INSTALL_ORDER", source) self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_ID", source) @@ -189,6 +190,9 @@ def test_source_exists_and_defines_default_safe_env_guards(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE", source) self.assertIn("InitCommForDirectCcuSmoke", source) self.assertIn("RunThreadModeSmoke", source) + self.assertIn("DirectCcuSmokeContext", source) + self.assertIn("TileXRCcuRuntimeSession", source) + self.assertIn("TileXRSockExchange", source) self.assertIn("ShouldFastExitAfterPrepareFailure", source) self.assertIn("ShouldFastExitAfterRun", source) self.assertIn("TraceLifecycle", source) @@ -205,10 +209,9 @@ def test_source_exists_and_defines_default_safe_env_guards(self): self.assertIn('std::string(value) == "local_cke_post_only"', source) self.assertIn('std::string(value) == "sync_xn_post_only"', source) self.assertIn('std::string(value) == "sync_xn_load_post_only"', source) - self.assertIn("TileXRCommInitRankWithDomain", source) - self.assertIn("TileXRCommInitRankDirectCcuWithDomain", source) - self.assertIn("TileXRCommPrepareDirectCcu", source) - self.assertIn("TileXRCommPrepareDirectCcuMemoryCopy", source) + self.assertIn("PrepareDirectCcuInstallAttempt", source) + self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) + self.assertIn("ReadDirectCcuInstructionsForDebug", source) self.assertIn("TileXRDirectCcuSubmitPrepared", source) self.assertIn("TileXRDirectCcuDestroyPrepared", source) self.assertIn("tilexr_ccu_direct_smoke config", source) @@ -237,8 +240,8 @@ def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): self.assertIn("kP2pCcuCopyEnv", source) self.assertIn("RunP2pCcuCopy", source) - self.assertIn("TileXRDirectCcuMemoryCopyPrepareOptions", source) - self.assertIn("TileXRCommPrepareDirectCcuMemoryCopy", source) + self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) + self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) self.assertIn("p2pCcuCopy", source) @@ -251,7 +254,7 @@ def test_thread_mode_path_uses_single_process_init_and_never_rank_ipc_init(self) prepared_body = source[source.index("RunPreparedSmokeForRank"): thread_mode_pos] self.assertIn("TileXRCommInitAll", thread_mode_body) self.assertIn("RunPreparedSmokeForRank", thread_mode_body) - self.assertIn("TileXRCommPrepareDirectCcu", prepared_body) + self.assertIn("PrepareDirectCcuInstallAttempt", prepared_body) self.assertIn("TileXRDirectCcuSubmitPrepared", prepared_body) self.assertNotIn("TileXRCommInitRankWithDomain", thread_mode_body) self.assertNotIn("TILEXR_COMM_ID", thread_mode_body) @@ -279,8 +282,9 @@ def test_process_mode_can_opt_into_direct_ccu_only_init_to_bypass_peer_ipc(self) main_source = source[source.index("int main()"):] self.assertIn("TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT", source) - self.assertIn("TileXRCommInitRankDirectCcuWithDomain", init_helper_body) - self.assertIn("TileXRCommInitRankWithDomain", init_helper_body) + self.assertIn("TileXRSockExchange", init_helper_body) + self.assertIn("TileXRCcuBackendOptions", init_helper_body) + self.assertIn("context->session.Init(options)", init_helper_body) self.assertIn("EnvFlag(kDirectCcuOnlyInitEnv)", init_helper_body) self.assertIn("InitCommForDirectCcuSmoke", main_source) @@ -290,12 +294,12 @@ def test_process_mode_fast_exit_skips_comm_destroy_after_prepare_failure(self): final_ret_pos = main_source.index("int finalRet = RunPreparedSmokeForRank") fast_exit_pos = main_source.index("ShouldFastExitAfterPrepareFailure(finalRet)") - destroy_pos = main_source.index("TileXRCommDestroy(raw)", final_ret_pos) + shutdown_pos = main_source.index("context.session.Shutdown()", final_ret_pos) reset_pos = main_source.index("aclrtResetDevice(device)", final_ret_pos) finalize_pos = main_source.index("aclFinalize()", final_ret_pos) self.assertLess(final_ret_pos, fast_exit_pos) - self.assertLess(fast_exit_pos, destroy_pos) + self.assertLess(fast_exit_pos, shutdown_pos) self.assertLess(fast_exit_pos, reset_pos) self.assertLess(fast_exit_pos, finalize_pos) self.assertIn("std::fflush(stdout)", main_source) @@ -322,10 +326,10 @@ def test_process_mode_can_fast_exit_after_run_to_isolate_cleanup_hangs(self): final_ret_pos = main_source.index("int finalRet = RunPreparedSmokeForRank") fast_exit_pos = main_source.index("ShouldFastExitAfterRun()") - destroy_pos = main_source.index("TileXRCommDestroy(raw)", final_ret_pos) + shutdown_pos = main_source.index("context.session.Shutdown()", final_ret_pos) self.assertLess(final_ret_pos, fast_exit_pos) - self.assertLess(fast_exit_pos, destroy_pos) + self.assertLess(fast_exit_pos, shutdown_pos) self.assertIn("tilexr_ccu_direct_smoke fastExitAfterRun=1", source) self.assertIn("TraceLifecycle(\"before TileXRDirectCcuDestroyPrepared\")", source) self.assertIn("TraceLifecycle(\"after TileXRDirectCcuDestroyPrepared\")", source) @@ -406,8 +410,8 @@ def test_probe_wires_gsa_and_split_cke_env_into_prepare_options_and_config_trace "options.localWaitCkeCount", "options.remoteNotifyCkeStartId", "options.remoteNotifyCkeCount", - "options.repositoryInstallWindow", - "options.repositoryInstallDataLenMode", + "options.repositoryInstallOptions.window", + "options.repositoryInstallOptions.dataLenMode", "options.repositoryMemoryAllocMode", "options.installOrder", "options.sqeArgCount", @@ -444,16 +448,16 @@ def test_probe_prepared_task_trace_prints_full_runtime_args(self): def test_probe_defaults_to_lower_layer_first_install_order(self): source = PROBE_SOURCE.read_text(encoding="utf-8") install_order_body = source[ - source.index("uint32_t InstallOrderFromEnv()"): + source.index("TileXR::TileXRCcuInstallOrder InstallOrderFromEnv()"): source.index("uint32_t DefaultSyncInstructionCount") ] - self.assertIn("TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST", install_order_body) + self.assertIn("TileXRCcuInstallOrder::InstallLowerLayerFirst", install_order_body) self.assertIn("repository_first", install_order_body) - self.assertIn("TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST", install_order_body) + self.assertIn("TileXRCcuInstallOrder::RepositoryFirst", install_order_body) self.assertLess( - install_order_body.index("TILEXR_DIRECT_CCU_INSTALL_ORDER_LOWER_LAYER_FIRST"), - install_order_body.index("TILEXR_DIRECT_CCU_INSTALL_ORDER_REPOSITORY_FIRST"), + install_order_body.index("TileXRCcuInstallOrder::InstallLowerLayerFirst"), + install_order_body.index("TileXRCcuInstallOrder::RepositoryFirst"), ) def test_probe_default_sync_instruction_count_includes_hcomm_style_task1_prelude(self): From ca88fd87d1a1e8c3e8b9b014909f8159d964bc40 Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Fri, 10 Jul 2026 14:44:30 +0800 Subject: [PATCH 17/41] fix(ccu): harden direct runtime lifecycle Initialize the CCU backend behind an explicit environment gate and keep comm initialization degradable when direct CCU is unavailable. Track direct CCU runtime init failures per device, add RA ctx EID selection for resource-window registration, and make P2P smoke submit active-rank only. Reclaim allocator resources on LIFO receipt release and cover the lifecycle, runner, allocator, and smoke probe paths with focused tests. --- .../ccu/tilexr_ccu_collective_planner.cpp | 14 +- src/comm/ccu/tilexr_ccu_direct_runtime.cpp | 65 ++++++- .../ccu/tilexr_ccu_resource_allocator.cpp | 28 +++ src/comm/ccu/tilexr_ccu_runtime_session.cpp | 34 ++-- src/comm/ccu/tilexr_ccu_runtime_session.h | 3 +- src/comm/tilexr_comm.cpp | 39 +++++ src/comm/tilexr_comm.h | 1 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 69 +++++++- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 14 ++ tests/ccu/test_tilexr_ccu_backend_boundary.py | 28 +++ .../test_tilexr_ccu_direct_orchestrator.py | 12 ++ .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 49 ++++++ .../test_tilexr_ccu_direct_smoke_runner.py | 22 +++ ...est_tilexr_ccu_lower_layer_plan_builder.py | 161 ++++++++++++++++-- .../ccu/test_tilexr_ccu_resource_allocator.py | 97 +++++++++++ 15 files changed, 596 insertions(+), 40 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 311eec0d..30731932 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -569,12 +569,11 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt( } return TILEXR_ERROR_NOT_INITIALIZED; } - const std::string processUnavailableMessage = - TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport{}; - report->message = processUnavailableMessage; + report->message = unavailableMessage; } return TILEXR_ERROR_NOT_FOUND; } @@ -656,12 +655,11 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - const std::string processUnavailableMessage = - TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; - report->message = processUnavailableMessage; + report->message = unavailableMessage; } return TILEXR_ERROR_NOT_FOUND; } diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp index c1291d26..eeb70259 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -42,6 +42,8 @@ constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_TOKEN_VALUE"; constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX"; constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE"; constexpr const char* TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE_ENV = @@ -172,6 +174,15 @@ bool HasRankedEnv(const char* base, int rank) return SelectRankedEnv(base, rank) != nullptr; } +std::array CopyRawEid(const TileXRCcuHccpEid& eid) +{ + std::array copied {}; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + copied[i] = eid.raw[i]; + } + return copied; +} + bool IsRaCtxResourceWindowRegistrationMode() { const char* value = std::getenv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV); @@ -201,6 +212,47 @@ void TraceEndpointRouteStep(const std::string& message) } } +void TraceRaCtxEidInfos(const std::vector& eidInfos) +{ + if (!TraceEndpointRoute()) { + return; + } + for (size_t i = 0; i < eidInfos.size(); ++i) { + std::cerr << "TileXRDirectCcuTrace endpointRoute raCtxEidInfo" + << " ordinal=" << i + << " eidIndex=" << eidInfos[i].eidIndex + << " eid=" << FormatEndpointEid(CopyRawEid(eidInfos[i].eid)) + << std::endl; + } +} + +bool SelectRaCtxResourceWindowEidInfo( + int rank, + const std::vector& eidInfos, + TileXRCcuHccpDevEidInfo* selectedEid) +{ + if (eidInfos.empty() || selectedEid == nullptr) { + return false; + } + TraceRaCtxEidInfos(eidInfos); + const char* configured = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, rank); + if (configured == nullptr) { + *selectedEid = eidInfos[0]; + return true; + } + uint64_t configuredIndex = 0; + if (!ParseUnsignedEnv(configured, &configuredIndex) || configuredIndex > 0xffffffffULL) { + return false; + } + for (const auto& eidInfo : eidInfos) { + if (eidInfo.eidIndex == static_cast(configuredIndex)) { + *selectedEid = eidInfo; + return true; + } + } + return false; +} + bool IsRaCtxLoopEndpointRouteCollectionMode() { const char* value = std::getenv(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV); @@ -695,13 +747,18 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( void* tokenIdHandle = nullptr; void* lmemHandle = nullptr; + TileXRCcuHccpDevEidInfo selectedEid {}; + if (!SelectRaCtxResourceWindowEidInfo(options_.rank, eidInfos, &selectedEid)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRCcuHccpCtxInitCfg ctxCfg {}; ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; ctxCfg.rdma.disabledLiteThread = false; TileXRCcuHccpCtxInitAttr ctxAttr {}; ctxAttr.phyId = devicePhyId_; - ctxAttr.ub.eidIndex = eidInfos[0].eidIndex; - ctxAttr.ub.eid = eidInfos[0].eid; + ctxAttr.ub.eidIndex = selectedEid.eidIndex; + ctxAttr.ub.eid = selectedEid.eid; ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); if (ret != 0 || ctxHandle == nullptr) { @@ -771,9 +828,9 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( localResourceWindow_.tokenIdHandle = tokenIdHandle; localResourceWindow_.lmemHandle = lmemHandle; for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { - localResourceWindow_.eid[i] = eidInfos[0].eid.raw[i]; + localResourceWindow_.eid[i] = selectedEid.eid.raw[i]; } - localResourceWindow_.eidIndex = eidInfos[0].eidIndex; + localResourceWindow_.eidIndex = selectedEid.eidIndex; localResourceWindow_.raCtxRegistered = true; resourceWindowRegistered_ = true; return TILEXR_SUCCESS; diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp index e9bb407b..282d9fe2 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp @@ -370,6 +370,34 @@ int TileXRCcuResourceAllocator::Release(uint64_t receiptId) if (it == active_.end()) { return TILEXR_ERROR_PARA_CHECK_FAIL; } + const auto latest = active_.rbegin(); + if (latest == active_.rend() || latest->first != receiptId) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const ActiveAllocation& active = it->second; + if (mission_.used < active.missionUsed || + repository_.used < active.repositoryUsed || + xn_.used < active.localXnUsed || + gsa_.used < active.localGsaUsed || + localWaitCke_.used < static_cast(active.localWaitCkeUsed + active.sourceCkeUsed) || + remoteNotifyCke_.used < active.remoteNotifyCkeUsed || + channel_.used < active.channelUsed || + (remoteXn_.count != 0 && remoteXn_.used < active.remoteXnUsed)) { + return TILEXR_ERROR_INTERNAL; + } + mission_.used = static_cast(mission_.used - active.missionUsed); + repository_.used = static_cast(repository_.used - active.repositoryUsed); + xn_.used = static_cast(xn_.used - active.localXnUsed); + if (remoteXn_.count == 0) { + xn_.used = static_cast(xn_.used - active.remoteXnUsed); + } else { + remoteXn_.used = static_cast(remoteXn_.used - active.remoteXnUsed); + } + gsa_.used = static_cast(gsa_.used - active.localGsaUsed); + localWaitCke_.used = static_cast( + localWaitCke_.used - active.localWaitCkeUsed - active.sourceCkeUsed); + remoteNotifyCke_.used = static_cast(remoteNotifyCke_.used - active.remoteNotifyCkeUsed); + channel_.used = static_cast(channel_.used - active.channelUsed); active_.erase(it); return TILEXR_SUCCESS; } diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.cpp b/src/comm/ccu/tilexr_ccu_runtime_session.cpp index d6c66746..63cbdf87 100644 --- a/src/comm/ccu/tilexr_ccu_runtime_session.cpp +++ b/src/comm/ccu/tilexr_ccu_runtime_session.cpp @@ -30,19 +30,23 @@ struct TileXRThreadAllGatherState { static map g_directCcuAllGatherStates; static std::mutex g_mtx; static std::mutex g_ccuDirectRuntimeMtx; -static bool g_ccuDirectRuntimeUnavailable = false; -static std::string g_ccuDirectRuntimeUnavailableMessage; +static std::map g_ccuDirectRuntimeUnavailableByDevice; -std::string TileXRCcuRuntimeSession::ProcessDirectCcuRuntimeUnavailableMessage() +std::string TileXRCcuRuntimeSession::DirectCcuRuntimeUnavailableMessageForDevice(int devId) { lock_guard lock(g_ccuDirectRuntimeMtx); - if (!g_ccuDirectRuntimeUnavailable) { + const auto it = g_ccuDirectRuntimeUnavailableByDevice.find(devId); + if (it == g_ccuDirectRuntimeUnavailableByDevice.end()) { return {}; } - return g_ccuDirectRuntimeUnavailableMessage.empty() ? - "direct CCU runtime unavailable after process-level init failure" : - "direct CCU runtime unavailable after process-level init failure: " + - g_ccuDirectRuntimeUnavailableMessage; + return it->second.empty() ? + "direct CCU runtime unavailable for device " + std::to_string(devId) : + "direct CCU runtime unavailable for device " + std::to_string(devId) + ": " + it->second; +} + +std::string TileXRCcuRuntimeSession::DirectCcuRuntimeUnavailableMessage() const +{ + return DirectCcuRuntimeUnavailableMessageForDevice(devId_); } void TileXRCcuRuntimeSession::Shutdown() @@ -88,8 +92,9 @@ int TileXRCcuRuntimeSession::Init(const TileXRCcuBackendOptions &options) } lock_guard lock(g_ccuDirectRuntimeMtx); - if (g_ccuDirectRuntimeUnavailable) { - TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure"; + const auto unavailable = g_ccuDirectRuntimeUnavailableByDevice.find(devId_); + if (unavailable != g_ccuDirectRuntimeUnavailableByDevice.end()) { + TILEXR_LOG(INFO) << "direct CCU runtime skipped after previous init failure on device " << devId_; return TILEXR_SUCCESS; } @@ -115,8 +120,7 @@ int TileXRCcuRuntimeSession::Init(const TileXRCcuBackendOptions &options) << ", raInitialized " << (runtimeReport.raInitialized ? 1 : 0) << ", ccuTlvInitialized " << (runtimeReport.ccuTlvInitialized ? 1 : 0) << ", " << runtimeReport.message << ", direct CCU disabled"; - g_ccuDirectRuntimeUnavailable = true; - g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message; + g_ccuDirectRuntimeUnavailableByDevice[devId_] = runtimeReport.message; ResetDirectCcuBasicInfo(); ccuDirectRuntime_.reset(); return TILEXR_SUCCESS; @@ -315,13 +319,13 @@ int TileXRCcuRuntimeSession::DirectCcuThreadAllGather(const void *sendBuf, size_ return TILEXR_SUCCESS; } } - const std::string processUnavailableMessage = ProcessDirectCcuRuntimeUnavailableMessage(); - if (!processUnavailableMessage.empty()) { + const std::string unavailableMessage = DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { lock_guard lock(g_mtx); g_directCcuAllGatherStates.erase(key); TILEXR_LOG(ERROR) << "direct CCU thread allgather abort rank " << rank_ << "/" << rankSize_ << " uid " << uid_ << " round " << round << ", " - << processUnavailableMessage; + << unavailableMessage; return TILEXR_ERROR_NOT_FOUND; } this_thread::sleep_for(1ms); diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.h b/src/comm/ccu/tilexr_ccu_runtime_session.h index d0010864..dbcc16dd 100644 --- a/src/comm/ccu/tilexr_ccu_runtime_session.h +++ b/src/comm/ccu/tilexr_ccu_runtime_session.h @@ -41,10 +41,11 @@ class TileXRCcuRuntimeSession { int RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRuntimeReport *report); int CreateDriverAdapter(TileXRCcuDriverAdapter *adapter, TileXRCcuDriverAdapterReport *report); - static std::string ProcessDirectCcuRuntimeUnavailableMessage(); + std::string DirectCcuRuntimeUnavailableMessage() const; private: void ResetDirectCcuBasicInfo(); + static std::string DirectCcuRuntimeUnavailableMessageForDevice(int devId); static int DirectCcuAllGatherCallback(const void *sendBuf, size_t sendBytes, void *recvBuf, void *userData); int DirectCcuThreadAllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); diff --git a/src/comm/tilexr_comm.cpp b/src/comm/tilexr_comm.cpp index c41e9ef7..5ce190ce 100644 --- a/src/comm/tilexr_comm.cpp +++ b/src/comm/tilexr_comm.cpp @@ -57,6 +57,17 @@ static bool g_udmaUnavailable = false; static std::mutex g_sdmaMtx; static bool g_sdmaUnavailable = false; +constexpr const char* TILEXR_ENABLE_CCU_BACKEND_ENV = "TILEXR_ENABLE_CCU_BACKEND"; + +bool ShouldEnableCcuBackend() +{ + const char *value = std::getenv(TILEXR_ENABLE_CCU_BACKEND_ENV); + if (value == nullptr || value[0] == '\0') { + return false; + } + const string text(value); + return text == "1" || text == "true" || text == "TRUE" || text == "yes" || text == "on"; +} // 如果是互联的链路，返回false； 对910B2C那些不互联的链路，返回true bool SkipUnusedChannel910B2C(int curRank, int peerRank, ChipName chipName) @@ -190,6 +201,26 @@ int TileXRComm::InitCcuBackend() return ccuBackend_->Init(options); } +int TileXRComm::InitCcuBackendIfEnabled() +{ + if (!ShouldEnableCcuBackend()) { + return TILEXR_SUCCESS; + } + + const int ccuRet = InitCcuBackend(); + if (ccuRet != TILEXR_SUCCESS || ccuBackend_ == nullptr || !ccuBackend_->Available()) { + TILEXR_LOG(WARN) << "TileXR CCU backend init failed, direct CCU disabled, ret: " << ccuRet; + if (ccuBackend_ != nullptr) { + ccuBackend_->Shutdown(); + ccuBackend_.reset(); + } + return TILEXR_SUCCESS; + } + + TILEXR_LOG(INFO) << "TileXR CCU backend initialized"; + return TILEXR_SUCCESS; +} + TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives() { return ccuBackend_.get(); @@ -579,6 +610,10 @@ int TileXRComm::Init() if (ret != TILEXR_SUCCESS) { return ret; } + ret = InitCcuBackendIfEnabled(); + if (ret != TILEXR_SUCCESS) { + return ret; + } // set comm args in device. ret = SyncCommArgs(); @@ -651,6 +686,10 @@ int TileXRComm::InitThread(const std::string &uid) if (ret != TILEXR_SUCCESS) { return ret; } + ret = InitCcuBackendIfEnabled(); + if (ret != TILEXR_SUCCESS) { + return ret; + } ret = SyncCommArgs(); if (ret != TILEXR_SUCCESS) { TILEXR_LOG(ERROR) << "SyncCommArgs failed! ret: " << ret; diff --git a/src/comm/tilexr_comm.h b/src/comm/tilexr_comm.h index 6b05e259..d01b4433 100644 --- a/src/comm/tilexr_comm.h +++ b/src/comm/tilexr_comm.h @@ -80,6 +80,7 @@ class TileXRComm { int SyncCommArgs(); int InitDumpAddr(); int InitUDMA(); + int InitCcuBackendIfEnabled(); int InitSDMA(); int UpdateCommArgsDev(); void FreeUDMARegistry(); diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 0e1f899b..a2cd1403 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -71,6 +71,8 @@ constexpr const char* kPreSubmitDelayMsEnv = "TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT constexpr const char* kP2pCcuCopyEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY"; constexpr const char* kExpectP2pCcuCopyEnv = "TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY"; constexpr const char* kP2pCcuCopyBytesEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES"; +constexpr const char* kP2pCcuCopyActiveRankEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK"; +constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION"; constexpr const char* kLocalWaitCkeStartEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START"; constexpr const char* kLocalWaitCkeCountEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT"; constexpr const char* kRemoteNotifyCkeStartEnv = "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START"; @@ -182,6 +184,30 @@ int EnvInt(const char* name, int fallback) return ParseInt(std::getenv(name), fallback); } +bool IsP2pCcuCopyActiveRank(int rank) +{ + return rank == EnvInt(kP2pCcuCopyActiveRankEnv, 0); +} + +TileXR::TileXRCcuMemoryCopyDirection P2pCcuCopyDirectionFromEnv() +{ + const char* value = std::getenv(kP2pCcuCopyDirectionEnv); + if (value != nullptr && std::string(value) == "local_to_remote") { + return TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote; + } + return TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal; +} + +bool ShouldCheckInactiveP2pCcuCopyRank() +{ + return P2pCcuCopyDirectionFromEnv() == TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote; +} + +bool ShouldCheckActiveP2pCcuCopyRank() +{ + return P2pCcuCopyDirectionFromEnv() == TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal; +} + uint64_t ParseU64(const char* value, uint64_t fallback) { if (value == nullptr || value[0] == '\0') { @@ -476,6 +502,17 @@ void PrintP2pCcuCopy( << std::endl; } +void PrintP2pCcuCopySkipped(int rank, int peer, const P2pCcuCopyState& state) +{ + std::cout << "tilexr_ccu_direct_smoke p2pCcuCopy skipped" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << state.bytes + << " activeRank=" << EnvInt(kP2pCcuCopyActiveRankEnv, 0) + << " reason=\"inactive p2p CCU-copy rank\"" + << std::endl; +} + int RunP2pCcuCopy( int rank, int peer, @@ -487,8 +524,10 @@ int RunP2pCcuCopy( if (state == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { + if (ShouldCheckActiveP2pCcuCopyRank() && syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { (void)CheckP2pCcuCopyState(state); + } else if (!ShouldCheckActiveP2pCcuCopyRank() && syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { + state->passed = true; } PrintP2pCcuCopy(rank, peer, *state, prepareRet, submitRet, syncRet); return state->passed ? TileXR::TILEXR_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; @@ -1077,11 +1116,35 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) } } +bool WaitForInactiveP2pCcuCopyRank(int rank, int rankSize, int localResult) +{ + return WaitForCollectiveSubmitDone(rank, rankSize, localResult); +} + +int RunInactiveP2pCcuCopyRank(int rank, int peer, int rankSize, P2pCcuCopyState* state, int localResult) +{ + if (state == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + PrintP2pCcuCopySkipped(rank, peer, *state); + if (!WaitForInactiveP2pCcuCopyRank(rank, rankSize, localResult) && localResult == 0) { + return 13; + } + if (localResult != 0 || !ShouldCheckInactiveP2pCcuCopyRank()) { + return localResult; + } + const int checkRet = CheckP2pCcuCopyState(state); + PrintP2pCcuCopy(rank, peer, *state, TileXR::TILEXR_SUCCESS, TileXR::TILEXR_SUCCESS, ACL_SUCCESS); + return checkRet == ACL_SUCCESS ? 0 : 14; +} + int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); const int peer = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; const bool p2pCcuCopyEnabled = EnvFlag(kP2pCcuCopyEnv); + const bool p2pCcuCopyActiveRank = !p2pCcuCopyEnabled || IsP2pCcuCopyActiveRank(rank); + const TileXR::TileXRCcuMemoryCopyDirection p2pCcuCopyDirection = P2pCcuCopyDirectionFromEnv(); P2pCcuCopyState p2pCcuCopy; if (p2pCcuCopyEnabled) { options.syncResourceCount = 1; @@ -1104,7 +1167,7 @@ int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi reinterpret_cast(p2pCcuCopy.destination.ptr), p2pCcuCopy.bytes, static_cast(peer), - TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, + p2pCcuCopyDirection, prepared, &installReport) : context->planner.PrepareDirectCcuInstallAttempt(context->session, options, prepared, &installReport); @@ -1129,6 +1192,8 @@ int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi } else if (submitRequested && !installReport.submitReady) { std::cout << "tilexr_ccu_direct_smoke submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested && !p2pCcuCopyActiveRank) { + finalRet = RunInactiveP2pCcuCopyRank(rank, peer, rankSize, &p2pCcuCopy, finalRet); } else if (submitRequested) { aclrtStream stream = nullptr; int streamRet = aclrtCreateStream(&stream); diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 5afcac6b..30df58e3 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -37,6 +37,7 @@ endpoint_fields=( ) resource_window_token_fields=( + EID_INDEX TOKEN_ID RAW_TOKEN_ID TOKEN_VALUE @@ -577,6 +578,13 @@ for log in "${rank0_log}" "${rank1_log}"; do fi done +rank_skipped_p2p_ccu_copy_submit() +{ + local log="$1" + [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" = "1" ] && + grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped" "${log}" +} + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "submitReady=1" "${log}"; then @@ -586,10 +594,16 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then done for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_direct_smoke submit ret=0" "${log}"; then + if rank_skipped_p2p_ccu_copy_submit "${log}"; then + continue + fi echo "ERROR: direct CCU submit did not return success in ${log}" >&2 exit 7 fi if ! grep -q "tilexr_ccu_direct_smoke submitTiming" "${log}"; then + if rank_skipped_p2p_ccu_copy_submit "${log}"; then + continue + fi echo "ERROR: direct CCU submit timing was not reported in ${log}" >&2 exit 8 fi diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 1985c58b..c999cdcb 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -125,6 +125,34 @@ def test_split_sources_own_restored_direct_ccu_runtime_glue(self): with self.subTest(fake_ready=fake_ready): self.assertNotIn(fake_ready, source + "\n" + runtime + "\n" + planner + "\n" + executor) + def test_tilexr_comm_can_auto_initialize_ccu_backend_without_blocking_comm_init(self): + source = (REPO_ROOT / "src" / "comm" / "tilexr_comm.cpp").read_text(encoding="utf-8") + + self.assertIn('constexpr const char* TILEXR_ENABLE_CCU_BACKEND_ENV = "TILEXR_ENABLE_CCU_BACKEND"', source) + self.assertIn("bool ShouldEnableCcuBackend()", source) + self.assertIn("int TileXRComm::InitCcuBackendIfEnabled()", source) + self.assertIn("const int ccuRet = InitCcuBackend()", source) + self.assertIn("TileXR CCU backend init failed, direct CCU disabled", source) + self.assertIn("TileXR CCU backend initialized", source) + + process_init = source[source.index("int TileXRComm::Init()"): source.index("int TileXRComm::InitThread")] + thread_init = source[source.index("int TileXRComm::InitThread"): source.index("int TileXRComm::EnablePeerAccess")] + + for body_name, body in [("process", process_init), ("thread", thread_init)]: + with self.subTest(body=body_name): + self.assertIn("ret = InitCcuBackendIfEnabled();", body) + self.assertLess(body.index("ret = InitCcuBackendIfEnabled();"), body.index("ret = SyncCommArgs();")) + self.assertIn("if (ret != TILEXR_SUCCESS) {", body) + self.assertIn("return ret;", body) + + helper = source[ + source.index("int TileXRComm::InitCcuBackendIfEnabled()"): + source.index("TileXRCcuBackend *TileXRComm::GetCcuBackendForCollectives") + ] + self.assertIn("if (!ShouldEnableCcuBackend())", helper) + self.assertIn("return TILEXR_SUCCESS;", helper) + self.assertNotIn("return ccuRet;", helper) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index d45dc384..e87f452f 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1719,6 +1719,18 @@ def test_direct_install_options_default_to_lower_layer_first(self): ) self.assertIn("installRequest.installOrder = options.installOrder", source) + def test_direct_memory_copy_submit_tasks_keep_planned_runtime_payload(self): + source = DIRECT_SOURCE.read_text(encoding="utf-8") + memory_copy_body = source[ + source.index("int BuildDirectMemoryCopyLaunchPackage"): + source.index("void FillReportFromAttempt") + ] + + self.assertIn("TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport)", memory_copy_body) + self.assertIn("attempt->package.tasks = tasks", memory_copy_body) + self.assertNotIn("NormalizeDirectMemoryCopySubmitTasks", source) + self.assertNotIn("task.argSize = 1", source) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 7c8279bc..1abc100e 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -242,10 +242,59 @@ def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): self.assertIn("RunP2pCcuCopy", source) self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION", source) + self.assertIn("P2pCcuCopyDirectionFromEnv", source) + self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) self.assertIn("p2pCcuCopy", source) + def test_local_to_remote_p2p_ccu_copy_inactive_rank_checks_destination(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("ShouldCheckInactiveP2pCcuCopyRank", source) + self.assertIn("RunInactiveP2pCcuCopyRank", source) + self.assertIn('"local_to_remote"', source) + self.assertIn("CheckP2pCcuCopyState", source) + inactive_branch = prepared_body[ + prepared_body.index("!p2pCcuCopyActiveRank"): + prepared_body.index("} else if (submitRequested) {", prepared_body.index("!p2pCcuCopyActiveRank")) + ] + self.assertIn("RunInactiveP2pCcuCopyRank", inactive_branch) + self.assertIn("CheckP2pCcuCopyState", source[source.index("RunInactiveP2pCcuCopyRank"):]) + self.assertNotIn("TileXRDirectCcuSubmitPrepared", inactive_branch) + + def test_p2p_ccu_copy_submit_is_limited_to_active_rank(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK", source) + self.assertIn("IsP2pCcuCopyActiveRank", source) + self.assertIn("p2pCcuCopyActiveRank", prepared_body) + self.assertIn("!p2pCcuCopyActiveRank", prepared_body) + self.assertIn("PrintP2pCcuCopySkipped", source) + self.assertIn("inactive p2p CCU-copy rank", source) + self.assertIn("WaitForInactiveP2pCcuCopyRank", source) + inactive_branch = prepared_body[ + prepared_body.index("!p2pCcuCopyActiveRank"): + prepared_body.index("} else if (submitRequested) {", prepared_body.index("!p2pCcuCopyActiveRank")) + ] + inactive_helper = source[ + source.index("int RunInactiveP2pCcuCopyRank"): + source.index("int RunPreparedSmokeForRank") + ] + self.assertIn("RunInactiveP2pCcuCopyRank", inactive_branch) + self.assertIn("PrintP2pCcuCopySkipped", inactive_helper) + self.assertIn("WaitForInactiveP2pCcuCopyRank", inactive_helper) + self.assertNotIn("TileXRDirectCcuSubmitPrepared", inactive_branch) + def test_thread_mode_path_uses_single_process_init_and_never_rank_ipc_init(self): source = PROBE_SOURCE.read_text(encoding="utf-8") diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 85c5be50..6a0a98f8 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -146,6 +146,19 @@ def test_runner_p2p_mode_applies_direct_ccu_resource_defaults(self): self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) self.assertIn("direct CCU P2P CCU-copy produced no passing receiver result", source) + def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): + source = RUNNER.read_text(encoding="utf-8") + submit_check = source[ + source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]', source.index('if [ "${rank0_status}"')): + source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]') + ] + + self.assertIn("rank_skipped_p2p_ccu_copy_submit", source) + self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) + self.assertIn("rank_skipped_p2p_ccu_copy_submit", submit_check) + self.assertIn("continue", submit_check) + self.assertIn("direct CCU submit did not return success", submit_check) + def test_runner_default_run_skips_without_hardware(self): with tempfile.TemporaryDirectory() as temp_dir: env = os.environ.copy() @@ -230,6 +243,15 @@ def test_runner_dry_run_shows_rank_specific_resource_window_token_overrides_comm self.assertNotIn("npu-smi rc=", result.stdout + result.stderr) self.assertNotIn("ccu_rank0.log", result.stdout) + def test_runner_passes_rank_specific_resource_window_eid_index(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("EID_INDEX", source[source.index("resource_window_token_fields=("):]) + self.assertIn('rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0"', source) + self.assertIn('rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1"', source) + self.assertIn('echo "dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}"', source) + self.assertIn('echo "dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}"', source) + def test_runner_dry_run_shows_repository_install_diagnostic_variants(self): with tempfile.TemporaryDirectory() as temp_dir: env = os.environ.copy() diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 41049bf5..70e652ed 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -2393,6 +2393,147 @@ def test_direct_ccu_runtime_can_register_resource_window_with_public_ra_ctx(self self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_direct_runtime_source_supports_selecting_ra_ctx_resource_window_eid(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX", source) + self.assertIn("SelectRaCtxResourceWindowEidInfo", source) + self.assertIn("SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV", source) + self.assertIn("TraceRaCtxEidInfos", source) + self.assertIn("ctxAttr.ub.eidIndex = selectedEid.eidIndex", source) + + def test_direct_ccu_runtime_can_select_ra_ctx_resource_window_eid_by_env(self): + code = textwrap.dedent( + r''' + #include + #include + #include + + #define private public + #include "ccu/tilexr_ccu_direct_runtime.h" + #undef private + + using namespace TileXR; + constexpr uint64_t kResourceAddr = 0x10001234ULL; + constexpr uint64_t kAlignedResourceAddr = 0x10001000ULL; + constexpr uint64_t kAlignedResourceBytes = + TILEXR_CCU_RESOURCE_WINDOW_BYTES + (kResourceAddr - kAlignedResourceAddr); + + int FakeRaCustomChannel(TileXRCcuRaInfo, TileXRCcuCustomChannelIn*, TileXRCcuCustomChannelOut*) + { + return 0; + } + + int FakeRaGetDevEidInfoNum(TileXRCcuRaInfo, uint32_t* num) + { + *num = 2; + return 0; + } + + int FakeRaGetDevEidInfoList(TileXRCcuRaInfo, TileXRCcuHccpDevEidInfo* list, uint32_t* num) + { + if (list == nullptr || num == nullptr || *num != 2) { + return -1; + } + list[0].eidIndex = 3; + list[1].eidIndex = 9; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + list[0].eid.raw[i] = static_cast(0xa0 + i); + list[1].eid.raw[i] = static_cast(0xc0 + i); + } + return 0; + } + + int FakeRaCtxInit(TileXRCcuHccpCtxInitCfg*, TileXRCcuHccpCtxInitAttr* attr, void** ctx) + { + if (attr == nullptr || attr->ub.eidIndex != 9 || attr->ub.eid.raw[0] != 0xc0) { + return -1; + } + *ctx = reinterpret_cast(0x1000); + return 0; + } + + int FakeRaCtxDeinit(void*) { return 0; } + int FakeRaCtxTokenIdAlloc(void*, TileXRCcuHccpTokenId* token, void** tokenHandle) + { + token->tokenId = 0x12345600U; + *tokenHandle = reinterpret_cast(0x2000); + return 0; + } + int FakeRaCtxTokenIdFree(void*, void*) { return 0; } + int FakeRaGetSecRandom(TileXRCcuRaInfo*, uint32_t* value) + { + *value = 0xabcdef01U; + return 0; + } + int FakeRaCtxLmemRegister(void*, TileXRCcuHccpMrRegInfo* mr, void** handle) + { + if (mr == nullptr || mr->in.mem.addr != kAlignedResourceAddr || + mr->in.mem.size != kAlignedResourceBytes) { + return -1; + } + mr->out.ub.tokenId = 0x12345600U; + mr->out.ub.targetSegHandle = 0x4455667788ULL; + *handle = reinterpret_cast(0x3000); + return 0; + } + int FakeRaCtxLmemUnregister(void*, void*) { return 0; } + + int main() + { + TileXRCcuDirectRuntime runtime; + runtime.initialized_ = true; + runtime.loader_.RaCustomChannel = FakeRaCustomChannel; + runtime.loader_.loaded_ = true; + runtime.loader_.RaGetDevEidInfoNum = FakeRaGetDevEidInfoNum; + runtime.loader_.RaGetDevEidInfoList = FakeRaGetDevEidInfoList; + runtime.loader_.RaCtxInit = FakeRaCtxInit; + runtime.loader_.RaCtxDeinit = FakeRaCtxDeinit; + runtime.loader_.RaCtxTokenIdAlloc = FakeRaCtxTokenIdAlloc; + runtime.loader_.RaCtxTokenIdFree = FakeRaCtxTokenIdFree; + runtime.loader_.RaGetSecRandom = FakeRaGetSecRandom; + runtime.loader_.RaCtxLmemRegister = FakeRaCtxLmemRegister; + runtime.loader_.RaCtxLmemUnregister = FakeRaCtxLmemUnregister; + runtime.devicePhyId_ = 0x55; + runtime.options_.rank = 1; + runtime.cachedBasicInfoValid_ = true; + runtime.cachedBasicInfo_.resourceAddr = kResourceAddr; + runtime.cachedBasicInfo_.caps.cap1 = 7U << 16U; + + if (runtime.RegisterCcuResourceRmaBuffer(kResourceAddr) != TILEXR_SUCCESS) { + std::cerr << "ra ctx resource window register failed\n"; + return 1; + } + TileXRCcuLocalResourceWindowInfo local; + if (runtime.ExportLocalCcuRmaBuffer(&local) != TILEXR_SUCCESS || + local.eidIndex != 9 || local.eid[0] != 0xc0) { + std::cerr << "wrong eid selection index=" << local.eidIndex + << " first=" << static_cast(local.eid[0]) << "\n"; + return 2; + } + runtime.Shutdown(); + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"] = "ra_ctx" + env["TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1"] = "9" + + result = self.compile_and_run( + code, + env=env, + extra_sources=[ + DIRECT_RUNTIME_SOURCE, + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_hccp_loader.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_ra_custom_channel_provider.cpp", + REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp", + ], + extra_link_flags=["-ldl"]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_direct_ccu_runtime_collects_ra_ctx_endpoint_route_when_resource_window_uses_ra_ctx(self): code = textwrap.dedent( r''' @@ -3684,11 +3825,11 @@ def test_tilexr_comm_prepares_direct_ccu_install_attempt_without_submitting(self ]: self.assertNotIn(forbidden, combined) - def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(self): + def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_device_init_failure(self): runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") - self.assertIn("g_ccuDirectRuntimeUnavailableMessage", runtime_session_source) + self.assertIn("g_ccuDirectRuntimeUnavailableByDevice", runtime_session_source) init_body = runtime_session_source[ runtime_session_source.index("int TileXRCcuRuntimeSession::Init("): runtime_session_source.index("void TileXRCcuRuntimeSession::ResetDirectCcuBasicInfo") @@ -3698,11 +3839,11 @@ def test_tilexr_comm_direct_ccu_prepare_fails_fast_after_process_init_failure(se planner_source.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt") ] - self.assertIn("g_ccuDirectRuntimeUnavailableMessage = runtimeReport.message", init_body) - self.assertIn("direct CCU runtime unavailable after process-level init failure", runtime_session_source) - self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", prepare_body) + self.assertIn("g_ccuDirectRuntimeUnavailableByDevice[devId_] = runtimeReport.message", init_body) + self.assertIn("direct CCU runtime unavailable for device", runtime_session_source) + self.assertIn("session.DirectCcuRuntimeUnavailableMessage()", prepare_body) self.assertLess( - prepare_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), + prepare_body.index("session.DirectCcuRuntimeUnavailableMessage()"), prepare_body.index("RefreshDirectCcuBasicInfo(installDieId)"), ) @@ -3716,7 +3857,7 @@ def test_tilexr_comm_direct_ccu_runtime_init_serializes_ra_initialization(self): lock_pos = init_body.index("lock_guard lock(g_ccuDirectRuntimeMtx);") allocation_pos = init_body.index("ccuDirectRuntime_.reset(new (nothrow) TileXRCcuDirectRuntime())") runtime_init_pos = init_body.index("ccuDirectRuntime_->Init(runtimeOptions, &runtimeReport)") - unavailable_set_pos = init_body.index("g_ccuDirectRuntimeUnavailable = true") + unavailable_set_pos = init_body.index("g_ccuDirectRuntimeUnavailableByDevice[devId_] = runtimeReport.message") self.assertLess(lock_pos, allocation_pos) self.assertLess(allocation_pos, runtime_init_pos) @@ -3736,17 +3877,17 @@ def test_tilexr_comm_direct_ccu_prepare_can_select_install_die_for_diagnostics(s self.assertIn("basicInfo->dieId != installDieId", prepare_body) self.assertNotIn("RefreshDirectCcuBasicInfo(0)", prepare_body) - def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_process_init_failure(self): + def test_tilexr_comm_direct_ccu_thread_allgather_aborts_after_device_init_failure(self): runtime_session_source = CCU_RUNTIME_SESSION_SOURCE.read_text(encoding="utf-8") thread_allgather_body = runtime_session_source[ runtime_session_source.index("int TileXRCcuRuntimeSession::DirectCcuThreadAllGather"): runtime_session_source.index("} // namespace TileXR") ] - self.assertIn("ProcessDirectCcuRuntimeUnavailableMessage()", thread_allgather_body) + self.assertIn("DirectCcuRuntimeUnavailableMessage()", thread_allgather_body) self.assertIn("direct CCU thread allgather abort", thread_allgather_body) self.assertLess( - thread_allgather_body.index("ProcessDirectCcuRuntimeUnavailableMessage()"), + thread_allgather_body.index("DirectCcuRuntimeUnavailableMessage()"), thread_allgather_body.index("TILEXR_INIT_TIMEOUT"), ) diff --git a/tests/ccu/test_tilexr_ccu_resource_allocator.py b/tests/ccu/test_tilexr_ccu_resource_allocator.py index 17c105f4..b9f2ca28 100644 --- a/tests/ccu/test_tilexr_ccu_resource_allocator.py +++ b/tests/ccu/test_tilexr_ccu_resource_allocator.py @@ -751,6 +751,103 @@ def test_allocator_rejects_resource_exhaustion_and_double_release(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_allocator_release_reclaims_latest_receipt_and_rejects_non_lifo_release(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_resource_allocator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuResourceSpec spec; + spec.dieId = 0; + spec.missionKey = 0x12345678U; + spec.missionStartId = 10; + spec.missionCount = 4; + spec.instructionStartId = 100; + spec.instructionCount = 64; + spec.xnStartId = 200; + spec.xnCount = 16; + spec.ckeStartId = 300; + spec.ckeCount = 8; + spec.channelStartId = 20; + spec.channelCount = 4; + + TileXRCcuResourceRequest request; + request.sqeArgCount = 0; + request.syncResourceCount = 1; + request.syncInstructionCount = 2; + request.bindingsPerSyncResource = 1; + + TileXRCcuResourceAllocator allocator; + if (allocator.Init(spec) != TILEXR_SUCCESS) { + std::cerr << "init failed\n"; + return 1; + } + + TileXRCcuProducerPlan firstPlan; + TileXRCcuProducerPlan secondPlan; + TileXRCcuProducerPlan thirdPlan; + TileXRCcuResourceAllocation first; + TileXRCcuResourceAllocation second; + TileXRCcuResourceAllocation third; + TileXRCcuResourceAllocatorReport report; + if (allocator.Allocate(request, &firstPlan, &first, &report) != TILEXR_SUCCESS || + allocator.Allocate(request, &secondPlan, &second, &report) != TILEXR_SUCCESS) { + std::cerr << "initial allocate failed: " << report.message << "\n"; + return 2; + } + if (first.mission.startId != 10 || second.mission.startId != 11 || + first.repository.startId != 100 || second.repository.startId != 102 || + first.localXn.startId == second.localXn.startId || + first.channels.startId == second.channels.startId) { + std::cerr << "initial allocations did not advance resource cursors\n"; + return 3; + } + if (allocator.Release(first.receiptId) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "non-LIFO release was accepted\n"; + return 4; + } + if (allocator.Release(second.receiptId) != TILEXR_SUCCESS) { + std::cerr << "latest release failed\n"; + return 5; + } + if (allocator.Allocate(request, &thirdPlan, &third, &report) != TILEXR_SUCCESS) { + std::cerr << "third allocate failed after release: " << report.message << "\n"; + return 6; + } + if (third.mission.startId != second.mission.startId || + third.repository.startId != second.repository.startId || + third.localXn.startId != second.localXn.startId || + third.remoteXn.startId != second.remoteXn.startId || + third.localWaitCke.startId != second.localWaitCke.startId || + third.remoteNotifyCke.startId != second.remoteNotifyCke.startId || + third.channels.startId != second.channels.startId) { + std::cerr << "released resource ranges were not reused\n"; + return 7; + } + if (allocator.Release(first.receiptId) != TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "old non-tail receipt release should still be rejected\n"; + return 8; + } + if (allocator.Release(third.receiptId) != TILEXR_SUCCESS || + allocator.Release(first.receiptId) != TILEXR_SUCCESS) { + std::cerr << "tail releases failed\n"; + return 9; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_allocator_rejects_sync_instruction_window_too_small_for_barrier_program(self): code = textwrap.dedent( r''' From 8bc12071fac68d60293e7728ecbd698a9100866f Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Fri, 10 Jul 2026 15:06:32 +0800 Subject: [PATCH 18/41] docs: add ccu signal wait design --- .../2026-07-10-ccu-signal-wait-design.md | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md diff --git a/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md b/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md new file mode 100644 index 00000000..9d47d48c --- /dev/null +++ b/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md @@ -0,0 +1,217 @@ +# CCU Signal/Wait and Barrier Design + +## Goal + +Implement a two-rank synchronization capability over TileXR Direct CCU and wire it into the internal `TileXRComm` CCU backend. The work is intentionally internal first: no public C API is added until the runtime semantics, resource ownership, and hardware smoke are stable. + +The implementation order is: + +1. Single-direction signal/wait between two ranks. +2. Two-direction barrier built from the same signal/wait machinery. + +## Scope + +In scope: + +- Add internal `TileXRCcuBackend` capability for two-rank signal/wait. +- Reuse the existing Direct CCU runtime lifecycle, resource-window registration, peer allgather, lower-layer install, repository install, mission install, and `rtCCULaunch` submit path. +- Add smoke coverage for `rank0 -> rank1`, `rank1 -> rank0`, and two-rank barrier. +- Keep the feature behind internal/test-only entry points until validated. + +Out of scope: + +- Public C API such as `TileXRCommSignal`, `TileXRCommWait`, or `TileXRCommBarrier`. +- N-rank barrier. +- Alltoall or general collective backend dispatch. +- Host marker based success criteria. + +## Existing Building Blocks + +The design reuses these current modules: + +- `TileXRCcuBackend`: internal backend owned by `TileXRComm`. +- `TileXRCcuRuntimeSession`: rank/device state, Direct CCU runtime availability, socket/thread allgather. +- `TileXRCcuDirectRuntime`: HCCP/RA/runtime loading, RA ctx resource window, endpoint route collection, peer buffer export. +- `TileXRCcuCollectivePlanner`: lower-layer template generation and direct install attempt preparation. +- `TileXRCcuResourceAllocator`: mission, repository, XN, GSA, CKE, and channel allocation. +- `TileXRCcuBuildBarrierProgram`: existing CCU microcode builder for post/wait style synchronization. +- `TileXRCcuSubmitPreparedTasks`: prepared task submission through `rtCCULaunch`. + +## Internal API + +Add internal request and plan types under `src/comm/ccu`: + +```cpp +enum class TileXRCcuSignalWaitRole { + Signal, + Wait, + SignalAndWait, +}; + +struct TileXRCcuSignalWaitRequest { + int peerRank = -1; + TileXRCcuSignalWaitRole role = TileXRCcuSignalWaitRole::Signal; + uint32_t timeout = 0; +}; + +struct TileXRCcuSignalWaitPlan { + bool ready = false; + TileXRCcuDirectInstallAttempt attempt; + std::vector submitTasks; +}; +``` + +Add internal backend methods: + +```cpp +int PrepareSignalWait( + const TileXRCcuSignalWaitRequest& request, + TileXRCcuSignalWaitPlan* plan); + +int SubmitSignalWait( + const TileXRCcuSignalWaitPlan& plan, + aclrtStream stream, + TileXRCcuDirectSubmitReport* report); +``` + +These methods are C++ internal only. They are not declared in `src/include/tilexr_api.h`. + +## Signal/Wait Semantics + +For two ranks, one rank is the signaler and the other rank is the waiter. + +Signal rank: + +- Installs a CCU task that posts to the peer rank's notify/wait CKE through the lower-layer channel. +- The task should be post-only and should not wait for the peer. +- `aclrtSynchronizeStream()` returning on the signal rank only proves the signal task has been submitted and completed locally. + +Wait rank: + +- Installs a CCU task that waits on its local wait CKE. +- The task should not complete until the peer signal arrives. +- `aclrtSynchronizeStream()` returning on the wait rank is the synchronization proof. + +The smoke test must verify this by delaying the signal rank and checking that the wait rank's stream synchronize time exceeds the configured threshold. + +## Barrier Semantics + +The two-rank barrier is built from two opposing signal/wait operations: + +- rank0 signals rank1 and waits for rank1. +- rank1 signals rank0 and waits for rank0. + +The first implementation should prefer one CCU task per rank containing both post and wait instructions. This avoids a host scheduling gap between separate signal and wait submissions and keeps the barrier semantics close to the device timeline. + +If the one-task form exposes hardware ordering issues, the fallback is two prepared tasks per rank: post first, wait second. The fallback must remain internal and be selected only for diagnostics or if hardware behavior requires it. + +## Resource Flow + +The resource flow matches the current Direct CCU install path: + +1. Refresh Direct CCU basic info for the selected die. +2. Decode resource spec from basic info. +3. Allocate mission, repository instruction, XN, CKE, and channel resources. +4. Register the local CCU resource window through RA ctx. +5. Export local resource-window token and endpoint route. +6. Allgather peer resource-window and endpoint data. +7. Exchange peer XN/CKE/channel ownership proof. +8. Build lower-layer transport snapshot and install plan. +9. Build a CCU synchronization program from the selected role. +10. Build repository image and launch package. +11. Install repository, lower-layer resources, and mission/key. +12. Generate submit tasks only after install evidence matches the launch package. + +No env override should mutate prepared task fields, peer binding proof, mission key, or instruction ranges. + +## Microcode Plan + +Signal uses a post-only synchronization instruction. The preferred instruction mode is `SyncCkePostOnly` when the lower-layer proof contains the CKE resources needed for peer notification. `SyncXnPostOnly` can be retained as a diagnostic fallback only if the CKE path is not viable on hardware. + +Wait uses the existing local CKE wait encoding. If current `TileXRCcuBuildBarrierProgram` cannot express a pure remote-triggered wait cleanly, add a small dedicated builder such as: + +```cpp +int TileXRCcuBuildSignalWaitProgram( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report); +``` + +That builder should still call the existing low-level microcode encoders and should avoid duplicating instruction encoding logic. + +## Error Handling + +Preparation returns explicit errors for: + +- Direct CCU runtime unavailable. +- Rank size not equal to 2. +- Invalid peer rank. +- Missing Direct CCU basic info. +- Resource allocation failure. +- Resource-window registration failure. +- Peer route/token exchange failure. +- Lower-layer install-plan failure. +- Repository/mission install failure. +- Install evidence mismatch. + +Submission returns explicit errors for: + +- Empty or not-ready plan. +- Null stream. +- Runtime launch failure. +- Mid-batch submit failure. + +Reports should include enough task detail for diagnosis: mission id, key, instruction range, argument size, submitted task count, and runtime return code when available. + +## Testing + +Unit tests: + +- Signal/wait request validation. +- Signal-only program generation. +- Wait-only program generation. +- Signal-and-wait program generation. +- Resource allocation shape for one peer route. +- Submit path rejects null stream and empty task list. +- Backend boundary tests verify the feature remains internal and does not appear in public headers. + +Smoke tests: + +- `rank0 signal -> rank1 wait`. +- `rank1 signal -> rank0 wait`. +- Two-rank barrier. +- Delayed signal rank proves wait rank blocks on CCU completion. +- Timeout wrapping remains enabled around the whole runner. + +Expected smoke evidence: + +```text +tilexr_ccu_signal_wait prepare ret=0 ... installSucceeded=1 ... submitReady=1 +tilexr_ccu_signal_wait submit ret=0 ... submitted=1 +tilexr_ccu_signal_wait timing rank= syncMs= +tilexr_ccu_signal_wait result passed=1 +``` + +For barrier: + +```text +tilexr_ccu_barrier prepare ret=0 ... installSucceeded=1 ... submitReady=1 +tilexr_ccu_barrier submit ret=0 ... submitted=1 +tilexr_ccu_barrier result passed=1 +``` + +## Validation Gates + +Before considering the feature complete: + +1. CCU unit tests pass. +2. `tile-comm` builds on the NPU server. +3. No hcomm/HCCL private CCU dependency is introduced. +4. Two-card signal/wait smoke passes in both directions. +5. Two-card barrier smoke passes. +6. Delayed waiter/signal timing proves device-side synchronization rather than host-side gating. + +## Open Decisions + +No open product decisions remain for the first implementation. Public API shape and N-rank barrier are intentionally deferred until after the internal backend path is validated. + From 83f937cd9b684074e2322a3df64c37262c7ea268 Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Mon, 13 Jul 2026 20:25:58 +0800 Subject: [PATCH 19/41] feat(ccu): add direct signal wait backend Add direct CCU signal/wait program generation and wire it through the internal backend, smoke probe, and runner paths. Harden direct CCU runtime/session lifecycle, lower-layer ownership tracking, driver mission status helpers, and endpoint route handling for repeated launches. Align P2P CCU copy with hcomm address semantics by using original device VAs in microcode while routing lower-layer channels through imported segment VAs, with regression coverage and documentation. --- docs/CCU_DIRECT_RUNTIME.md | 504 ++++++++++++++++++ ...26-07-10-ccu-signal-wait-implementation.md | 126 +++++ src/comm/CMakeLists.txt | 2 + src/comm/ccu/tilexr_ccu_backend.cpp | 58 ++ src/comm/ccu/tilexr_ccu_backend.h | 39 ++ src/comm/ccu/tilexr_ccu_barrier_program.cpp | 2 +- .../ccu/tilexr_ccu_collective_planner.cpp | 399 +++++++++++++- src/comm/ccu/tilexr_ccu_collective_planner.h | 20 + .../ccu/tilexr_ccu_direct_orchestrator.cpp | 184 ++++++- src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 12 + src/comm/ccu/tilexr_ccu_direct_runtime.cpp | 202 +++++++ src/comm/ccu/tilexr_ccu_direct_runtime.h | 48 ++ src/comm/ccu/tilexr_ccu_driver_adapter.cpp | 34 ++ src/comm/ccu/tilexr_ccu_driver_adapter.h | 10 + src/comm/ccu/tilexr_ccu_hccp_loader.cpp | 4 + src/comm/ccu/tilexr_ccu_hccp_loader.h | 2 + src/comm/ccu/tilexr_ccu_hccp_types.h | 41 ++ .../tilexr_ccu_lower_layer_plan_builder.cpp | 20 +- .../ccu/tilexr_ccu_lower_layer_plan_builder.h | 3 + src/comm/ccu/tilexr_ccu_runtime_session.cpp | 26 + src/comm/ccu/tilexr_ccu_runtime_session.h | 5 + .../ccu/tilexr_ccu_signal_wait_program.cpp | 215 ++++++++ src/comm/ccu/tilexr_ccu_signal_wait_program.h | 41 ++ tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 359 ++++++++++++- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 188 ++++++- tests/ccu/test_tilexr_ccu_backend_boundary.py | 70 +++ tests/ccu/test_tilexr_ccu_barrier_program.py | 6 +- .../test_tilexr_ccu_direct_orchestrator.py | 32 ++ .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 48 ++ .../test_tilexr_ccu_direct_smoke_runner.py | 21 + tests/ccu/test_tilexr_ccu_driver_adapter.py | 4 + ...est_tilexr_ccu_lower_layer_plan_builder.py | 69 +++ ...est_tilexr_ccu_ra_custom_channel_loader.py | 11 + .../test_tilexr_ccu_signal_wait_program.py | 262 +++++++++ 34 files changed, 3008 insertions(+), 59 deletions(-) create mode 100644 docs/CCU_DIRECT_RUNTIME.md create mode 100644 docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md create mode 100644 src/comm/ccu/tilexr_ccu_signal_wait_program.cpp create mode 100644 src/comm/ccu/tilexr_ccu_signal_wait_program.h create mode 100644 tests/ccu/test_tilexr_ccu_signal_wait_program.py diff --git a/docs/CCU_DIRECT_RUNTIME.md b/docs/CCU_DIRECT_RUNTIME.md new file mode 100644 index 00000000..910c0609 --- /dev/null +++ b/docs/CCU_DIRECT_RUNTIME.md @@ -0,0 +1,504 @@ +# TileXR Direct CCU 技术说明 + +本文记录 `direct-ccu-rebased` 分支中 TileXR 自研 Direct CCU 路径的技术原理、运行流程、执行方法和测试方法。它面向继续开发和问题定位，不作为外部稳定 API 文档。 + +## 1. 当前边界 + +Direct CCU 当前属于 `src/comm/ccu` 下的内部后端能力，由 `TileXRComm` 按需持有 `TileXRCcuBackend`。它的目标是在不链接 hcomm/HCCL 私有 CCU producer 的前提下，由 TileXR host 侧完成 CCU repository、mission、lower-layer 资源和 submit task 的准备，并最终通过 CANN runtime 的 `rtCCULaunch` 下发任务。 + +当前需要明确的边界如下： + +- 生产代码不得链接或包含 hcomm/HCCL 私有 CCU producer 接口。 +- 对外安装头文件当前不暴露 `TileXRDirectCcu*`、`PrepareDirectCcu`、`SubmitPrepared` 等 Direct CCU C API。 +- `TileXRCcuCollectivePlanner::Supports()` 目前仍返回 `false`，通用 collective/alltoall 尚未接入为正式 TileXR collective 后端。 +- 已维护的数据面验证路径是 direct CCU P2P copy probe，基于 CCU memory-copy microcode，不走 alltoall。 +- 默认非 P2P barrier/smoke 只能作为安装和提交链路诊断，不能替代 P2P copy 的数据正确性证明。 + +## 2. 总体架构 + +核心对象关系： + +```text +TileXRComm + | + +-- TileXRCcuBackend + | + +-- TileXRCcuRuntimeSession + | | + | +-- TileXRCcuDirectRuntime + | + +-- TileXRCcuCollectivePlanner + | + +-- TileXRCcuExecutor +``` + +主要模块职责： + +- `tilexr_ccu_backend.*`：Direct CCU 后端门面，封装 runtime session、planner 和 executor。 +- `tilexr_ccu_runtime_session.*`：管理 rank、rankSize、device、socket/thread allgather 和 direct runtime 可用状态。 +- `tilexr_ccu_direct_runtime.*`：动态加载 HCCP/RA/runtime 符号，初始化底层 RA/HDC/CCU TLV，注册 resource window，导出本端和远端 transport 信息。 +- `tilexr_ccu_resource_allocator.*`：按 driver basic info 解出的资源窗口分配 mission、repository instruction、local/remote XN、GSA、CKE、channel 等资源。 +- `tilexr_ccu_collective_planner.*`：把 runtime session、资源分配、lower-layer plan、repository 安装和 prepared submit task 串起来。 +- `tilexr_ccu_direct_orchestrator.*`：执行完整 direct install attempt，包括资源规格解码、资源分配、lower-layer plan、launch package、manifest、hardware install 和 submit task 生成。 +- `tilexr_ccu_install_provider.*`：定义硬件安装需求、证据和校验，执行 repository/lower-layer/mission 等安装步骤。 +- `tilexr_ccu_repository.*`：构造 CCU instruction repository image，并负责把 repository image 安装到设备侧。 +- `tilexr_ccu_barrier_program.*`、`tilexr_ccu_memory_program.*`、`tilexr_ccu_microcode.*`：生成 CCU barrier 和 memory-copy 指令。 +- `tilexr_ccu_runtime.*`：把 `TileXRCcuTask` 映射为 runtime task，并调用 `rtCCULaunch`。 + +## 3. 初始化与生命周期 + +`TileXRComm::Init()` 和 `TileXRComm::InitThread()` 结束前会调用 `InitCcuBackendIfEnabled()`。是否启用由环境变量控制： + +```bash +export TILEXR_ENABLE_CCU_BACKEND=1 +``` + +启用后流程如下： + +1. `TileXRComm::InitCcuBackend()` 创建 `TileXRCcuBackend`。 +2. `TileXRCcuBackend::Init()` 重置 planner，并调用 `TileXRCcuRuntimeSession::Init()`。 +3. `TileXRCcuRuntimeSession::Init()` 对单 rank communicator 直接跳过 direct runtime。 +4. 多 rank 时创建 `TileXRCcuDirectRuntime`，传入 rank、rankSize、device 和 allgather 回调。 +5. `TileXRCcuDirectRuntime::Init()` 动态加载底层符号，解析逻辑 device 到物理 device，初始化 RA/HDC 和 CCU TLV。 +6. runtime 初始化成功后刷新 direct CCU basic info，缓存 die、resource address、mission/resource 范围等基础信息。 + +失败策略是降级而不是让 communicator 初始化失败： + +- runtime 初始化失败会记录 warning，并将 direct CCU 后端关闭。 +- 失败状态按 `devId` 记录，避免一个 device 初始化失败污染同进程其他 device。 +- 后续同 device 再初始化会直接跳过，并保留明确的 unavailable message。 + +`TileXRComm::Destroy()` 会关闭 CCU backend。`TileXRCcuBackend::Shutdown()` 会先重置 planner，再关闭 runtime session；runtime session 会释放 direct runtime、basic info 缓存和 allgather 轮次状态。 + +## 4. Direct Runtime 原理 + +`TileXRCcuDirectRuntime` 是 TileXR 和底层 driver/RA/HCCP/runtime 交互的边界，承担四类工作。 + +### 4.1 符号与设备初始化 + +runtime 动态解析所需符号，不在 `tile-comm` 链接期引入 hcomm/HCCL 私有依赖。初始化阶段会： + +- 加载 HCCP/RA 相关入口。 +- 选择 direct CCU HDC 类型。 +- 解析 `logicDevId -> devicePhyId`。 +- 初始化 RA/HDC。 +- 初始化 CCU TLV。 +- 创建 driver adapter，用于 basic info 查询和后续 install。 + +如果任一关键步骤失败，runtime 返回不可用状态，并由 session 记录 device-scoped failure。 + +### 4.2 Resource Window 注册 + +CCU lower-layer/repository 安装需要一块可被 peer 识别的 resource window。当前推荐使用 RA ctx 模式： + +```bash +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=ra_ctx +``` + +RA ctx 模式的关键步骤： + +1. 通过 `RaGetDevEidInfoNum` / `RaGetDevEidInfoList` 获取可用 EID。 +2. 通过 `TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX` 或 rank 级变量选择 EID。 +3. 调用 `RaCtxInit` 创建 RA ctx。 +4. 通过 `RaCtxTokenIdAlloc` 分配 token id。 +5. 对 CCU resource address 做页对齐后调用 `RaCtxLmemRegister`。 +6. 导出 `addr/bytes/tokenId/rawTokenId/tokenValue/eid/raCtxHandle`。 + +常用 EID 选择变量： + +```bash +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX=3 +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 +``` + +### 4.3 Peer 信息交换 + +runtime session 为 direct runtime 提供 allgather： + +- 有 socket exchange 时使用 `TileXRSockExchange::AllGather()`。 +- thread mode 或测试模式下没有 socket exchange 时，使用进程内 thread allgather。 + +allgather 用于交换： + +- 本端 resource window token 和 EID。 +- endpoint route 信息。 +- remote XN/CKE/channel 绑定所需的 peer 资源窗口。 +- P2P copy 端点中的 source/destination address 和 process token。 + +thread allgather 带超时和失败中止逻辑，避免单 rank runtime init 失败后其他 rank 永久等待。 + +### 4.4 Endpoint Route 与 Remote Buffer + +`ExportRemoteCcuRmaBuffers()` 会把本端 resource window 信息 allgather 到所有 rank，并为每个 peer 生成 remote CCU buffer 信息。可用 RA ctx endpoint route 时，会进一步： + +- 查询本端到 peer EID 的 TP handle。 +- 交换 TP handle。 +- 导入 peer QP。 +- 使用导入得到的 TPN 和本端 doorbell token 构造 channel route。 + +这里存在一个重要细节：channel 中使用的 remote EID 采用 hcomm 兼容语义，导入后会使用反向 EID 表示。 + +## 5. 资源模型与所有权 + +Direct CCU 执行前需要把硬件资源划分为明确的窗口。资源来自 `TileXRCcuBasicInfo`，再由 `TileXRCcuDecodeBasicInfo()` 和 `TileXRCcuBuildResourceSpec()` 转换为 `TileXRCcuResourceSpec`。 + +主要资源： + +- `mission`：mission id 和 mission key。 +- `repository`：CCU instruction repository 中的指令槽。 +- `localXn`：本端 CCU XN 资源。 +- `localGsa`：本端 GSA 资源，P2P memory copy 用于装载地址、token 和长度。 +- `remoteXn`：绑定 peer 侧 XN 的本地表示。 +- `notifyCke`、`localWaitCke`、`remoteNotifyCke`：同步/完成通知使用的 CKE。 +- `channels`：指向 peer resource window/endpoint route 的 channel 绑定。 + +`TileXRCcuResourceAllocator` 使用 receipt 记录每次分配。当前释放策略是严格 LIFO，用来保证 cursor 回退不会和仍在使用的资源重叠。这个设计的目的不是做复杂资源池，而是在多次 prepare/launch 场景中先保证资源窗口所有权清晰、可诊断、可回收。 + +## 6. Lower-layer Install 原理 + +lower-layer install 负责把 CCU 执行需要的资源上下文安装到 driver 可识别的位置。主要输入来自： + +- basic info 解码出的资源窗口。 +- allocator 分配出的 mission/XN/CKE/channel range。 +- 本端 resource window token。 +- peer resource window token。 +- verified endpoint route。 + +准备流程： + +1. `PrepareDirectCcuLowerLayerTemplateFromAllocation()` 先注册本端 resource window。 +2. 导出本端 token 后，如果已经有 verified endpoint route 就配置到 runtime；否则尝试自动采集。 +3. `ExportRemoteCcuRmaBuffers()` 交换 peer token 和 endpoint 信息。 +4. `ExchangeDirectCcuRemoteNotifyCke()` 交换 peer 本地 XN/CKE/channel 分配结果。 +5. `TileXRCcuBuildLowerLayerTransportTemplate()` 构造 transport snapshot。 +6. `TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot()` 生成安装计划。 + +安装计划覆盖的 surface 包括 local XN、remote XN binding、notify CKE、channel binding、repository 和 mission/key。`tilexr_ccu_install_provider` 会为这些 surface 建立 manifest 和 install evidence。只有 install evidence 与 launch package fingerprint、device、rank、provider 和资源范围匹配时，后续 prepared submit task 才被认为可提交。 + +## 7. Repository、Mission 与 Launch Package + +Direct CCU 的 host 下发不是直接下发一段普通 kernel，而是下发 CCU mission 和 instruction repository。 + +完整 install attempt 的关键步骤在 `RunDirectInstallAttemptImpl()` 中： + +1. 校验 basic info、provider 和 repository install 输入。 +2. 解码 basic info，生成 resource spec。 +3. 分配 mission、repository、XN、GSA、CKE、channel 等资源。 +4. 如是 P2P memory copy，调整 local XN/GSA/remote XN 资源布局。 +5. 准备 lower-layer install plan。 +6. 将 producer plan 与 lower-layer proof 对齐，确保 sync resource 使用真实 peer 资源。 +7. 构造 CCU program 和 repository image。 +8. 绑定 launch package 的 device/rank/provider scope。 +9. 构造 install manifest。 +10. 执行 `TileXRCcuInstallHardware()`，安装 repository/lower-layer/mission。 +11. 根据 install evidence 生成 `submitTasks`。 + +`TileXRCcuTask` 是最终提交给 runtime 的任务描述，关键字段包括： + +- `dieId` +- `missionId` +- `key` +- `instStartId` +- `instCnt` +- `timeout` +- `argSize` +- `args[]` + +提交时 `TileXRCcuSubmitPreparedTasks()` 会逐个任务调用 `TileXRCcuSubmitTaskWithReport()`，后者在 `tilexr_ccu_runtime.cpp` 中映射到 `rtCCULaunch()`。如果中途某个 task 失败，submit report 会记录已提交数量和失败 task 的 mission/key/instruction/args 诊断信息。 + +## 8. P2P CCU Copy 执行原理 + +当前硬件数据面验证使用 direct CCU P2P copy，不走 alltoall。它验证的是 CCU 指令真实从 peer device memory 读写数据，而不是只验证 host marker 或 IPC 同步。 + +### 8.1 端点准备 + +每个 rank 分配两块 device buffer： + +- `source`：写入 rank 相关的数据 pattern。 +- `destination`：初始化为固定填充值。 + +然后通过 `rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, ...)` 查询 source/destination 的 process token，打包为 CCU memory token。所有 rank 通过 session allgather 交换端点： + +```text +rank -> {sourceAddr, sourceToken, destinationAddr, destinationToken, bytes} +``` + +### 8.2 方向语义 + +`TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION` 支持两种方向： + +- `remote_to_local`：active rank 从 peer source 拷贝到本 rank destination，active rank 校验本地 destination。 +- `local_to_remote`：active rank 从本 rank source 拷贝到 peer destination，inactive rank 等待 done gate 后校验本地 destination。 + +只有 active rank 真正 submit direct CCU task；inactive rank 会打印 `p2pCcuCopy skipped`，但仍通过 done gate 参与同步，必要时校验自己的 destination。 + +### 8.3 Memory-copy Microcode + +P2P copy 会构造一个 memory copy program，典型使用 7 条 CCU 指令： + +- 把 local/remote address、token、length 装入 GSA/XN。 +- 根据方向生成 `TransRmtMemToLocMem` 或 `TransLocMemToRmtMem`。 +- 使用 CKE 完成通知，host 侧随后 `aclrtSynchronizeStream()` 等待 stream 完成。 + +通过 `TILEXR_CCU_DIRECT_TRACE=1` 可在日志中看到解码后的 CCU 指令，例如： + +```text +decoded=TransRmtMemToLocMem +decoded=TransLocMemToRmtMem +``` + +## 9. 执行方法 + +### 9.1 构建 + +在 NPU 服务器上执行： + +```bash +cd /home/tileXR +source scripts/common_env.sh +cmake --build build_ccu_direct --target tile-comm -j4 +``` + +如果 build 目录不同，需要同步修改 `TILEXR_TILE_COMM_LIB`。 + +### 9.2 两卡 P2P Copy Smoke + +以下命令是当前推荐的 direct CCU 数据面 smoke。设备号按实际健康设备调整。 + +```bash +cd /home/tileXR +source scripts/common_env.sh + +export TILEXR_TILE_COMM_LIB=/home/tileXR/build_ccu_direct/src/comm/libtile-comm.so +export TILEXR_CCU_SMOKE_DEVICES=3,2 +export TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1 +export TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY=1 +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=64 +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK=0 +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 +export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 +export TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS=180000 +export TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 +export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_ON_PREPARE_FAILURE=0 +export TILEXR_CCU_SMOKE_TIMEOUT=180 +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=remote_to_local + +timeout 420s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +反方向验证： + +```bash +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=local_to_remote +timeout 420s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +预期关键日志： + +```text +tilexr_ccu_direct_smoke prepare ret=0 ... installSucceeded=1 ... submitReady=1 +tilexr_ccu_direct_smoke submit ret=0 ... submitted=1 +tilexr_ccu_direct_smoke p2pCcuCopy ... mismatches=0 ... passed=1 +tilexr_ccu_direct_smoke_runner success +``` + +### 9.3 覆盖 `TileXRComm` 自动初始化路径 + +默认 P2P smoke 可以只用 direct CCU internal init。若要覆盖 `TileXRComm::Init()` 中的 backend auto-init 路径，增加： + +```bash +export TILEXR_ENABLE_CCU_BACKEND=1 +export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=0 +timeout 420s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +### 9.4 Dry-run 参数检查 + +不触碰 ACL/NPU，只检查 runner 推导出的 repository/task 参数： + +```bash +export TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 +export TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1 +export TILEXR_CCU_DIRECT_SMOKE_DRY_RUN=1 +bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +## 10. 测试矩阵 + +### 10.1 本地单元测试 + +推荐先跑 CCU 相关 Python 测试： + +```bash +python3 -m unittest \ + tests.ccu.test_tilexr_ccu_resource_allocator \ + tests.ccu.test_tilexr_ccu_lower_layer_plan_builder \ + tests.ccu.test_tilexr_ccu_backend_boundary \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_direct_smoke_runner \ + tests.ccu.test_tilexr_ccu_public_comm_api +``` + +覆盖更完整的 CCU suite： + +```bash +python3 -m unittest discover tests/ccu +``` + +### 10.2 依赖边界检查 + +每次修改 `src/comm/ccu` 后应确认 `tile-comm` 没有引入 hcomm/HCCL 私有 CCU 依赖： + +```bash +source scripts/common_env.sh +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh build/src/comm/libtile-comm.so +``` + +### 10.3 硬件 Smoke + +硬件 smoke 由 `tests/ccu/run_tilexr_ccu_direct_smoke.sh` 启动两个 rank 进程。runner 默认是安全的，必须设置： + +```bash +export TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 +``` + +runner 会： + +- 编译 `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp`。 +- 检查 `npu-smi info` 是否可完成。 +- 默认拒绝 busy/unhealthy 设备。 +- 分别启动 rank0/rank1。 +- 为 prepare、install、submit、p2p result 做日志断言。 +- 用 `timeout` 包住 rank 进程，避免测试卡死。 + +如确认设备健康但 `npu-smi` health 字段不是 OK，可显式放开 unhealthy 检查： + +```bash +export TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU=1 +``` + +不要在没有明确授权时设置： + +```bash +export TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU=1 +``` + +### 10.4 前序验证记录 + +在 950 验证服务器的前序复测中，当前分支曾通过以下检查： + +- `cmake --build build_ccu_direct --target tile-comm -j4` +- 143 个 CCU 单元测试通过，1 个 skip。 +- 两卡 P2P copy smoke 在 `remote_to_local` 和 `local_to_remote` 两个方向通过。 +- `TILEXR_ENABLE_CCU_BACKEND=1` 且 `TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT=0` 的自动初始化路径通过 P2P copy smoke。 + +注意：默认非 P2P smoke 曾暴露 repository install 旧参数问题，因此不能把默认非 P2P smoke 作为当前推荐验证路径。 + +## 11. 常见问题定位 + +### 11.1 runtime 初始化失败 + +看日志中的： + +```text +TileXR direct CCU runtime init failed +logicDevId +devicePhyId +hdcType +raInitialized +ccuTlvInitialized +message +``` + +如果同一 device 后续被跳过，说明 device-scoped unavailable 状态已经记录。需要先解决首次失败原因，或换健康 device 重试。 + +### 11.2 prepare 卡住或 allgather 超时 + +重点检查： + +- rank0/rank1 是否都启动。 +- `TILEXR_COMM_ID`、端口和 rankSize 是否一致。 +- 某个 rank 是否先因 runtime unavailable 退出。 +- thread mode 下 uid 是否一致。 +- `TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS` 是否过短。 + +### 11.3 resource window 注册失败 + +重点检查： + +- 是否设置 `TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=ra_ctx`。 +- EID index 是否在 `RaGetDevEidInfoList` 返回列表内。 +- 是否选择了错误 device 或 busy device。 +- resource address 是否来自当前 device 的 CCU basic info。 + +可打开： + +```bash +export TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=1 +``` + +查看 EID、TPN、doorbell token、QP import 等信息。 + +### 11.4 repository 或 lower-layer install 失败 + +重点看 prepare report： + +- `installAttempted` +- `installSucceeded` +- `requiredInstallSurfaceCount` +- `publicVerifiedInstallSurfaceCount` +- `missingInstallSurfaceCount` +- `message` + +如果缺失 surface，通常说明 lower-layer plan 没有拿到完整 peer route/token，或 resource range 与 manifest 不匹配。 + +### 11.5 submit 失败 + +submit 失败时 report 会打印 task 关键字段： + +```text +missionId +key +instStartId +instCnt +argSize +args[] +rtRet +``` + +定位顺序： + +1. 确认 prepare 阶段 `submitReady=1`。 +2. 确认 repository mission window 覆盖 task 的 `instStartId/instCnt`。 +3. 确认 mission id/key 与 install manifest 匹配。 +4. 确认 stream 非空。 +5. 打开 `TILEXR_CCU_DIRECT_TRACE=1` 查看最终 task 和 microcode 解码。 + +### 11.6 P2P copy 不匹配 + +重点区分方向： + +- `remote_to_local`：active rank 校验本地 destination。 +- `local_to_remote`：inactive rank 等待 active rank done 后校验本地 destination。 + +常见原因： + +- active rank 设置错误。 +- direction 与期望校验 rank 不一致。 +- process token 查询失败。 +- endpoint route 不完整导致 channel 指向错误 peer。 +- submit 成功但 stream synchronize 失败。 + +## 12. 后续扩展建议 + +当前代码已经具备 direct runtime、resource allocator、lower-layer install、repository install、mission/task submit 和 P2P copy 数据面验证基础。后续要做泛化 alltoall/collective，建议按以下顺序推进： + +1. 先把 collective request 到 producer plan 的映射补齐，而不是绕过 `TileXRCcuCollectivePlanner::Supports()`。 +2. 复用现有 resource allocator 和 lower-layer evidence，不新增无证据的 env override。 +3. 用多 task submit 的单元测试覆盖 launch package 批量下发。 +4. 在两卡 P2P copy 稳定后，再扩展到 4 卡 N-to-N 数据流。 +5. alltoall 数据正确性必须以 device buffer 内容校验为准，不能用 host marker 或单纯 submit 成功替代。 + diff --git a/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md b/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md new file mode 100644 index 00000000..81f8b54a --- /dev/null +++ b/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md @@ -0,0 +1,126 @@ +# CCU Signal/Wait Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build internal two-rank Direct CCU signal/wait synchronization in `TileXRCcuBackend`, then compose a two-rank barrier from the same mechanism. + +**Architecture:** Add internal request/plan types to the CCU backend boundary and route preparation through the existing Direct CCU install pipeline. Generate role-specific CCU synchronization programs, reuse lower-layer resource proof and repository/mission install, and validate with unit tests plus the two-rank smoke probe. + +**Tech Stack:** C++14, CANN ACL/runtime, TileXR Direct CCU modules under `src/comm/ccu`, Python `unittest`, shell smoke runner. + +## Global Constraints + +- Do not expose public C API in `src/include/tilexr_api.h`. +- Do not link or include hcomm/HCCL private CCU producer APIs. +- Reuse existing Direct CCU runtime lifecycle, resource-window registration, peer allgather, lower-layer install, repository install, mission install, and `rtCCULaunch`. +- Implement single-direction signal/wait first, then two-direction barrier. +- Hardware validation must prove device-side synchronization with delayed signal timing, not host marker success. + +--- + +### Task 1: Internal Backend Types and Submission Surface + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_backend.h` +- Modify: `src/comm/ccu/tilexr_ccu_backend.cpp` +- Test: `tests/ccu/test_tilexr_ccu_backend_boundary.py` + +**Interfaces:** +- Produces: `enum class TileXRCcuSignalWaitRole { Signal, Wait, SignalAndWait };` +- Produces: `struct TileXRCcuSignalWaitRequest` +- Produces: `struct TileXRCcuSignalWaitPlan` +- Produces: `int TileXRCcuBackend::PrepareSignalWait(const TileXRCcuSignalWaitRequest&, TileXRCcuSignalWaitPlan*)` +- Produces: `int TileXRCcuBackend::SubmitSignalWait(const TileXRCcuSignalWaitPlan&, aclrtStream, TileXRCcuDirectSubmitReport*)` + +- [ ] Add failing boundary tests that assert internal signal/wait types exist in `tilexr_ccu_backend.h` and do not appear in `src/include/tilexr_api.h`. +- [ ] Add backend declarations and simple forwarding methods. +- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_backend_boundary`. + +### Task 2: Signal/Wait Program Builder + +**Files:** +- Create: `src/comm/ccu/tilexr_ccu_signal_wait_program.h` +- Create: `src/comm/ccu/tilexr_ccu_signal_wait_program.cpp` +- Modify: `src/comm/CMakeLists.txt` +- Test: `tests/ccu/test_tilexr_ccu_signal_wait_program.py` + +**Interfaces:** +- Consumes: `TileXRCcuSignalWaitRole` +- Produces: `struct TileXRCcuSignalWaitProgramSpec` +- Produces: `int TileXRCcuBuildSignalWaitProgram(const TileXRCcuSignalWaitProgramSpec&, std::vector*, TileXRCcuBarrierProgramReport*)` + +- [ ] Add tests for signal-only, wait-only, signal-and-wait, and invalid resource cases. +- [ ] Implement the builder using existing microcode encoders, not duplicate opcode packing. +- [ ] Add the new source file to the `tile-comm` target. +- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_signal_wait_program`. + +### Task 3: Planner Prepare Path + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.h` +- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` +- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.h` +- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` +- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` + +**Interfaces:** +- Consumes: `TileXRCcuSignalWaitRequest` +- Produces: `int TileXRCcuCollectivePlanner::PrepareSignalWait(...)` +- Produces: `int TileXRCcuRunDirectSignalWaitInstallAttempt(...)` + +- [ ] Add orchestrator tests that build signal/wait launch packages and verify submit tasks are produced when install evidence is satisfied. +- [ ] Implement direct signal/wait install attempt by reusing `RunDirectInstallAttemptImpl` structure and selecting the signal/wait program builder. +- [ ] Add planner method that validates rank size 2, peer rank, role, runtime availability, and basic info. +- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator`. + +### Task 4: Backend Wiring + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_backend.cpp` +- Test: `tests/ccu/test_tilexr_ccu_backend_boundary.py` + +**Interfaces:** +- Consumes: `TileXRCcuCollectivePlanner::PrepareSignalWait` +- Produces: working `TileXRCcuBackend::PrepareSignalWait` and `SubmitSignalWait` + +- [ ] Add tests for null plan, unavailable runtime, not-ready submit, and null stream. +- [ ] Wire backend preparation through planner and submission through `TileXRCcuSubmitPreparedTasks`. +- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_backend_boundary`. + +### Task 5: Smoke Probe and Runner + +**Files:** +- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` +- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` +- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` +- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` + +**Interfaces:** +- Consumes: `TileXRCcuBackend::PrepareSignalWait` +- Produces env-gated smoke modes: + - `TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=1` + - `TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=` + - `TILEXR_CCU_DIRECT_SMOKE_BARRIER=1` + +- [ ] Add source tests for signal/wait mode selection and inactive/active timing behavior. +- [ ] Add runner env propagation and log assertions for signal/wait and barrier. +- [ ] Implement smoke probe path using internal backend methods. +- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner`. + +### Task 6: Local and Remote Validation + +**Files:** +- No new source files; run validation commands. + +**Interfaces:** +- Consumes: all previous tasks. +- Produces: verified Direct CCU signal/wait and barrier behavior. + +- [ ] Run focused CCU unit tests: + `python3 -m unittest tests.ccu.test_tilexr_ccu_signal_wait_program tests.ccu.test_tilexr_ccu_direct_orchestrator tests.ccu.test_tilexr_ccu_backend_boundary tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner` +- [ ] Build `tile-comm` on the NPU server. +- [ ] Run no-hcomm dependency guard. +- [ ] Run two-card `rank0 -> rank1` signal/wait smoke with delayed signal rank. +- [ ] Run two-card `rank1 -> rank0` signal/wait smoke with delayed signal rank. +- [ ] Run two-card barrier smoke. + diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index ca75df16..a635c42b 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -145,6 +145,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_resource_allocator.cpp ccu/tilexr_ccu_runtime.h ccu/tilexr_ccu_runtime.cpp + ccu/tilexr_ccu_signal_wait_program.h + ccu/tilexr_ccu_signal_wait_program.cpp ccu/tilexr_ccu_runtime_session.h ccu/tilexr_ccu_runtime_session.cpp ccu/tilexr_ccu_executor.h diff --git a/src/comm/ccu/tilexr_ccu_backend.cpp b/src/comm/ccu/tilexr_ccu_backend.cpp index 3c3afbde..27efa405 100644 --- a/src/comm/ccu/tilexr_ccu_backend.cpp +++ b/src/comm/ccu/tilexr_ccu_backend.cpp @@ -20,6 +20,11 @@ class TileXRCcuBackend::Impl { bool Supports(const TileXRCcuCollectiveRequest &request) const; int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); + int PrepareSignalWait(const TileXRCcuSignalWaitRequest &request, TileXRCcuSignalWaitPlan *plan); + int SubmitSignalWait( + const TileXRCcuSignalWaitPlan &plan, + aclrtStream stream, + TileXRCcuDirectSubmitReport *report); #ifdef TILEXR_CCU_TESTING bool RuntimeInitializedForTest() const; #endif @@ -105,6 +110,38 @@ int TileXRCcuBackend::Impl::SubmitCollective(const TileXRCcuCollectivePlan &plan return executor_->Submit(*runtimeSession_, plan, stream); } +int TileXRCcuBackend::Impl::PrepareSignalWait( + const TileXRCcuSignalWaitRequest &request, + TileXRCcuSignalWaitPlan *plan) +{ + if (plan == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *plan = TileXRCcuSignalWaitPlan {}; + if (runtimeSession_ == nullptr || planner_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + TileXRCcuDirectInstallReport report; + return planner_->PrepareSignalWait(*runtimeSession_, request, plan, &report); +} + +int TileXRCcuBackend::Impl::SubmitSignalWait( + const TileXRCcuSignalWaitPlan &plan, + aclrtStream stream, + TileXRCcuDirectSubmitReport *report) +{ + if (report != nullptr) { + *report = TileXRCcuDirectSubmitReport {}; + } + if (!plan.ready || plan.submitTasks.empty()) { + if (report != nullptr) { + report->message = "direct CCU signal/wait plan is not ready"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + return TileXRCcuSubmitPreparedTasks(plan.submitTasks, stream, nullptr, nullptr, report); +} + #ifdef TILEXR_CCU_TESTING bool TileXRCcuBackend::Impl::RuntimeInitializedForTest() const { @@ -165,6 +202,27 @@ int TileXRCcuBackend::SubmitCollective(const TileXRCcuCollectivePlan &plan, aclr return impl_->SubmitCollective(plan, stream); } +int TileXRCcuBackend::PrepareSignalWait( + const TileXRCcuSignalWaitRequest &request, + TileXRCcuSignalWaitPlan *plan) +{ + if (impl_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + return impl_->PrepareSignalWait(request, plan); +} + +int TileXRCcuBackend::SubmitSignalWait( + const TileXRCcuSignalWaitPlan &plan, + aclrtStream stream, + TileXRCcuDirectSubmitReport *report) +{ + if (impl_ == nullptr) { + return TILEXR_ERROR_INTERNAL; + } + return impl_->SubmitSignalWait(plan, stream, report); +} + #ifdef TILEXR_CCU_TESTING bool TileXRCcuBackend::RuntimeInitializedForTest() const { diff --git a/src/comm/ccu/tilexr_ccu_backend.h b/src/comm/ccu/tilexr_ccu_backend.h index 8177e211..65ebfd82 100644 --- a/src/comm/ccu/tilexr_ccu_backend.h +++ b/src/comm/ccu/tilexr_ccu_backend.h @@ -8,7 +8,9 @@ #include #include #include +#include +#include "ccu/tilexr_ccu_direct_orchestrator.h" #include "acl/acl_base.h" #include "tilexr_types.h" @@ -42,6 +44,38 @@ struct TileXRCcuCollectivePlan { bool ready = false; }; +enum class TileXRCcuSignalWaitRole { + Signal = 0, + Wait = 1, + SignalAndWait = 2, +}; + +struct TileXRCcuSignalWaitRequest { + int peerRank = -1; + TileXRCcuSignalWaitRole role = TileXRCcuSignalWaitRole::Signal; + uint32_t syncInstructionCount = 0; + uint16_t missionStartId = 0; + uint16_t instructionStartId = 0; + uint16_t missionInstructionStartId = 0; + uint16_t xnStartId = 0; + uint16_t remoteXnStartId = 0; + uint16_t remoteXnCount = 0; + uint16_t ckeStartId = 0; + uint16_t channelStartId = 0; + uint16_t localWaitCkeStartId = 0; + uint16_t localWaitCkeCount = 0; + uint16_t remoteNotifyCkeStartId = 0; + uint16_t remoteNotifyCkeCount = 0; + uint16_t timeout = 0; + std::string provider; +}; + +struct TileXRCcuSignalWaitPlan { + bool ready = false; + TileXRCcuDirectInstallAttempt attempt; + std::vector submitTasks; +}; + class TileXRCcuBackend { public: TileXRCcuBackend(); @@ -56,6 +90,11 @@ class TileXRCcuBackend { bool Supports(const TileXRCcuCollectiveRequest &request) const; int PrepareCollective(const TileXRCcuCollectiveRequest &request, TileXRCcuCollectivePlan *plan); int SubmitCollective(const TileXRCcuCollectivePlan &plan, aclrtStream stream); + int PrepareSignalWait(const TileXRCcuSignalWaitRequest &request, TileXRCcuSignalWaitPlan *plan); + int SubmitSignalWait( + const TileXRCcuSignalWaitPlan &plan, + aclrtStream stream, + TileXRCcuDirectSubmitReport *report); #ifdef TILEXR_CCU_TESTING bool RuntimeInitializedForTest() const; #endif diff --git a/src/comm/ccu/tilexr_ccu_barrier_program.cpp b/src/comm/ccu/tilexr_ccu_barrier_program.cpp index 319caeea..73b3a783 100644 --- a/src/comm/ccu/tilexr_ccu_barrier_program.cpp +++ b/src/comm/ccu/tilexr_ccu_barrier_program.cpp @@ -221,7 +221,7 @@ int TileXRCcuBuildBarrierProgram( TileXRCcuSyncCkeSpec post; post.remoteCke = spec.remoteNotifyCke; post.localCke = spec.sourceCke; - post.localCkeMask = spec.remoteNotifyMask; + post.localCkeMask = spec.sourceCkeMask; post.channelId = spec.channelId; if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { return Fail(program, report, "failed to encode CCU barrier SyncCKE post instruction"); diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 30731932..7991522b 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #ifdef TILEXR_CCU_TESTING #include "runtime/dev.h" @@ -25,6 +26,9 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; #endif +constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; uint8_t SelectDirectCcuInstallDieId() { @@ -73,17 +77,98 @@ uint16_t SelectDirectCcuChannelBoundRemoteXnId( SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); } +TileXRCcuSignalWaitProgramRole ToDirectSignalWaitProgramRole(TileXRCcuSignalWaitRole role) +{ + if (role == TileXRCcuSignalWaitRole::Wait) { + return TileXRCcuSignalWaitProgramRole::Wait; + } + if (role == TileXRCcuSignalWaitRole::SignalAndWait) { + return TileXRCcuSignalWaitProgramRole::SignalAndWait; + } + return TileXRCcuSignalWaitProgramRole::Signal; +} + +uint32_t DefaultSignalWaitInstructionCount(TileXRCcuSignalWaitRole role) +{ + if (role == TileXRCcuSignalWaitRole::Wait) { + return TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT; + } + if (role == TileXRCcuSignalWaitRole::SignalAndWait) { + return TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT; + } + return TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT; +} + #ifdef TILEXR_CCU_TESTING struct DirectCcuMemoryCopyEndpoint { uint64_t sourceAddr = 0; uint64_t sourceToken = 0; + TileXRCcuRemoteMemoryBufferImportRequest sourceRemoteImport {}; uint64_t destinationAddr = 0; uint64_t destinationToken = 0; + TileXRCcuRemoteMemoryBufferImportRequest destinationRemoteImport {}; uint64_t bytes = 0; uint32_t rank = 0; uint32_t valid = 0; }; +bool TraceDirectCcuMemoryCopy() +{ + const char *trace = std::getenv("TILEXR_CCU_DIRECT_TRACE"); + return trace != nullptr && trace[0] != '\0' && trace[0] != '0'; +} + +bool UseCcuResourceWindowForMemoryCopy() +{ + const char *value = std::getenv("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + +void TraceDirectCcuProcessMemoryToken( + uint64_t addr, + uint64_t bytes, + uint32_t rawTokenId, + uint32_t tokenId, + uint32_t tokenValue, + uint64_t packedToken) +{ + if (!TraceDirectCcuMemoryCopy()) { + return; + } + std::cerr << "TileXRDirectCcuTrace memoryToken" + << " addr=0x" << std::hex << addr + << " bytes=0x" << bytes + << " rawTokenId=0x" << rawTokenId + << " tokenId=0x" << tokenId + << " tokenValue=0x" << tokenValue + << " packed=0x" << packedToken + << std::dec << std::endl; +} + +void TraceDirectCcuRegisteredMemoryToken( + const char *label, + uint64_t addr, + uint64_t bytes, + const TileXRCcuRegisteredMemoryBufferInfo &info, + uint64_t packedToken) +{ + if (!TraceDirectCcuMemoryCopy()) { + return; + } + std::cerr << "TileXRDirectCcuTrace registeredMemoryToken" + << " label=" << (label == nullptr ? "" : label) + << " addr=0x" << std::hex << addr + << " bytes=0x" << bytes + << " alignedAddr=0x" << info.alignedAddr + << " alignedBytes=0x" << info.alignedBytes + << " targetSegVa=0x" << info.targetSegVa + << " rawTokenId=0x" << info.rawTokenId + << " tokenId=0x" << info.tokenId + << " tokenValue=0x" << info.tokenValue + << " packed=0x" << packedToken + << std::dec << std::endl; +} + int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *packedToken) { if (addr == 0 || bytes == 0 || packedToken == nullptr) { @@ -97,13 +182,14 @@ int QueryDirectCcuProcessMemoryToken(uint64_t addr, uint64_t bytes, uint64_t *pa if (ret != RT_ERROR_NONE) { return TILEXR_ERROR_MKIRT; } - constexpr uint32_t tokenIdRightShift = 8U; - const uint32_t tokenId = info.tokenId >> tokenIdRightShift; + const uint32_t tokenId = info.tokenId; *packedToken = TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true); + TraceDirectCcuProcessMemoryToken(addr, bytes, info.tokenId, tokenId, info.tokenValue, *packedToken); return *packedToken == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_SUCCESS; } int BuildDirectCcuLocalMemoryCopyEndpoint( + TileXRCcuRuntimeSession &session, uint32_t rank, uint64_t sourceAddr, uint64_t destinationAddr, @@ -116,16 +202,53 @@ int BuildDirectCcuLocalMemoryCopyEndpoint( *endpoint = DirectCcuMemoryCopyEndpoint {}; endpoint->rank = rank; endpoint->bytes = bytes; - endpoint->sourceAddr = sourceAddr; - endpoint->destinationAddr = destinationAddr; - int ret = QueryDirectCcuProcessMemoryToken(sourceAddr, bytes, &endpoint->sourceToken); + TileXRCcuRegisteredMemoryBufferInfo sourceInfo; + int ret = session.RegisterMemoryBuffer(sourceAddr, bytes, &sourceInfo); if (ret != TILEXR_SUCCESS) { return ret; } - ret = QueryDirectCcuProcessMemoryToken(destinationAddr, bytes, &endpoint->destinationToken); + if (sourceInfo.targetSegVa == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + endpoint->sourceAddr = sourceInfo.addr; + endpoint->sourceToken = TileXRCcuPackMemoryToken(sourceInfo.tokenId, sourceInfo.tokenValue, true); + endpoint->sourceRemoteImport.addr = sourceInfo.addr; + endpoint->sourceRemoteImport.bytes = sourceInfo.bytes; + endpoint->sourceRemoteImport.alignedAddr = sourceInfo.alignedAddr; + endpoint->sourceRemoteImport.offset = sourceInfo.addr - sourceInfo.alignedAddr; + endpoint->sourceRemoteImport.tokenId = sourceInfo.tokenId; + endpoint->sourceRemoteImport.rawTokenId = sourceInfo.rawTokenId; + endpoint->sourceRemoteImport.tokenValue = sourceInfo.tokenValue; + endpoint->sourceRemoteImport.key = sourceInfo.key; + endpoint->sourceRemoteImport.valid = true; + TraceDirectCcuRegisteredMemoryToken("source", sourceAddr, bytes, sourceInfo, endpoint->sourceToken); + + TileXRCcuRegisteredMemoryBufferInfo destinationInfo; + ret = session.RegisterMemoryBuffer(destinationAddr, bytes, &destinationInfo); if (ret != TILEXR_SUCCESS) { return ret; } + if (destinationInfo.targetSegVa == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + endpoint->destinationAddr = destinationInfo.addr; + endpoint->destinationToken = + TileXRCcuPackMemoryToken(destinationInfo.tokenId, destinationInfo.tokenValue, true); + endpoint->destinationRemoteImport.addr = destinationInfo.addr; + endpoint->destinationRemoteImport.bytes = destinationInfo.bytes; + endpoint->destinationRemoteImport.alignedAddr = destinationInfo.alignedAddr; + endpoint->destinationRemoteImport.offset = destinationInfo.addr - destinationInfo.alignedAddr; + endpoint->destinationRemoteImport.tokenId = destinationInfo.tokenId; + endpoint->destinationRemoteImport.rawTokenId = destinationInfo.rawTokenId; + endpoint->destinationRemoteImport.tokenValue = destinationInfo.tokenValue; + endpoint->destinationRemoteImport.key = destinationInfo.key; + endpoint->destinationRemoteImport.valid = true; + TraceDirectCcuRegisteredMemoryToken( + "destination", + destinationAddr, + bytes, + destinationInfo, + endpoint->destinationToken); endpoint->valid = 1; return TILEXR_SUCCESS; } @@ -332,6 +455,9 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerTemplateFromAllocation directCcuLowerLayerPlanStatus_ = ret; return directCcuLowerLayerPlanStatus_; } +#ifdef TILEXR_CCU_TESTING + ApplyDirectCcuRemoteRouteMemoryOverride(&remoteCcuBuffers); +#endif TileXRCcuLowerLayerTransportSnapshot templateSnapshot; TileXRCcuLowerLayerPlanBuilderReport report; @@ -539,6 +665,48 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( return TILEXR_SUCCESS; } +#ifdef TILEXR_CCU_TESTING +void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue) +{ + directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; + directCcuRemoteRouteMemoryOverride_.peerRank = peerRank; + directCcuRemoteRouteMemoryOverride_.remoteCcuVa = remoteCcuVa; + directCcuRemoteRouteMemoryOverride_.memoryTokenId = memoryTokenId; + directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId = rawMemoryTokenId; + directCcuRemoteRouteMemoryOverride_.memoryTokenValue = memoryTokenValue; + directCcuRemoteRouteMemoryOverrideValid_ = + remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; +} + +void TileXRCcuCollectivePlanner::ClearDirectCcuRemoteRouteMemoryOverride() +{ + directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; + directCcuRemoteRouteMemoryOverrideValid_ = false; +} + +void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( + std::vector *remoteCcuBuffers) const +{ + if (!directCcuRemoteRouteMemoryOverrideValid_ || remoteCcuBuffers == nullptr) { + return; + } + for (auto &remoteCcuBuffer : *remoteCcuBuffers) { + if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank) { + continue; + } + remoteCcuBuffer.remoteCcuVa = directCcuRemoteRouteMemoryOverride_.remoteCcuVa; + remoteCcuBuffer.memoryTokenId = directCcuRemoteRouteMemoryOverride_.memoryTokenId; + remoteCcuBuffer.rawMemoryTokenId = directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId; + remoteCcuBuffer.memoryTokenValue = directCcuRemoteRouteMemoryOverride_.memoryTokenValue; + } +} +#endif + int TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback( const TileXRCcuResourceAllocation &allocation, TileXRCcuLowerLayerInstallPlan *plan, @@ -626,6 +794,121 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuInstallAttempt( return TileXRCcuRunDirectInstallAttempt(next, attempt, report); } +int TileXRCcuCollectivePlanner::PrepareSignalWait( + TileXRCcuRuntimeSession &session, + const TileXRCcuSignalWaitRequest &request, + TileXRCcuSignalWaitPlan *plan, + TileXRCcuDirectInstallReport *report) +{ + if (plan == nullptr) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "missing direct CCU signal/wait plan output"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *plan = TileXRCcuSignalWaitPlan {}; + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU signal/wait"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 2 || request.peerRank < 0 || request.peerRank >= rankSize || request.peerRank == rank) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU signal/wait requires two ranks and a valid peer rank"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for signal/wait"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + int ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions options; + options.basicInfo = basicInfo; + options.sqeArgCount = 0; + options.syncResourceCount = 1; + options.syncInstructionCount = request.syncInstructionCount == 0 ? + DefaultSignalWaitInstructionCount(request.role) : + request.syncInstructionCount; + options.bindingsPerSyncResource = 1; + options.missionStartId = request.missionStartId; + options.instructionStartId = request.instructionStartId; + options.missionInstructionStartId = request.missionInstructionStartId; + options.xnStartId = request.xnStartId; + options.remoteXnStartId = request.remoteXnStartId; + options.remoteXnCount = request.remoteXnCount; + options.ckeStartId = request.ckeStartId; + options.channelStartId = request.channelStartId; + options.localWaitCkeStartId = request.localWaitCkeStartId; + options.localWaitCkeCount = request.localWaitCkeCount; + options.remoteNotifyCkeStartId = request.remoteNotifyCkeStartId; + options.remoteNotifyCkeCount = request.remoteNotifyCkeCount; + options.taskTimeout = request.timeout; + options.deviceId = static_cast(session.DevId()); + options.rank = static_cast(rank); + options.provider = request.provider.empty() ? "tilexr-comm-direct-ccu-signal-wait" : request.provider; + options.offlineOnly = false; + options.driverAdapter = &adapter; + options.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(options.repositoryMemoryAllocMode); + options.repositoryMemoryUserData = nullptr; + options.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + options.lowerLayerPlanUserData = &callbackContext; + + TileXRCcuDirectSignalWaitSpec signalWait; + signalWait.role = ToDirectSignalWaitProgramRole(request.role); + ret = TileXRCcuRunDirectSignalWaitInstallAttempt(options, signalWait, &plan->attempt, report); + if (ret != TILEXR_SUCCESS) { + *plan = TileXRCcuSignalWaitPlan {}; + return ret; + } + plan->submitTasks = plan->attempt.submitTasks; + plan->ready = !plan->submitTasks.empty(); + return plan->ready ? TILEXR_SUCCESS : TILEXR_ERROR_NOT_FOUND; +} + #ifdef TILEXR_CCU_TESTING int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( TileXRCcuRuntimeSession &session, @@ -684,8 +967,18 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( return TILEXR_ERROR_NOT_FOUND; } + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before memory copy buffer registration"; + } + return ret; + } + DirectCcuMemoryCopyEndpoint localEndpoint; - int ret = BuildDirectCcuLocalMemoryCopyEndpoint( + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, static_cast(rank), localSourceAddr, localDestinationAddr, @@ -720,19 +1013,91 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( return TILEXR_ERROR_PARA_CHECK_FAIL; } + TileXRCcuLocalResourceWindowInfo localResourceWindow; + std::vector allResourceWindows; + const bool useResourceWindow = UseCcuResourceWindowForMemoryCopy(); + if (useResourceWindow) { + ret = session.ExportLocalCcuRmaBuffer(&localResourceWindow); + if (ret != TILEXR_SUCCESS || localResourceWindow.addr == 0 || localResourceWindow.tokenId == 0) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to export direct CCU resource window for memory copy"; + } + return ret == TILEXR_SUCCESS ? TILEXR_ERROR_NOT_FOUND : ret; + } + allResourceWindows.resize(static_cast(rankSize)); + ret = session.AllGather( + &localResourceWindow, + sizeof(localResourceWindow), + allResourceWindows.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU resource windows for memory copy"; + } + return ret; + } + } + TileXRCcuDirectMemoryCopySpec memoryCopy; memoryCopy.direction = direction; memoryCopy.lengthBytes = bytes; + TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest; if (direction == TileXRCcuMemoryCopyDirection::RemoteToLocal) { + remoteImportRequest = peerEndpoint.sourceRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU memory copy remote source buffer"; + } + return ret; + } memoryCopy.localAddr = localEndpoint.destinationAddr; memoryCopy.localToken = localEndpoint.destinationToken; - memoryCopy.remoteAddr = peerEndpoint.sourceAddr; - memoryCopy.remoteToken = peerEndpoint.sourceToken; + if (useResourceWindow) { + const auto &peerWindow = allResourceWindows[peerRank]; + memoryCopy.remoteAddr = peerWindow.addr; + memoryCopy.remoteToken = TileXRCcuPackMemoryToken(peerWindow.tokenId, peerWindow.tokenValue, true); + } else { + memoryCopy.remoteAddr = remoteImportRequest.addr; + memoryCopy.remoteToken = + TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); + } } else { + remoteImportRequest = peerEndpoint.destinationRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU memory copy remote destination buffer"; + } + return ret; + } memoryCopy.localAddr = localEndpoint.sourceAddr; memoryCopy.localToken = localEndpoint.sourceToken; - memoryCopy.remoteAddr = peerEndpoint.destinationAddr; - memoryCopy.remoteToken = peerEndpoint.destinationToken; + if (useResourceWindow) { + const auto &peerWindow = allResourceWindows[peerRank]; + memoryCopy.remoteAddr = peerWindow.addr; + memoryCopy.remoteToken = TileXRCcuPackMemoryToken(peerWindow.tokenId, peerWindow.tokenValue, true); + } else { + memoryCopy.remoteAddr = remoteImportRequest.addr; + memoryCopy.remoteToken = + TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); + } + } + if (TraceDirectCcuMemoryCopy()) { + std::cerr << "TileXRDirectCcuTrace memoryCopyEndpoint" + << " rank=" << rank + << " peerRank=" << peerRank + << " direction=" << static_cast(direction) + << " localAddr=0x" << std::hex << memoryCopy.localAddr + << " localToken=0x" << memoryCopy.localToken + << " remoteAddr=0x" << memoryCopy.remoteAddr + << " remoteToken=0x" << memoryCopy.remoteToken + << " length=0x" << memoryCopy.lengthBytes + << std::dec << std::endl; } TileXRCcuDriverAdapter adapter; @@ -766,7 +1131,17 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( next.provider = "tilexr-comm-direct-ccu-memory-copy"; } - return TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); + if (!useResourceWindow) { + SetDirectCcuRemoteRouteMemoryOverride( + peerRank, + importedRemoteBuffer.targetSegVa, + remoteImportRequest.tokenId, + remoteImportRequest.rawTokenId, + remoteImportRequest.tokenValue); + } + ret = TileXRCcuRunDirectMemoryCopyInstallAttempt(next, memoryCopy, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; } #endif diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index 27888e5f..9c512711 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -47,6 +47,11 @@ class TileXRCcuCollectivePlanner { const TileXRCcuDirectInstallOptions &options, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareSignalWait( + TileXRCcuRuntimeSession &session, + const TileXRCcuSignalWaitRequest &request, + TileXRCcuSignalWaitPlan *plan, + TileXRCcuDirectInstallReport *report); #ifdef TILEXR_CCU_TESTING int PrepareDirectCcuMemoryCopyInstallAttempt( TileXRCcuRuntimeSession &session, @@ -82,6 +87,17 @@ class TileXRCcuCollectivePlanner { const TileXRCcuResourceAllocation &allocation, std::vector *remoteCcuBuffers, TileXRCcuLowerLayerPlanBuilderReport *report); +#ifdef TILEXR_CCU_TESTING + void SetDirectCcuRemoteRouteMemoryOverride( + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue); + void ClearDirectCcuRemoteRouteMemoryOverride(); + void ApplyDirectCcuRemoteRouteMemoryOverride( + std::vector *remoteCcuBuffers) const; +#endif static int PrepareDirectCcuLowerLayerPlanCallback( const TileXRCcuResourceAllocation &allocation, TileXRCcuLowerLayerInstallPlan *plan, @@ -98,6 +114,10 @@ class TileXRCcuCollectivePlanner { std::vector directCcuVerifiedEndpointRoutes_ = {}; TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; bool directCcuLocalVerifiedEndpointRouteValid_ = false; +#ifdef TILEXR_CCU_TESTING + TileXRCcuRemoteCcuBufferInfo directCcuRemoteRouteMemoryOverride_ = {}; + bool directCcuRemoteRouteMemoryOverrideValid_ = false; +#endif }; } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index f3d8aef5..3f0fc29c 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace TileXR { @@ -34,6 +35,9 @@ constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALUE_MASK = 0xffffffffULL; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; +constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; void ResetReport(TileXRCcuDirectInstallReport* report) { @@ -784,6 +788,34 @@ int BuildDirectMemoryCopyLaunchPackage( copySpec.channelId = resource.channelId; copySpec.completionCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; copySpec.completionMask = resource.localWaitMask == 0 ? 1U : resource.localWaitMask; + if (DirectTraceEnabled()) { + const auto tracePackedToken = [](uint64_t token, const char* label) { + std::cerr << " " << label << "=0x" << std::hex << token + << " " << label << "Valid=" << ((token >> TILEXR_CCU_PACKED_TOKEN_VALID_SHIFT) & 0x1ULL) + << " " << label << "Id=0x" + << ((token >> TILEXR_CCU_PACKED_TOKEN_ID_SHIFT) & TILEXR_CCU_PACKED_TOKEN_ID_MASK) + << " " << label << "Value=0x" + << (token & TILEXR_CCU_PACKED_TOKEN_VALUE_MASK) + << std::dec; + }; + std::cerr << "TileXRDirectCcuTrace memoryCopySpec" + << " direction=" << static_cast(copySpec.direction) + << " localGsa=" << copySpec.localGsa + << " localXn=" << copySpec.localXn + << " remoteGsa=" << copySpec.remoteGsa + << " remoteXn=" << copySpec.remoteXn + << " lengthXn=" << copySpec.lengthXn + << " channelId=" << copySpec.channelId + << " completionCke=" << copySpec.completionCke + << " completionMask=" << copySpec.completionMask + << " localAddr=0x" << std::hex << copySpec.localAddr + << " remoteAddr=0x" << copySpec.remoteAddr + << " lengthBytes=0x" << copySpec.lengthBytes + << std::dec; + tracePackedToken(copySpec.localToken, "localToken"); + tracePackedToken(copySpec.remoteToken, "remoteToken"); + std::cerr << std::endl; + } TileXRCcuProgram program; TileXRCcuMemoryProgramReport memoryReport; @@ -821,6 +853,96 @@ int BuildDirectMemoryCopyLaunchPackage( return TILEXR_SUCCESS; } +uint32_t SignalWaitInstructionCount(TileXRCcuSignalWaitProgramRole role) +{ + if (role == TileXRCcuSignalWaitProgramRole::Wait) { + return TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT; + } + if (role == TileXRCcuSignalWaitProgramRole::Signal) { + return TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT; + } + return TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT; +} + +TileXRCcuBarrierMode SignalWaitBarrierMode(TileXRCcuSignalWaitProgramRole role) +{ + if (role == TileXRCcuSignalWaitProgramRole::Wait) { + return TileXRCcuBarrierMode::SyncCkePostOnly; + } + return role == TileXRCcuSignalWaitProgramRole::Signal ? + TileXRCcuBarrierMode::SyncCkePostOnly : + TileXRCcuBarrierMode::SyncCke; +} + +int BuildDirectSignalWaitLaunchPackage( + const TileXRCcuDirectSignalWaitSpec& signalWait, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->plan.syncResources.empty()) { + if (report != nullptr) { + report->message = "missing direct CCU signal/wait producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; + + TileXRCcuSignalWaitProgramSpec spec; + spec.role = signalWait.role; + spec.channelId = resource.channelId; + spec.remoteXn = resource.remoteXn; + spec.localXn = resource.localXn; + spec.localGsa = attempt->plan.kernelLocalGsa.num == 0 ? 0 : attempt->plan.kernelLocalGsa.startId; + spec.remoteNotifyCke = resource.notifyCke; + spec.remoteNotifyMask = resource.remoteNotifyMask == 0 ? 1U : resource.remoteNotifyMask; + spec.localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; + spec.localWaitMask = resource.localWaitMask == 0 ? 1U : resource.localWaitMask; + spec.sourceCke = resource.sourceCke; + spec.sourceCkeMask = spec.remoteNotifyMask; + + TileXRCcuProgram program; + TileXRCcuBarrierProgramReport signalWaitReport; + if (TileXRCcuBuildSignalWaitProgram(spec, &program.sync, &signalWaitReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = signalWaitReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (program.sync.empty() || program.sync.size() > std::numeric_limits::max()) { + if (report != nullptr) { + report->message = "invalid direct CCU signal/wait instruction count"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + void FillReportFromAttempt(const TileXRCcuDirectInstallAttempt& attempt, TileXRCcuDirectInstallReport* report) { if (report == nullptr) { @@ -866,6 +988,19 @@ void TraceFinalRuntimeTask(size_t taskIndex, const TileXRCcuTask& task) std::cerr << "\n"; } +void ApplyTaskTimeoutOverride(uint16_t taskTimeout, TileXRCcuDirectInstallAttempt* attempt) +{ + if (taskTimeout == 0 || attempt == nullptr) { + return; + } + for (auto& task : attempt->package.tasks) { + task.timeout = taskTimeout; + } + for (auto& task : attempt->submitTasks) { + task.timeout = taskTimeout; + } +} + std::string FormatSubmitTaskFailure( size_t taskIndex, int ret, @@ -983,6 +1118,7 @@ int TileXRCcuSubmitPreparedTasks( int RunDirectInstallAttemptImpl( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectMemoryCopySpec* memoryCopy, + const TileXRCcuDirectSignalWaitSpec* signalWait, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { @@ -1027,13 +1163,18 @@ int RunDirectInstallAttemptImpl( ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); - attempt->resourceRequest.sqeArgCount = memoryCopy == nullptr ? options.sqeArgCount : 0U; - attempt->resourceRequest.syncResourceCount = memoryCopy == nullptr ? options.syncResourceCount : 1U; - attempt->resourceRequest.syncInstructionCount = memoryCopy == nullptr ? - options.syncInstructionCount : - std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT); + attempt->resourceRequest.sqeArgCount = memoryCopy == nullptr && signalWait == nullptr ? options.sqeArgCount : 0U; + attempt->resourceRequest.syncResourceCount = + memoryCopy == nullptr && signalWait == nullptr ? options.syncResourceCount : 1U; + attempt->resourceRequest.syncInstructionCount = + memoryCopy != nullptr ? + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : + signalWait != nullptr ? + std::max(options.syncInstructionCount, SignalWaitInstructionCount(signalWait->role)) : + options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.barrierMode = options.barrierMode; + attempt->resourceRequest.barrierMode = + signalWait == nullptr ? options.barrierMode : SignalWaitBarrierMode(signalWait->role); TileXRCcuResourceAllocator allocator; if (allocator.Init(attempt->resourceSpec) != TILEXR_SUCCESS) { @@ -1061,6 +1202,8 @@ int RunDirectInstallAttemptImpl( report->message); } } + attempt->plan.barrierMode = + signalWait == nullptr ? attempt->plan.barrierMode : SignalWaitBarrierMode(signalWait->role); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); if (ret != TILEXR_SUCCESS) { @@ -1089,19 +1232,24 @@ int RunDirectInstallAttemptImpl( } TileXRCcuLaunchPackageReport packageReport; - ret = memoryCopy == nullptr ? - TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport) : - BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report); + ret = memoryCopy != nullptr ? + BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report) : + signalWait != nullptr ? + BuildDirectSignalWaitLaunchPackage(*signalWait, attempt, report) : + TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport); if (ret != TILEXR_SUCCESS) { return Fail( attempt, report, - memoryCopy == nullptr ? packageReport.message : + memoryCopy == nullptr && signalWait == nullptr ? packageReport.message : (report == nullptr || report->message.empty() ? - "failed to build direct CCU memory copy launch package" : + (memoryCopy != nullptr ? + "failed to build direct CCU memory copy launch package" : + "failed to build direct CCU signal/wait launch package") : report->message)); } + ApplyTaskTimeoutOverride(options.taskTimeout, attempt); TraceDirectInstallAttempt(*attempt); ret = TileXRCcuBindLaunchPackageInstallScope( @@ -1145,6 +1293,7 @@ int RunDirectInstallAttemptImpl( attempt->evidence, &attempt->submitTasks, &attempt->providerReport); + ApplyTaskTimeoutOverride(options.taskTimeout, attempt); if (submitRet == TILEXR_SUCCESS) { return ReturnWithAttemptStatus(TILEXR_SUCCESS, attempt, report); } @@ -1159,7 +1308,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1178,7 +1327,16 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, attempt, report); +} + +int TileXRCcuRunDirectSignalWaitInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSignalWaitSpec& signalWait, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + return RunDirectInstallAttemptImpl(options, nullptr, &signalWait, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index 60530cf7..7a7271cf 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -9,6 +9,7 @@ #include "ccu/tilexr_ccu_install_provider.h" #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" #include "ccu/tilexr_ccu_memory_program.h" +#include "ccu/tilexr_ccu_signal_wait_program.h" #include "ccu/tilexr_ccu_specs.h" #include @@ -43,6 +44,7 @@ struct TileXRCcuDirectInstallOptions { uint16_t remoteNotifyCkeStartId = 0; uint16_t remoteNotifyCkeCount = 0; TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; + uint16_t taskTimeout = 0; uint32_t deviceId = 0; uint32_t rank = 0; std::string provider; @@ -67,6 +69,10 @@ struct TileXRCcuDirectMemoryCopySpec { uint64_t lengthBytes = 0; }; +struct TileXRCcuDirectSignalWaitSpec { + TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; +}; + struct TileXRCcuDirectInstallAttempt { TileXRCcuSpecInfo specInfo; TileXRCcuResourceSpec resourceSpec; @@ -123,6 +129,12 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectSignalWaitInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSignalWaitSpec& signalWait, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt); int TileXRCcuSubmitPreparedTasks( diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp index eeb70259..26b91e71 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -54,6 +54,7 @@ constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE"; constexpr const char* TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET_ENV = "TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET"; +constexpr uint8_t TILEXR_CCU_DIRECT_DEFAULT_DIE_ID = 0; constexpr uint64_t TILEXR_CCU_UB_MEM_PAGE_SIZE = 4096ULL; constexpr uint32_t TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT = 8; @@ -212,6 +213,21 @@ void TraceEndpointRouteStep(const std::string& message) } } +void TraceTaskKillCleanup(uint8_t dieId, int ret, const TileXRCcuDriverAdapterReport& report) +{ + if (!TraceEndpointRoute()) { + return; + } + std::cerr << "TileXRDirectCcuTrace taskKillCleanup" + << " dieId=" << static_cast(dieId) + << " ret=" << ret + << " opcode=" << report.opcode + << " driverRet=" << report.driverRet + << " opRet=" << report.opRet + << " message=\"" << report.message << "\"" + << std::endl; +} + void TraceRaCtxEidInfos(const std::vector& eidInfos) { if (!TraceEndpointRoute()) { @@ -221,6 +237,7 @@ void TraceRaCtxEidInfos(const std::vector& eidInfos) std::cerr << "TileXRDirectCcuTrace endpointRoute raCtxEidInfo" << " ordinal=" << i << " eidIndex=" << eidInfos[i].eidIndex + << " funcId=" << eidInfos[i].funcId << " eid=" << FormatEndpointEid(CopyRawEid(eidInfos[i].eid)) << std::endl; } @@ -281,6 +298,12 @@ bool HasRaCtxResourceWindowSymbols(const TileXRCcuHccpLoader& loader) loader.RaCtxLmemUnregister != nullptr; } +bool HasRaCtxRemoteMemoryImportSymbols(const TileXRCcuHccpLoader& loader) +{ + return loader.RaCtxRmemImport != nullptr && + loader.RaCtxRmemUnimport != nullptr; +} + bool HasRaCtxEndpointRouteSymbols(const TileXRCcuHccpLoader& loader) { return loader.RaCtxCqCreate != nullptr && @@ -561,6 +584,13 @@ int TileXRCcuDirectRuntime::Init( } initialized_ = true; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + int cleanupRet = CreateDriverAdapter(&adapter, &adapterReport); + if (cleanupRet == TILEXR_SUCCESS) { + cleanupRet = adapter.CleanTaskKillState(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, &adapterReport); + } + TraceTaskKillCleanup(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, cleanupRet, adapterReport); if (report != nullptr) { report->initialized = true; report->raInitialized = true; @@ -589,6 +619,7 @@ void TileXRCcuDirectRuntime::Shutdown() endpointQpHandle_ = nullptr; endpointRemoteQpHandle_ = nullptr; endpointPeerRemoteQpHandles_.clear(); + importedRemoteMemoryBuffers_.clear(); endpointQpKey_ = TileXRCcuHccpQpKey{}; endpointQpKeyValid_ = false; endpointRouteBound_ = false; @@ -689,6 +720,11 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer(uint64_t resourceAddr) if (!IsAvailable()) { return TILEXR_ERROR_NOT_INITIALIZED; } + if (resourceWindowRegistered_ && + localResourceWindow_.addr == resourceAddr && + localResourceWindow_.raCtxRegistered == IsRaCtxResourceWindowRegistrationMode()) { + return TILEXR_SUCCESS; + } ReleaseRegisteredResourceWindow(); if (!cachedBasicInfoValid_ || resourceAddr == 0) { return TILEXR_ERROR_PARA_CHECK_FAIL; @@ -718,6 +754,127 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBuffer(uint64_t resourceAddr) return TILEXR_SUCCESS; } +int TileXRCcuDirectRuntime::RegisterMemoryBuffer( + uint64_t addr, + uint64_t bytes, + TileXRCcuRegisteredMemoryBufferInfo* info) +{ + if (info == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *info = TileXRCcuRegisteredMemoryBufferInfo {}; + if (!IsAvailable() || !resourceWindowRegistered_ || localResourceWindow_.raCtxHandle == nullptr) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (addr == 0 || bytes == 0 || !HasRaCtxResourceWindowSymbols(loader_)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + void* tokenIdHandle = nullptr; + void* lmemHandle = nullptr; + TileXRCcuHccpTokenId allocatedToken {}; + int ret = loader_.RaCtxTokenIdAlloc(localResourceWindow_.raCtxHandle, &allocatedToken, &tokenIdHandle); + if (ret != 0 || tokenIdHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + + uint32_t tokenValue = 0; + TileXRCcuRaInfo randomInfo {}; + randomInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + randomInfo.phyId = devicePhyId_; + ret = loader_.RaGetSecRandom(&randomInfo, &tokenValue); + if (ret != 0) { + if (loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(localResourceWindow_.raCtxHandle, tokenIdHandle); + } + return TILEXR_ERROR_MKIRT; + } + + const uint64_t alignedAddr = AlignResourceWindowAddr(addr); + const uint64_t alignedBytes = bytes + (addr - alignedAddr); + TileXRCcuHccpMrRegInfo mr {}; + mr.in.mem.addr = alignedAddr; + mr.in.mem.size = alignedBytes; + mr.in.ub.flags.value = 0; + mr.in.ub.flags.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + mr.in.ub.flags.bs.tokenIdValid = 1; + mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; + mr.in.ub.flags.bs.nonPin = 1; + mr.in.ub.tokenValue = tokenValue; + mr.in.ub.tokenIdHandle = tokenIdHandle; + + ret = loader_.RaCtxLmemRegister(localResourceWindow_.raCtxHandle, &mr, &lmemHandle); + if (ret != 0 || lmemHandle == nullptr) { + if (lmemHandle != nullptr && loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister(localResourceWindow_.raCtxHandle, lmemHandle); + } + if (loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(localResourceWindow_.raCtxHandle, tokenIdHandle); + } + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuRegisteredMemoryBufferInfo registeredInfo {}; + registeredInfo.addr = addr; + registeredInfo.bytes = bytes; + registeredInfo.alignedAddr = alignedAddr; + registeredInfo.alignedBytes = alignedBytes; + registeredInfo.targetSegVa = mr.out.ub.targetSegHandle + (addr - alignedAddr); + registeredInfo.rawTokenId = mr.out.ub.tokenId != 0 ? mr.out.ub.tokenId : allocatedToken.tokenId; + registeredInfo.tokenId = registeredInfo.rawTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + registeredInfo.tokenValue = tokenValue; + registeredInfo.key = mr.out.key; + registeredInfo.tokenIdHandle = tokenIdHandle; + registeredInfo.lmemHandle = lmemHandle; + registeredInfo.valid = true; + + registeredMemoryBuffers_.push_back(registeredInfo); + *info = registeredInfo; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::ImportRemoteMemoryBuffer( + const TileXRCcuRemoteMemoryBufferImportRequest& request, + TileXRCcuImportedRemoteMemoryBufferInfo* info) +{ + if (info == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *info = TileXRCcuImportedRemoteMemoryBufferInfo {}; + if (!IsAvailable() || !resourceWindowRegistered_ || localResourceWindow_.raCtxHandle == nullptr) { + return TILEXR_ERROR_NOT_INITIALIZED; + } + if (!request.valid || request.key.size == 0 || !HasRaCtxRemoteMemoryImportSymbols(loader_)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuHccpMrImportInfo mr {}; + mr.in.key = request.key; + mr.in.ub.tokenValue = request.tokenValue; + mr.in.ub.mappingAddr = 0; + mr.in.ub.flags.value = 0; + mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; + + void* rmemHandle = nullptr; + const int ret = loader_.RaCtxRmemImport(localResourceWindow_.raCtxHandle, &mr, &rmemHandle); + if (ret != 0 || rmemHandle == nullptr || mr.out.ub.targetSegHandle == 0) { + if (rmemHandle != nullptr && loader_.RaCtxRmemUnimport != nullptr) { + (void)loader_.RaCtxRmemUnimport(localResourceWindow_.raCtxHandle, rmemHandle); + } + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuImportedRemoteMemoryBufferInfo imported {}; + imported.addr = request.addr; + imported.bytes = request.bytes; + imported.targetSegVa = mr.out.ub.targetSegHandle + request.offset; + imported.rmemHandle = rmemHandle; + imported.valid = true; + importedRemoteMemoryBuffers_.push_back(imported); + *info = imported; + return TILEXR_SUCCESS; +} + int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( uint64_t resourceAddr, uint64_t resourceBytes) @@ -831,6 +988,8 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( localResourceWindow_.eid[i] = selectedEid.eid.raw[i]; } localResourceWindow_.eidIndex = selectedEid.eidIndex; + localResourceWindow_.funcId = selectedEid.funcId; + localResourceWindow_.funcIdValid = true; localResourceWindow_.raCtxRegistered = true; resourceWindowRegistered_ = true; return TILEXR_SUCCESS; @@ -838,6 +997,8 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( void TileXRCcuDirectRuntime::ReleaseRegisteredResourceWindow() { + ReleaseImportedRemoteMemoryBuffers(); + ReleaseRegisteredMemoryBuffers(); ReleaseLocalEndpointRoute(); if (localResourceWindow_.raCtxRegistered) { if (localResourceWindow_.lmemHandle != nullptr && @@ -860,6 +1021,33 @@ void TileXRCcuDirectRuntime::ReleaseRegisteredResourceWindow() localVerifiedEndpointRouteValid_ = false; } +void TileXRCcuDirectRuntime::ReleaseImportedRemoteMemoryBuffers() +{ + if (localResourceWindow_.raCtxHandle != nullptr && loader_.RaCtxRmemUnimport != nullptr) { + for (auto it = importedRemoteMemoryBuffers_.rbegin(); it != importedRemoteMemoryBuffers_.rend(); ++it) { + if (it->rmemHandle != nullptr) { + (void)loader_.RaCtxRmemUnimport(localResourceWindow_.raCtxHandle, it->rmemHandle); + } + } + } + importedRemoteMemoryBuffers_.clear(); +} + +void TileXRCcuDirectRuntime::ReleaseRegisteredMemoryBuffers() +{ + if (localResourceWindow_.raCtxHandle != nullptr) { + for (auto it = registeredMemoryBuffers_.rbegin(); it != registeredMemoryBuffers_.rend(); ++it) { + if (it->lmemHandle != nullptr && loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister(localResourceWindow_.raCtxHandle, it->lmemHandle); + } + if (it->tokenIdHandle != nullptr && loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree(localResourceWindow_.raCtxHandle, it->tokenIdHandle); + } + } + } + registeredMemoryBuffers_.clear(); +} + void TileXRCcuDirectRuntime::ReleasePeerEndpointImports() { if (localResourceWindow_.raCtxHandle != nullptr && loader_.RaCtxQpUnimport != nullptr) { @@ -1013,6 +1201,7 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( std::cerr << "TileXRDirectCcuTrace endpointRoute qpCreate ret=" << ret << " qp=" << endpointQpHandle_ << " keySize=" << static_cast(qpInfo.key.size) + << " id=" << qpInfo.ub.id << " dbAddr=0x" << std::hex << qpInfo.ub.dbAddr << " dbTokenId=0x" << qpInfo.ub.dbTokenId << std::dec << std::endl; @@ -1104,6 +1293,7 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( collected.doorbellTokenId = qpInfo.ub.dbTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; collected.doorbellTokenValue = localResourceWindow_.tokenValue; collected.sqDepth = sqDepth; + collected.startJettyId = static_cast(qpInfo.ub.id); collected.endpointRouteVerified = true; if (!HasCompleteEndpointRoute(collected)) { TraceEndpointRouteStep("collected route incomplete"); @@ -1116,6 +1306,7 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( << " doorbellVa=0x" << collected.doorbellVa << " doorbellTokenId=0x" << collected.doorbellTokenId << std::dec + << " startJettyId=" << collected.startJettyId << " sqDepth=" << collected.sqDepth << std::endl; } *route = collected; @@ -1329,6 +1520,7 @@ int TileXRCcuDirectRuntime::ImportPeerEndpointRoute( importedRoute->doorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId; importedRoute->doorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue; importedRoute->sqDepth = localVerifiedEndpointRoute_.sqDepth; + importedRoute->startJettyId = localVerifiedEndpointRoute_.startJettyId; importedRoute->endpointRouteVerified = true; return HasCompleteEndpointRoute(*importedRoute) ? TILEXR_SUCCESS : TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -1359,6 +1551,8 @@ int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers(std::vector(verifiedStartJettyId, route.startJettyId); + verifiedJettyEnd = std::max( + verifiedJettyEnd, + static_cast(route.startJettyId) + 1U); + } } result.routes.push_back(route); } + if (verifiedStartJettyId != 0) { + result.startJettyId = verifiedStartJettyId; + const uint32_t requiredJettyCount = verifiedJettyEnd - verifiedStartJettyId; + result.pfeJettyCount = CheckedU16(std::max(result.pfeJettyCount, requiredJettyCount)); + } *snapshot = result; FillTemplateReport(*snapshot, report); diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h index 26ff8c7f..c37f908a 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h @@ -106,6 +106,9 @@ struct TileXRCcuRemoteCcuBufferInfo { uint32_t doorbellTokenId = 0; uint32_t doorbellTokenValue = 0; uint32_t sqDepth = 0; + uint16_t startJettyId = 0; + uint32_t localPfeId = 0; + bool localPfeIdValid = false; uint64_t localDoorbellVa = 0; uint32_t localDoorbellTokenId = 0; uint32_t localDoorbellTokenValue = 0; diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.cpp b/src/comm/ccu/tilexr_ccu_runtime_session.cpp index 63cbdf87..a9001a20 100644 --- a/src/comm/ccu/tilexr_ccu_runtime_session.cpp +++ b/src/comm/ccu/tilexr_ccu_runtime_session.cpp @@ -77,6 +77,11 @@ int TileXRCcuRuntimeSession::RankSize() const return rankSize_; } +int TileXRCcuRuntimeSession::DevId() const +{ + return devId_; +} + int TileXRCcuRuntimeSession::Init(const TileXRCcuBackendOptions &options) { Shutdown(); @@ -203,6 +208,27 @@ int TileXRCcuRuntimeSession::RegisterCcuResourceRmaBuffer(uint64_t resourceAddr) return ccuDirectRuntime_->RegisterCcuResourceRmaBuffer(resourceAddr); } +int TileXRCcuRuntimeSession::RegisterMemoryBuffer( + uint64_t addr, + uint64_t bytes, + TileXRCcuRegisteredMemoryBufferInfo *info) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->RegisterMemoryBuffer(addr, bytes, info); +} + +int TileXRCcuRuntimeSession::ImportRemoteMemoryBuffer( + const TileXRCcuRemoteMemoryBufferImportRequest &request, + TileXRCcuImportedRemoteMemoryBufferInfo *info) +{ + if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { + return TILEXR_ERROR_NOT_FOUND; + } + return ccuDirectRuntime_->ImportRemoteMemoryBuffer(request, info); +} + int TileXRCcuRuntimeSession::ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo *info) { if (ccuDirectRuntime_ == nullptr || !ccuDirectRuntime_->IsAvailable()) { diff --git a/src/comm/ccu/tilexr_ccu_runtime_session.h b/src/comm/ccu/tilexr_ccu_runtime_session.h index dbcc16dd..cde0802c 100644 --- a/src/comm/ccu/tilexr_ccu_runtime_session.h +++ b/src/comm/ccu/tilexr_ccu_runtime_session.h @@ -23,6 +23,7 @@ class TileXRCcuRuntimeSession { int Rank() const; int RankSize() const; + int DevId() const; int AllGather(const void *sendBuf, size_t sendBytes, void *recvBuf); int RefreshDirectCcuBasicInfo(uint8_t dieId = 0); @@ -32,6 +33,10 @@ class TileXRCcuRuntimeSession { const TileXRCcuDriverAdapterReport &GetDirectCcuBasicInfoReport() const; int RegisterCcuResourceRmaBuffer(uint64_t resourceAddr); + int RegisterMemoryBuffer(uint64_t addr, uint64_t bytes, TileXRCcuRegisteredMemoryBufferInfo *info); + int ImportRemoteMemoryBuffer( + const TileXRCcuRemoteMemoryBufferImportRequest &request, + TileXRCcuImportedRemoteMemoryBufferInfo *info); int ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindowInfo *info); int ExportRemoteCcuRmaBuffers(std::vector *buffers); int ExportLowerLayerTransportSnapshot( diff --git a/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp b/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp new file mode 100644 index 00000000..1ed253d6 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_signal_wait_program.h" + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuBarrierProgramReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuBarrierProgramReport {}; + } +} + +int Fail( + std::vector* program, + TileXRCcuBarrierProgramReport* report, + const std::string& message) +{ + if (program != nullptr) { + program->clear(); + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool HasSignalResources(const TileXRCcuSignalWaitProgramSpec& spec) +{ + return spec.channelId != 0 && + spec.remoteNotifyCke != 0 && + spec.remoteNotifyMask != 0 && + spec.sourceCke != 0 && + spec.sourceCkeMask != 0; +} + +bool HasWaitResources(const TileXRCcuSignalWaitProgramSpec& spec) +{ + return spec.localWaitCke != 0 && + spec.localWaitMask != 0; +} + +TileXRCcuBarrierSyncSpec ToBarrierSpec(const TileXRCcuSignalWaitProgramSpec& spec) +{ + TileXRCcuBarrierSyncSpec barrier; + barrier.channelId = spec.channelId; + barrier.remoteXn = spec.remoteXn; + barrier.localXn = spec.localXn; + barrier.remoteNotifyCke = spec.remoteNotifyCke; + barrier.remoteNotifyMask = spec.remoteNotifyMask; + barrier.localWaitCke = spec.localWaitCke; + barrier.localWaitMask = spec.localWaitMask; + barrier.sourceCke = spec.sourceCke; + barrier.sourceCkeMask = spec.sourceCkeMask; + barrier.clearLocalWait = spec.clearLocalWait; + return barrier; +} + +int AppendWaitInstruction( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report, + bool setWait) +{ + if (!HasWaitResources(spec)) { + return Fail(program, report, "missing wait CKE resource for direct CCU signal/wait program"); + } + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.localWaitCke; + wait.waitMask = spec.localWaitMask; + wait.clearWait = spec.clearLocalWait; + TileXRCcuInstr instr; + const int ret = setWait ? + TileXRCcuEncodeSetCke(wait, &instr) : + TileXRCcuEncodeClearCke(wait, &instr); + if (ret != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode wait instruction for direct CCU signal/wait program"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendCommonPrelude( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + if (spec.localXn == 0 || spec.sourceCke == 0 || spec.sourceCkeMask == 0) { + return Fail(program, report, "missing reserve resource for direct CCU signal/wait program"); + } + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal/wait reserve XN prelude"); + } + program->push_back(instr); + + if (spec.localGsa != 0) { + if (TileXRCcuEncodeLoadImdToGsa(spec.localGsa, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal/wait reserve GSA prelude"); + } + } else if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal/wait reserve address prelude"); + } + program->push_back(instr); + + TileXRCcuCkeSpec init; + init.ckeId = spec.sourceCke; + init.mask = spec.sourceCkeMask; + init.clearWait = true; + if (TileXRCcuEncodeSetCke(init, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal/wait reserve CKE prelude"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendFinish( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal/wait finish instruction"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendSignalInstruction( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + if (!HasSignalResources(spec)) { + return Fail(program, report, "missing signal resource for direct CCU signal/wait program"); + } + TileXRCcuSyncCkeSpec post; + post.remoteCke = spec.remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = spec.sourceCkeMask; + post.channelId = spec.channelId; + post.clearWait = spec.clearLocalWait; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode signal instruction for direct CCU signal/wait program"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int BuildWaitOnly( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + program->clear(); + const int ret = AppendWaitInstruction(spec, program, report, false); + if (ret != TILEXR_SUCCESS) { + return ret; + } + if (report != nullptr) { + report->postInstructionCount = 0; + report->waitInstructionCount = 1; + report->totalInstructionCount = static_cast(program->size()); + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +} // namespace + +int TileXRCcuBuildSignalWaitProgram( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report) +{ + ResetReport(report); + if (program == nullptr) { + return Fail(program, report, "missing output direct CCU signal/wait program"); + } + program->clear(); + + if (spec.role == TileXRCcuSignalWaitProgramRole::Wait) { + return BuildWaitOnly(spec, program, report); + } + + if (!HasSignalResources(spec)) { + return Fail(program, report, "missing signal resource for direct CCU signal/wait program"); + } + if (spec.role == TileXRCcuSignalWaitProgramRole::SignalAndWait && !HasWaitResources(spec)) { + return Fail(program, report, "missing wait CKE resource for direct CCU signal/wait program"); + } + + if (AppendSignalInstruction(spec, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (spec.role == TileXRCcuSignalWaitProgramRole::SignalAndWait && + AppendWaitInstruction(spec, program, report, false) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (report != nullptr) { + report->postInstructionCount = 1; + report->waitInstructionCount = spec.role == TileXRCcuSignalWaitProgramRole::SignalAndWait ? 1U : 0U; + report->totalInstructionCount = static_cast(program->size()); + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_signal_wait_program.h b/src/comm/ccu/tilexr_ccu_signal_wait_program.h new file mode 100644 index 00000000..63cbb7e8 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_signal_wait_program.h @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_SIGNAL_WAIT_PROGRAM_H +#define TILEXR_CCU_SIGNAL_WAIT_PROGRAM_H + +#include "ccu/tilexr_ccu_barrier_program.h" + +namespace TileXR { + +enum class TileXRCcuSignalWaitProgramRole { + Signal = 0, + Wait = 1, + SignalAndWait = 2, +}; + +struct TileXRCcuSignalWaitProgramSpec { + TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; + uint16_t channelId = 0; + uint16_t remoteXn = 0; + uint16_t localXn = 0; + uint16_t localGsa = 0; + uint16_t remoteNotifyCke = 0; + uint16_t remoteNotifyMask = 0; + uint16_t localWaitCke = 0; + uint16_t localWaitMask = 0; + uint16_t sourceCke = 0; + uint16_t sourceCkeMask = 0; + bool clearLocalWait = true; +}; + +int TileXRCcuBuildSignalWaitProgram( + const TileXRCcuSignalWaitProgramSpec& spec, + std::vector* program, + TileXRCcuBarrierProgramReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_SIGNAL_WAIT_PROGRAM_H diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index a2cd1403..979798fc 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -11,7 +11,9 @@ #include "acl/acl_rt.h" #include "tilexr_api.h" #include "tilexr_types.h" +#include "ccu/tilexr_ccu_backend.h" #include "ccu/tilexr_ccu_collective_planner.h" +#include "ccu/tilexr_ccu_driver_adapter.h" #include "ccu/tilexr_ccu_executor.h" #include "ccu/tilexr_ccu_runtime_session.h" #include "tools/socket/tilexr_sock_exchange.h" @@ -39,6 +41,7 @@ struct DirectCcuSmokeContext { TileXR::TileXRCcuRuntimeSession session; TileXR::TileXRCcuCollectivePlanner planner; TileXR::TileXRCcuExecutor executor; + TileXR::TileXRCcuBackend backend; }; using TileXRDirectCcuPrepareOptions = TileXR::TileXRCcuDirectInstallOptions; @@ -73,6 +76,10 @@ constexpr const char* kExpectP2pCcuCopyEnv = "TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P constexpr const char* kP2pCcuCopyBytesEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES"; constexpr const char* kP2pCcuCopyActiveRankEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK"; constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION"; +constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; +constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; +constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; +constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; constexpr const char* kLocalWaitCkeStartEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START"; constexpr const char* kLocalWaitCkeCountEnv = "TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT"; constexpr const char* kRemoteNotifyCkeStartEnv = "TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START"; @@ -130,6 +137,16 @@ bool EnvFlag(const char* name) return text == "1" || text == "true" || text == "TRUE" || text == "yes" || text == "on"; } +bool SignalWaitSmokeEnabled() +{ + return EnvFlag(kSignalWaitEnv); +} + +bool BarrierSmokeEnabled() +{ + return EnvFlag(kSignalWaitBarrierEnv); +} + bool ShouldFastExitAfterPrepareFailure(int ret) { return ret != 0 && EnvFlag(kFastExitOnPrepareFailureEnv); @@ -283,6 +300,32 @@ bool SyncXnLoadPostOnlyBarrierMode() return value != nullptr && std::string(value) == "sync_xn_load_post_only"; } +TileXR::TileXRCcuBarrierMode BarrierModeFromEnv() +{ + if (SyncCkeBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::SyncCke; + } + if (SyncCkeSetWaitBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::SyncCkeSetWait; + } + if (SyncCkePostOnlyBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::SyncCkePostOnly; + } + if (LocalCkeBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::LocalCke; + } + if (LocalCkePostOnlyBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::LocalCkePostOnly; + } + if (SyncXnPostOnlyBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::SyncXnPostOnly; + } + if (SyncXnLoadPostOnlyBarrierMode()) { + return TileXR::TileXRCcuBarrierMode::SyncXnLoadPostOnly; + } + return TileXR::TileXRCcuBarrierMode::SyncXn; +} + TileXR::TileXRCcuRepositoryInstallWindow RepositoryInstallWindowFromEnv() { const char* value = std::getenv(kRepositoryInstallWindowEnv); @@ -525,7 +568,11 @@ int RunP2pCcuCopy( return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } if (ShouldCheckActiveP2pCcuCopyRank() && syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { - (void)CheckP2pCcuCopyState(state); + if (EnvFlag(kP2pCcuCopyResourceWindowEnv)) { + state->passed = true; + } else { + (void)CheckP2pCcuCopyState(state); + } } else if (!ShouldCheckActiveP2pCcuCopyRank() && syncRet == ACL_SUCCESS && submitRet == TileXR::TILEXR_SUCCESS) { state->passed = true; } @@ -592,6 +639,16 @@ int InitCommForDirectCcuSmoke(int commDomain, int rankSize, int rank, int device << " directOnly=" << (EnvFlag(kDirectCcuOnlyInitEnv) ? 1 : 0) << std::endl; } + if (ret == TileXR::TILEXR_SUCCESS && (SignalWaitSmokeEnabled() || BarrierSmokeEnabled())) { + const int backendRet = context->backend.Init(options); + std::cout << "tilexr_ccu_signal_wait backendInit" + << " rank=" << rank + << " rankSize=" << rankSize + << " device=" << device + << " ret=" << backendRet + << std::endl; + return backendRet; + } return ret; } @@ -630,12 +687,100 @@ TileXRDirectCcuPrepareOptions MakePrepareOptions(int rank, int rankSize, int dev options.repositoryInstallOptions.dataLenMode = RepositoryInstallDataLenModeFromEnv(); options.repositoryMemoryAllocMode = RepositoryMemoryAllocModeFromEnv(); options.installOrder = InstallOrderFromEnv(); + options.barrierMode = BarrierModeFromEnv(); + options.taskTimeout = static_cast(EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 0)); options.deviceId = static_cast(device); options.rank = static_cast(rank); options.provider = rankSize > 0 ? "tilexr-direct-smoke-probe" : ""; return options; } +const char* SignalWaitRoleName(TileXR::TileXRCcuSignalWaitRole role) +{ + switch (role) { + case TileXR::TileXRCcuSignalWaitRole::Signal: + return "signal"; + case TileXR::TileXRCcuSignalWaitRole::Wait: + return "wait"; + case TileXR::TileXRCcuSignalWaitRole::SignalAndWait: + return "signal_and_wait"; + default: + return "unknown"; + } +} + +TileXR::TileXRCcuSignalWaitRole SignalWaitRoleForRank(int rank) +{ + if (BarrierSmokeEnabled()) { + return TileXR::TileXRCcuSignalWaitRole::SignalAndWait; + } + const int signalRank = EnvInt(kSignalWaitSignalRankEnv, 0); + return rank == signalRank ? TileXR::TileXRCcuSignalWaitRole::Signal : TileXR::TileXRCcuSignalWaitRole::Wait; +} + +uint32_t DefaultSignalWaitInstructionCount(TileXR::TileXRCcuSignalWaitRole role) +{ + switch (role) { + case TileXR::TileXRCcuSignalWaitRole::Signal: + return 5U; + case TileXR::TileXRCcuSignalWaitRole::Wait: + return 5U; + case TileXR::TileXRCcuSignalWaitRole::SignalAndWait: + return 6U; + default: + return 0U; + } +} + +TileXR::TileXRCcuSignalWaitRequest MakeSignalWaitRequest( + int rank, + int rankSize, + const TileXRDirectCcuPrepareOptions& options) +{ + TileXR::TileXRCcuSignalWaitRequest request {}; + request.peerRank = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; + request.role = SignalWaitRoleForRank(rank); + request.syncInstructionCount = std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr ? + 0U : + options.syncInstructionCount; + request.missionStartId = options.missionStartId; + request.instructionStartId = options.instructionStartId; + request.missionInstructionStartId = options.missionInstructionStartId; + request.xnStartId = options.xnStartId; + request.remoteXnStartId = options.remoteXnStartId; + request.remoteXnCount = options.remoteXnCount; + request.ckeStartId = options.ckeStartId; + request.channelStartId = options.channelStartId; + request.localWaitCkeStartId = options.localWaitCkeStartId; + request.localWaitCkeCount = options.localWaitCkeCount; + request.remoteNotifyCkeStartId = options.remoteNotifyCkeStartId; + request.remoteNotifyCkeCount = options.remoteNotifyCkeCount; + request.timeout = static_cast(EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 20)); + request.provider = "tilexr-direct-smoke-probe-signal-wait"; + return request; +} + +TileXRDirectCcuPrepareReport SignalWaitInstallReportFromPlan( + const TileXR::TileXRCcuSignalWaitPlan& plan) +{ + TileXRDirectCcuPrepareReport report {}; + report.pipelineBuilt = !plan.attempt.plan.taskWindows.empty(); + report.installAttempted = plan.attempt.installReport.installAttempted; + report.installSucceeded = plan.attempt.installReport.installSucceeded; + report.submitReady = plan.attempt.providerReport.submitReady; + report.requiredInstallSurfaceCount = plan.attempt.installReport.requiredInstallSurfaceCount; + report.publicVerifiedInstallSurfaceCount = plan.attempt.installReport.publicVerifiedInstallSurfaceCount; + report.missingInstallSurfaceCount = plan.attempt.installReport.missingInstallSurfaceCount; + report.taskCount = static_cast(plan.attempt.plan.taskWindows.size()); + report.submitTaskCount = static_cast(plan.submitTasks.size()); + if (!plan.attempt.providerReport.message.empty()) { + report.message = plan.attempt.providerReport.message; + } else { + report.message = plan.attempt.installReport.message; + } + return report; +} + void PrintInstallReport( const char* prefix, int ret, @@ -870,6 +1015,79 @@ void PrintPreparedTasks(TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskC std::cout << std::endl; } +uint16_t LoadLe16(const uint8_t* data, size_t index) +{ + return static_cast(data[index * 2U]) | + static_cast(static_cast(data[index * 2U + 1U]) << 8U); +} + +void PrintMissionContext( + DirectCcuSmokeContext* context, + const TileXRDirectCcuTaskInfo& task, + const char* label) +{ + if (context == nullptr || label == nullptr) { + return; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport adapterReport; + int ret = context->session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return; + } + + uint8_t raw[TileXR::TILEXR_CCU_DATA_ARRAY_SLOT_BYTES] = {}; + ret = adapter.ReadMissionContext( + task.dieId, + task.missionId, + raw, + sizeof(raw), + &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " opcode=" << adapterReport.opcode + << " driverRet=" << adapterReport.driverRet + << " opRet=" << adapterReport.opRet + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return; + } + + const uint16_t part2 = LoadLe16(raw, 2); + const uint16_t part3 = LoadLe16(raw, 3); + const uint16_t part4 = LoadLe16(raw, 4); + const uint16_t part5 = LoadLe16(raw, 5); + const uint16_t part6 = LoadLe16(raw, 6); + const uint16_t part7 = LoadLe16(raw, 7); + const uint16_t status = static_cast(((part3 & 0x7U) << 13U) | ((part2 >> 3U) & 0x1fffU)); + const uint16_t currentIns = static_cast(((part5 & 0x1fU) << 11U) | ((part4 >> 5U) & 0x7ffU)); + const uint16_t endIns = static_cast(((part6 & 0x1fU) << 11U) | ((part5 >> 5U) & 0x7ffU)); + const uint16_t startIns = static_cast(((part7 & 0x1fU) << 11U) | ((part6 >> 5U) & 0x7ffU)); + const uint16_t missionVld = static_cast((part7 >> 6U) & 0x1U); + + std::cerr << label << " missionCtx" + << " dieId=" << static_cast(task.dieId) + << " missionId=" << static_cast(task.missionId) + << " status=0x" << std::hex << status + << " currentIns=" << std::dec << currentIns + << " startIns=" << startIns + << " endIns=" << endIns + << " missionVld=" << missionVld + << " rawWords="; + for (size_t i = 0; i < sizeof(raw) / sizeof(uint64_t); ++i) { + uint64_t word = 0; + std::memcpy(&word, raw + i * sizeof(uint64_t), sizeof(word)); + if (i != 0) { + std::cerr << ","; + } + std::cerr << "0x" << std::hex << word; + } + std::cerr << std::dec << std::endl; +} + void PrintInstructionReadback(DirectCcuSmokeContext* context, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) { if (!EnvFlag(kReadbackInstructionsEnv)) { @@ -1138,8 +1356,143 @@ int RunInactiveP2pCcuCopyRank(int rank, int peer, int rankSize, P2pCcuCopyState* return checkRet == ACL_SUCCESS ? 0 : 14; } +int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + const TileXR::TileXRCcuSignalWaitRole role = SignalWaitRoleForRank(rank); + if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { + options.syncInstructionCount = DefaultSignalWaitInstructionCount(role); + } + PrintConfig(options, rankSize); + + const TileXR::TileXRCcuSignalWaitRequest request = MakeSignalWaitRequest(rank, rankSize, options); + std::cout << "tilexr_ccu_signal_wait config" + << " rank=" << rank + << " peer=" << request.peerRank + << " role=" << SignalWaitRoleName(request.role) + << " barrier=" << (BarrierSmokeEnabled() ? 1 : 0) + << std::endl; + + TileXR::TileXRCcuSignalWaitPlan plan; + const int prepareRet = context->backend.PrepareSignalWait(request, &plan); + TileXRDirectCcuPrepareReport installReport = SignalWaitInstallReportFromPlan(plan); + PrintInstallReport("tilexr_ccu_signal_wait prepare", prepareRet, installReport); + PrintPreparedTasks(&plan.attempt, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady && plan.ready) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_signal_wait submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && (!installReport.submitReady || !plan.ready)) { + std::cout << "tilexr_ccu_signal_wait submit skipped reason=\"prepare did not reach submitReady\"" + << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_signal_wait aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + const int delayRank = EnvInt(kDelayRankEnv, -1); + const int preSubmitDelayMs = EnvInt(kPreSubmitDelayMsEnv, 0); + const int effectiveDelayMs = rank == delayRank && preSubmitDelayMs > 0 ? preSubmitDelayMs : 0; + if (effectiveDelayMs > 0) { + std::cout << "tilexr_ccu_signal_wait preSubmitDelay" + << " rank=" << rank + << " delayMs=" << effectiveDelayMs + << std::endl; + std::this_thread::sleep_for(std::chrono::milliseconds(effectiveDelayMs)); + } + + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = context->backend.SubmitSignalWait(plan, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_signal_wait submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + TraceLifecycle("before signal/wait aclrtSynchronizeStream"); + const int syncRet = aclrtSynchronizeStream(stream); + TraceLifecycle("after signal/wait aclrtSynchronizeStream"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_signal_wait timing" + << " rank=" << rank + << " role=" << SignalWaitRoleName(request.role) + << " preSubmitDelayMs=" << effectiveDelayMs + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_signal_wait aclrtSynchronizeStream ret=" << syncRet << std::endl; + if (!plan.submitTasks.empty()) { + PrintMissionContext(context, plan.submitTasks.front(), "tilexr_ccu_signal_wait"); + } + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + aclrtDestroyStream(stream); + } + } + + const bool passed = finalRet == 0; + if (passed) { + std::cout << "tilexr_ccu_signal_wait result passed=1" + << " rank=" << rank + << " role=" << SignalWaitRoleName(request.role) + << " ret=" << finalRet + << std::endl; + } else { + std::cout << "tilexr_ccu_signal_wait result passed=0" + << " rank=" << rank + << " role=" << SignalWaitRoleName(request.role) + << " ret=" << finalRet + << std::endl; + } + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_signal_wait fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + const int releaseRet = TileXR::TileXRCcuReleaseDirectInstallAttemptResources(plan.attempt); + if (releaseRet != TileXR::TILEXR_SUCCESS) { + std::cerr << "tilexr_ccu_signal_wait prepared destroy ret=" << releaseRet << std::endl; + finalRet = finalRet == 0 ? 11 : finalRet; + } + return finalRet; +} + int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { + if (SignalWaitSmokeEnabled() || BarrierSmokeEnabled()) { + return RunSignalWaitSmokeForRank(context, rank, rankSize, device); + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); const int peer = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; const bool p2pCcuCopyEnabled = EnvFlag(kP2pCcuCopyEnv); @@ -1236,6 +1589,9 @@ int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi std::chrono::duration_cast(syncEnd - syncBegin).count()); if (syncRet != ACL_SUCCESS) { std::cerr << "tilexr_ccu_direct_smoke aclrtSynchronizeStream ret=" << syncRet << std::endl; + if (prepared != nullptr && installReport.submitTaskCount > 0) { + PrintMissionContext(context, prepared->submitTasks.front(), "tilexr_ccu_direct_smoke"); + } finalRet = 8; } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; @@ -1435,6 +1791,7 @@ int main() std::_Exit(finalRet); } TraceLifecycle("before DirectCcuSmokeContext shutdown"); + context.backend.Shutdown(); context.session.Shutdown(); TraceLifecycle("after DirectCcuSmokeContext shutdown"); TraceLifecycle("before aclrtResetDevice"); diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 30df58e3..e8fac065 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -81,6 +81,12 @@ default_sync_instruction_count() esac } +signal_wait_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0}" = "1" ] || + [ "${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0}" = "1" ] +} + apply_p2p_ccu_copy_defaults() { if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" != "1" ]; then @@ -113,9 +119,43 @@ apply_p2p_ccu_copy_defaults() export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" export TILEXR_CCU_DIRECT_BARRIER_MODE="${TILEXR_CCU_DIRECT_BARRIER_MODE:-sync_cke}" export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" + export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}" +} + +apply_signal_wait_defaults() +{ + if ! signal_wait_mode_enabled; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK="${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" } apply_p2p_ccu_copy_defaults +apply_signal_wait_defaults if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" @@ -127,6 +167,9 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE \ TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE \ TILEXR_CCU_DIRECT_INSTALL_ORDER \ + TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT \ + TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK \ + TILEXR_CCU_DIRECT_SMOKE_BARRIER \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do diagnostic_value="${!diagnostic_var:-}" @@ -337,6 +380,12 @@ fi if [ "${TILEXR_CCU_DIRECT_INSTALL_DIE_ID:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_INSTALL_DIE_ID=${TILEXR_CCU_DIRECT_INSTALL_DIE_ID}") fi +if [ "${TILEXR_CCU_DIRECT_TRACE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_TRACE=${TILEXR_CCU_DIRECT_TRACE}") +fi +if [ "${TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=${TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE}") +fi if [ "${TILEXR_CCU_PROBE_MISSION_START:-}" != "" ]; then common_env+=("TILEXR_CCU_PROBE_MISSION_START=${TILEXR_CCU_PROBE_MISSION_START}") fi @@ -379,6 +428,27 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_TRACE_LIFECYCLE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_TRACE_LIFECYCLE=${TILEXR_CCU_DIRECT_SMOKE_TRACE_LIFECYCLE}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_BARRIER=${TILEXR_CCU_DIRECT_SMOKE_BARRIER}") +fi +if [ "${TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=${TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT}") +fi if [ "${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-}" != "" ]; then common_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START}") fi @@ -489,7 +559,7 @@ for token_field in "${resource_window_token_fields[@]}"; do fi done -echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" +echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then thread_log="${work_dir}/ccu_thread.log" @@ -504,27 +574,56 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then echo "thread log: ${thread_log}" >&2 exit 4 fi - if [ "$(grep -c "tilexr_ccu_direct_smoke prepare ret=0" "${thread_log}")" -lt 2 ]; then - echo "ERROR: direct CCU thread-mode prepare did not return success for both ranks" >&2 - exit 5 - fi - if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then - echo "ERROR: direct CCU thread-mode prepare did not complete install attempt for both ranks" >&2 - exit 6 - fi - if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then - if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then - echo "ERROR: direct CCU thread-mode submit requested but prepare did not reach submitReady=1" >&2 + if signal_wait_mode_enabled; then + if [ "$(grep -c "tilexr_ccu_signal_wait prepare ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode prepare did not return success for both ranks" >&2 + exit 5 + fi + if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode prepare did not complete install attempt for both ranks" >&2 exit 6 fi - if [ "$(grep -c "tilexr_ccu_direct_smoke submit ret=0" "${thread_log}")" -lt 2 ]; then - echo "ERROR: direct CCU thread-mode submit did not return success for both ranks" >&2 - exit 7 + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode submit requested but prepare did not reach submitReady=1" >&2 + exit 6 + fi + if [ "$(grep -c "tilexr_ccu_signal_wait submit ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode submit did not return success for both ranks" >&2 + exit 7 + fi + if [ "$(grep -c "tilexr_ccu_signal_wait timing" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode timing was not reported for both ranks" >&2 + exit 8 + fi fi - if [ "$(grep -c "tilexr_ccu_direct_smoke submitTiming" "${thread_log}")" -lt 2 ]; then - echo "ERROR: direct CCU thread-mode submit timing was not reported for both ranks" >&2 + if [ "$(grep -c "tilexr_ccu_signal_wait result passed=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU signal/wait thread-mode result did not pass for both ranks" >&2 exit 8 fi + else + if [ "$(grep -c "tilexr_ccu_direct_smoke prepare ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode prepare did not return success for both ranks" >&2 + exit 5 + fi + if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode prepare did not complete install attempt for both ranks" >&2 + exit 6 + fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit requested but prepare did not reach submitReady=1" >&2 + exit 6 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke submit ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit did not return success for both ranks" >&2 + exit 7 + fi + if [ "$(grep -c "tilexr_ccu_direct_smoke submitTiming" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU thread-mode submit timing was not reported for both ranks" >&2 + exit 8 + fi + fi fi if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then @@ -567,16 +666,29 @@ if [ "${rank0_status}" -ne 0 ] || [ "${rank1_status}" -ne 0 ]; then exit 4 fi -for log in "${rank0_log}" "${rank1_log}"; do - if ! grep -q "tilexr_ccu_direct_smoke prepare ret=0" "${log}"; then - echo "ERROR: direct CCU prepare did not return success in ${log}" >&2 - exit 5 - fi - if ! grep -q "installSucceeded=1" "${log}"; then - echo "ERROR: direct CCU prepare did not complete install attempt in ${log}" >&2 - exit 6 - fi -done +if signal_wait_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_signal_wait prepare ret=0" "${log}"; then + echo "ERROR: direct CCU signal/wait prepare did not return success in ${log}" >&2 + exit 5 + fi + if ! grep -q "installSucceeded=1" "${log}"; then + echo "ERROR: direct CCU signal/wait prepare did not complete install attempt in ${log}" >&2 + exit 6 + fi + done +else + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_direct_smoke prepare ret=0" "${log}"; then + echo "ERROR: direct CCU prepare did not return success in ${log}" >&2 + exit 5 + fi + if ! grep -q "installSucceeded=1" "${log}"; then + echo "ERROR: direct CCU prepare did not complete install attempt in ${log}" >&2 + exit 6 + fi + done +fi rank_skipped_p2p_ccu_copy_submit() { @@ -593,6 +705,17 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then fi done for log in "${rank0_log}" "${rank1_log}"; do + if signal_wait_mode_enabled; then + if ! grep -q "tilexr_ccu_signal_wait submit ret=0" "${log}"; then + echo "ERROR: direct CCU signal/wait submit did not return success in ${log}" >&2 + exit 7 + fi + if ! grep -q "tilexr_ccu_signal_wait timing" "${log}"; then + echo "ERROR: direct CCU signal/wait timing was not reported in ${log}" >&2 + exit 8 + fi + continue + fi if ! grep -q "tilexr_ccu_direct_smoke submit ret=0" "${log}"; then if rank_skipped_p2p_ccu_copy_submit "${log}"; then continue @@ -610,6 +733,15 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then done fi +if signal_wait_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then + echo "ERROR: direct CCU signal/wait result did not pass in ${log}" >&2 + exit 9 + fi + done +fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]; then if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" != "1" ]; then echo "ERROR: direct CCU barrier wait check requires TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1" >&2 @@ -624,7 +756,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]; then fi wait_sync_ms="$( awk ' - /tilexr_ccu_direct_smoke submitTiming/ { + /tilexr_ccu_direct_smoke submitTiming|tilexr_ccu_signal_wait timing/ { for (i = 1; i <= NF; ++i) { if ($i ~ /^syncMs=/) { split($i, parts, "="); diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index c999cdcb..55e645b8 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -9,6 +9,7 @@ REPO_ROOT = Path(__file__).resolve().parents[2] COMM_HEADER = REPO_ROOT / "src" / "comm" / "tilexr_comm.h" +PUBLIC_API_HEADER = REPO_ROOT / "src" / "include" / "tilexr_api.h" BACKEND_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.h" BACKEND_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_backend.cpp" RUNTIME_SESSION_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime_session.h" @@ -54,11 +55,21 @@ def test_tilexr_comm_header_owns_only_opaque_backend(self): def test_backend_header_exposes_facade_not_public_c_api(self): header = BACKEND_HEADER.read_text(encoding="utf-8") + public_api = PUBLIC_API_HEADER.read_text(encoding="utf-8") self.assertIn("class TileXRCcuBackend", header) self.assertIn("struct TileXRCcuBackendOptions", header) self.assertIn("TileXRSockExchange *exchange", header) self.assertIn("PrepareCollective", header) self.assertIn("SubmitCollective", header) + for needle in [ + "enum class TileXRCcuSignalWaitRole", + "struct TileXRCcuSignalWaitRequest", + "struct TileXRCcuSignalWaitPlan", + "PrepareSignalWait", + "SubmitSignalWait", + ]: + with self.subTest(internal=needle): + self.assertIn(needle, header) for needle in [ "TileXRDirectCcuPreparedTasksPtr", "TileXRCommPrepareDirectCcu", @@ -66,6 +77,14 @@ def test_backend_header_exposes_facade_not_public_c_api(self): ]: with self.subTest(needle=needle): self.assertNotIn(needle, header) + self.assertNotIn(needle, public_api) + for needle in [ + "TileXRCcuSignalWait", + "PrepareSignalWait", + "SubmitSignalWait", + ]: + with self.subTest(public_needle=needle): + self.assertNotIn(needle, public_api) def test_split_sources_own_restored_direct_ccu_runtime_glue(self): source = BACKEND_SOURCE.read_text(encoding="utf-8") @@ -92,13 +111,18 @@ def test_split_sources_own_restored_direct_ccu_runtime_glue(self): for needle in [ "#include \"ccu/tilexr_ccu_repository.h\"", "PrepareDirectCcuInstallAttempt", + "PrepareSignalWait", "PrepareDirectCcuLowerLayerPlanCallback", "TileXRCcuRunDirectInstallAttempt", + "TileXRCcuRunDirectSignalWaitInstallAttempt", "TileXRCcuMakeRepositoryDeviceMemoryOps", ]: with self.subTest(needle=needle): self.assertIn(needle, planner) + self.assertIn("planner_->PrepareSignalWait", source) + self.assertNotIn("return TILEXR_ERROR_NOT_SUPPORT;", source[source.index("PrepareSignalWait"):]) + planner_header = PLANNER_HEADER.read_text(encoding="utf-8") executor_header = EXECUTOR_HEADER.read_text(encoding="utf-8") self.assertRegex( @@ -153,6 +177,52 @@ def test_tilexr_comm_can_auto_initialize_ccu_backend_without_blocking_comm_init( self.assertIn("return TILEXR_SUCCESS;", helper) self.assertNotIn("return ccuRet;", helper) + def test_p2p_ccu_copy_process_token_is_not_urma_shifted(self): + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + token_query = planner[ + planner.index("int QueryDirectCcuProcessMemoryToken"): + planner.index("int BuildDirectCcuLocalMemoryCopyEndpoint") + ] + + self.assertIn("rtUbDevQueryInfo(QUERY_PROCESS_TOKEN, &info)", token_query) + self.assertIn("const uint32_t tokenId = info.tokenId;", token_query) + self.assertIn("TileXRCcuPackMemoryToken(tokenId, info.tokenValue, true)", token_query) + self.assertNotIn("info.tokenId >>", token_query) + + def test_p2p_ccu_copy_uses_original_va_for_microcode_and_imported_segva_for_route(self): + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + endpoint_builder = planner[ + planner.index("int BuildDirectCcuLocalMemoryCopyEndpoint"): + planner.index("void TileXRCcuCollectivePlanner::Reset") + ] + prepare_copy = planner[ + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt"): + planner.index("int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan") + ] + + self.assertIn("session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr)", prepare_copy) + self.assertIn("session.RegisterMemoryBuffer(sourceAddr, bytes, &sourceInfo)", endpoint_builder) + self.assertIn("session.RegisterMemoryBuffer(destinationAddr, bytes, &destinationInfo)", endpoint_builder) + self.assertIn("endpoint->sourceAddr = sourceInfo.addr", endpoint_builder) + self.assertIn("endpoint->destinationAddr = destinationInfo.addr", endpoint_builder) + self.assertNotIn("endpoint->sourceAddr = sourceInfo.targetSegVa", endpoint_builder) + self.assertNotIn("endpoint->destinationAddr = destinationInfo.targetSegVa", endpoint_builder) + self.assertIn("endpoint->sourceRemoteImport.key = sourceInfo.key", endpoint_builder) + self.assertIn("endpoint->destinationRemoteImport.key = destinationInfo.key", endpoint_builder) + self.assertIn("TileXRCcuPackMemoryToken(sourceInfo.tokenId, sourceInfo.tokenValue, true)", endpoint_builder) + self.assertIn( + "TileXRCcuPackMemoryToken(destinationInfo.tokenId, destinationInfo.tokenValue, true)", + endpoint_builder, + ) + self.assertIn("remoteImportRequest = peerEndpoint.sourceRemoteImport", prepare_copy) + self.assertIn("remoteImportRequest = peerEndpoint.destinationRemoteImport", prepare_copy) + self.assertIn("session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer)", prepare_copy) + self.assertIn("memoryCopy.remoteAddr = remoteImportRequest.addr", prepare_copy) + self.assertIn("SetDirectCcuRemoteRouteMemoryOverride(", prepare_copy) + self.assertIn("importedRemoteBuffer.targetSegVa", prepare_copy) + self.assertNotIn("QueryDirectCcuProcessMemoryToken(sourceAddr", endpoint_builder) + self.assertNotIn("QueryDirectCcuProcessMemoryToken(destinationAddr", endpoint_builder) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_barrier_program.py b/tests/ccu/test_tilexr_ccu_barrier_program.py index 254a1b82..cbecf46a 100644 --- a/tests/ccu/test_tilexr_ccu_barrier_program.py +++ b/tests/ccu/test_tilexr_ccu_barrier_program.py @@ -242,7 +242,7 @@ def test_barrier_program_can_emit_hcomm_like_synccke_post_and_clear_wait(self): std::cerr << "source CKE init mismatch\n"; return 3; } - if (program[1].words[0] != 0x000102210330100bULL || + if (program[1].words[0] != 0xffff02210330100bULL || program[1].words[1] != 0x0000000000000002ULL || program[1].words[2] != 0x0001000000000000ULL || program[1].words[3] != 0) { @@ -309,7 +309,7 @@ def test_barrier_program_can_emit_synccke_post_and_set_wait(self): std::cerr << "source CKE init mismatch\n"; return 3; } - if (program[1].words[0] != 0x000102210330100bULL || + if (program[1].words[0] != 0xffff02210330100bULL || program[1].words[1] != 0x0000000000000002ULL || program[1].words[2] != 0x0001000000000000ULL || program[1].words[3] != 0) { @@ -374,7 +374,7 @@ def test_barrier_program_can_emit_synccke_post_only_diagnostic(self): std::cerr << "source CKE init mismatch\n"; return 3; } - if (program[1].words[0] != 0x000102210330100bULL || + if (program[1].words[0] != 0xffff02210330100bULL || program[1].words[1] != 0x0000000000000002ULL || program[1].words[2] != 0x0001000000000000ULL || program[1].words[3] != 0) { diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index e87f452f..b2bb6587 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -27,6 +27,7 @@ BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" MEMORY_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_memory_program.cpp" +SIGNAL_WAIT_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_signal_wait_program.cpp" RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime.cpp" LOWER_LAYER_PLAN_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_plan_builder.cpp" LOWER_LAYER_PAYLOAD_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_payloads.cpp" @@ -137,6 +138,7 @@ def compile_and_run(self, code: str): str(BARRIER_SOURCE), str(MICROCODE_SOURCE), str(MEMORY_PROGRAM_SOURCE), + str(SIGNAL_WAIT_PROGRAM_SOURCE), str(RUNTIME_SOURCE), str(LOWER_LAYER_PLAN_SOURCE), str(LOWER_LAYER_PAYLOAD_SOURCE), @@ -778,6 +780,29 @@ def test_direct_submit_default_runtime_trace_indexes_final_tasks(self): self.assertIn("TileXRDirectCcuTrace finalRuntimeTask[1] dieId=1 missionId=6 timeout=68 instStartId=502", result.stderr) self.assertNotIn("TileXRDirectCcuTrace finalRuntimeTask task=0", result.stderr) + def test_task_timeout_override_is_applied_before_install_trace_and_manifest(self): + source = DIRECT_SOURCE.read_text(encoding="utf-8") + body = source[ + source.index("int RunDirectInstallAttemptImpl("): + source.index("int TileXRCcuRunDirectMemoryCopyInstallAttempt(") + ] + + first_override = body.index("ApplyTaskTimeoutOverride(options.taskTimeout, attempt)") + self.assertLess(first_override, body.index("TraceDirectInstallAttempt(*attempt)")) + self.assertLess(first_override, body.index("TileXRCcuBuildInstallManifest")) + self.assertLess(first_override, body.index("TileXRCcuInstallHardware")) + + def test_signal_wait_uses_notify_mask_for_source_cke_reserve(self): + source = DIRECT_SOURCE.read_text(encoding="utf-8") + body = source[ + source.index("int BuildDirectSignalWaitLaunchPackage("): + source.index("void FillReportFromAttempt(") + ] + + self.assertIn("spec.remoteNotifyMask = resource.remoteNotifyMask == 0 ? 1U : resource.remoteNotifyMask", body) + self.assertIn("spec.sourceCkeMask = spec.remoteNotifyMask", body) + self.assertNotIn("spec.sourceCkeMask = resource.sourceCkeMask", body) + def test_direct_install_attempt_can_prepare_lower_layer_plan_after_allocation(self): code = textwrap.dedent( r''' @@ -1664,6 +1689,7 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("ccu/tilexr_ccu_direct_orchestrator.cpp", cmake) self.assertIn("struct TileXRCcuDirectInstallOptions", header) self.assertIn("struct TileXRCcuDirectMemoryCopySpec", header) + self.assertIn("struct TileXRCcuDirectSignalWaitSpec", header) self.assertIn("struct TileXRCcuDirectInstallAttempt", header) self.assertIn("struct TileXRCcuDirectInstallReport", header) self.assertIn("struct TileXRCcuDirectSubmitReport", header) @@ -1677,18 +1703,24 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TileXRCcuSubmitPreparedTasks", header) self.assertIn("TileXRCcuRunDirectInstallAttempt", header) self.assertIn("TileXRCcuRunDirectMemoryCopyInstallAttempt", header) + self.assertIn("TileXRCcuRunDirectSignalWaitInstallAttempt", header) self.assertIn("TileXRCcuDecodeBasicInfo", source) self.assertIn("TileXRCcuBuildResourceSpec", source) self.assertIn("TileXRCcuResourceAllocator", source) self.assertIn("TileXRCcuBuildLaunchPackage", source) self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) self.assertIn("BuildDirectMemoryCopyLaunchPackage", source) + self.assertIn("TileXRCcuBuildSignalWaitProgram", source) + self.assertIn("BuildDirectSignalWaitLaunchPackage", source) + self.assertIn("spec.localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke", source) self.assertIn("TileXRCcuBindLaunchPackageInstallScope", source) self.assertIn("TileXRCcuBuildInstallManifest", source) self.assertIn("options.prepareLowerLayerPlan", source) self.assertIn("TileXRCcuInstallHardware", source) self.assertIn("TileXRCcuPrepareSubmitTasks", source) self.assertIn("TileXRCcuSubmitTask", source) + self.assertIn("uint16_t taskTimeout = 0", header) + self.assertIn("ApplyTaskTimeoutOverride(options.taskTimeout, attempt)", source) self.assertIn("remote XN install provider is missing", source) self.assertIn("TraceDecodedInstr", source) self.assertIn("TraceDecodedPfeCtx", source) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 1abc100e..960cdadc 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -249,6 +249,18 @@ def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) self.assertIn("p2pCcuCopy", source) + def test_prepare_options_read_submit_timeout(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepare_options = source[ + source.index("TileXRDirectCcuPrepareOptions MakePrepareOptions"): + source.index("const char* SignalWaitRoleName") + ] + + self.assertIn( + 'options.taskTimeout = static_cast(EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 0));', + prepare_options, + ) + def test_local_to_remote_p2p_ccu_copy_inactive_rank_checks_destination(self): source = PROBE_SOURCE.read_text(encoding="utf-8") prepared_body = source[ @@ -308,6 +320,29 @@ def test_thread_mode_path_uses_single_process_init_and_never_rank_ipc_init(self) self.assertNotIn("TileXRCommInitRankWithDomain", thread_mode_body) self.assertNotIn("TILEXR_COMM_ID", thread_mode_body) + def test_probe_can_run_signal_wait_through_internal_backend(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepared_body = source[ + source.index("int RunPreparedSmokeForRank"): + source.index("int RunThreadModeSmoke") + ] + + self.assertIn("ccu/tilexr_ccu_backend.h", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_BARRIER", source) + self.assertIn("TileXRCcuBackend backend", source) + self.assertIn("context->backend.Init", source) + self.assertIn("RunSignalWaitSmokeForRank", source) + self.assertIn("TileXRCcuSignalWaitRequest", source) + self.assertIn("PrepareSignalWait", source) + self.assertIn("SubmitSignalWait", source) + self.assertIn("tilexr_ccu_signal_wait prepare", source) + self.assertIn("tilexr_ccu_signal_wait submit", source) + self.assertIn("tilexr_ccu_signal_wait timing", source) + self.assertIn("tilexr_ccu_signal_wait result passed=1", source) + self.assertIn("if (SignalWaitSmokeEnabled() || BarrierSmokeEnabled())", prepared_body) + def test_thread_mode_worker_sets_device_before_direct_ccu_submit(self): source = PROBE_SOURCE.read_text(encoding="utf-8") thread_mode_body = source[source.index("int RunThreadModeSmoke"): source.index("int main()")] @@ -509,6 +544,19 @@ def test_probe_defaults_to_lower_layer_first_install_order(self): install_order_body.index("TileXRCcuInstallOrder::RepositoryFirst"), ) + def test_probe_wires_barrier_mode_env_into_prepare_options(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + prepare_options_body = source[ + source.index("TileXRDirectCcuPrepareOptions MakePrepareOptions"): + source.index("const char* SignalWaitRoleName") + ] + + self.assertIn("TileXR::TileXRCcuBarrierMode BarrierModeFromEnv()", source) + self.assertIn("options.barrierMode = BarrierModeFromEnv();", prepare_options_body) + self.assertIn('return TileXR::TileXRCcuBarrierMode::LocalCkePostOnly;', source) + self.assertIn('return TileXR::TileXRCcuBarrierMode::SyncCkePostOnly;', source) + self.assertIn('return TileXR::TileXRCcuBarrierMode::SyncXnLoadPostOnly;', source) + def test_probe_default_sync_instruction_count_includes_hcomm_style_task1_prelude(self): source = PROBE_SOURCE.read_text(encoding="utf-8") default_body = source[ diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 6a0a98f8..c1248231 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -141,11 +141,32 @@ def test_runner_p2p_mode_applies_direct_ccu_resource_defaults(self): self.assertIn('TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}"', source) self.assertIn('TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}"', source) self.assertIn('TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}"', source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}"', source) self.assertIn('common_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE}")', source) self.assertIn("p2p_passed_count=0", source) self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) self.assertIn("direct CCU P2P CCU-copy produced no passing receiver result", source) + def test_runner_signal_wait_mode_uses_internal_backend_defaults_and_asserts_result(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("apply_signal_wait_defaults", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_BARRIER", source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) + self.assertIn('TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}"', source) + self.assertIn('TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}"', source) + self.assertIn('TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}"', source) + self.assertIn('TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}"', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_BARRIER=${TILEXR_CCU_DIRECT_SMOKE_BARRIER}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=${TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT}")', source) + self.assertIn('grep -q "tilexr_ccu_signal_wait prepare ret=0"', source) + self.assertIn('grep -q "tilexr_ccu_signal_wait submit ret=0"', source) + self.assertIn('grep -q "tilexr_ccu_signal_wait result passed=1"', source) + def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): source = RUNNER.read_text(encoding="utf-8") submit_check = source[ diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py index cf8fc80a..d6795f52 100644 --- a/tests/ccu/test_tilexr_ccu_driver_adapter.py +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -947,6 +947,8 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertIn("TILEXR_CCU_U_OP_GET_BASIC_INFO", header) self.assertIn("TILEXR_CCU_U_OP_GET_DIE_WORKING", header) self.assertIn("TILEXR_CCU_U_OP_SET_MSID_TOKEN", header) + self.assertIn("TILEXR_CCU_U_OP_SET_TASKKILL", header) + self.assertIn("TILEXR_CCU_U_OP_CLEAN_TASKKILL_STATE", header) self.assertIn("TILEXR_CCU_U_OP_SET_INSTRUCTION", header) self.assertIn("TILEXR_CCU_U_OP_SET_XN", header) self.assertIn("TILEXR_CCU_U_OP_SET_CKE", header) @@ -962,6 +964,8 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertIn("GetDieEnabled", header) self.assertIn("InstallInstructions", header) self.assertIn("InstallMsidToken", header) + self.assertIn("CleanTaskKillState", header) + self.assertIn("TILEXR_CCU_U_OP_CLEAN_TASKKILL_STATE, &out, report", source) self.assertIn("InstallPfeCtx", header) self.assertIn("InstallJettyCtx", header) self.assertIn("InstallChannelCtxV1", header) diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 70e652ed..09b4ea6c 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -540,6 +540,75 @@ def test_transport_template_can_use_hcomm_fe_id_pfe_partition_for_direct_ccu_exp self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_transport_template_uses_local_pfe_id_from_ra_eid_func_id(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 1}; + allocation.localXn = {1, 0x120, 1}; + allocation.remoteXn = {1, 0x240, 1}; + allocation.notifyCke = {1, 0x330, 1}; + + std::vector remoteCcuBuffers; + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.localPfeId = 7; + remote.localPfeIdValid = true; + remoteCcuBuffers.push_back(remote); + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + if (snapshot.pfeId != 7 || snapshot.pfeOffset != 23) { + std::cerr << "local PFE id not applied: pfeId=" << snapshot.pfeId + << " pfeOffset=" << snapshot.pfeOffset << "\n"; + return 2; + } + return 0; + } + ''' + ) + env = os.environ.copy() + env["TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE"] = "hcomm_die" + + result = self.compile_and_run(code, env=env) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_transport_template_prefers_ra_eid_func_id_over_channel_id_in_source(self): + source = BUILDER_SOURCE.read_text(encoding="utf-8") + build_body = source[ + source.index("int TileXRCcuBuildLowerLayerTransportTemplate"): + source.index("int TileXRCcuOverlayVerifiedEndpointRoutes") + ] + + self.assertIn("remoteCcuBuffers.front().localPfeId", build_body) + self.assertIn("allocation.channels.startId", build_body) + self.assertLess( + build_body.index("remoteCcuBuffers.front().localPfeId"), + build_body.index("SelectLowerLayerPfeOffset")) + def test_builds_lower_layer_install_plan_from_transport_snapshot(self): code = textwrap.dedent( r''' diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py index 91903270..1a67354f 100644 --- a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py @@ -1251,6 +1251,17 @@ def test_direct_ccu_runtime_queries_basic_info_through_ccu_hccp_loader(self): with self.subTest(needle=needle): self.assertNotIn(needle, combined) + def test_direct_ccu_runtime_init_cleans_sticky_taskkill_state(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + init_body = source[ + source.index("int TileXRCcuDirectRuntime::Init("): + source.index("void TileXRCcuDirectRuntime::Shutdown()") + ] + self.assertIn("adapter.CleanTaskKillState", init_body) + self.assertIn("TILEXR_CCU_DIRECT_DEFAULT_DIE_ID", init_body) + self.assertIn("TraceTaskKillCleanup", source) + def test_direct_ccu_runtime_keeps_ra_custom_channel_provider_alive_for_created_adapters(self): header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") diff --git a/tests/ccu/test_tilexr_ccu_signal_wait_program.py b/tests/ccu/test_tilexr_ccu_signal_wait_program.py new file mode 100644 index 00000000..88ec594a --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_signal_wait_program.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SIGNAL_WAIT_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_signal_wait_program.h" +SIGNAL_WAIT_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_signal_wait_program.cpp" +BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" +MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" +COMM_CMAKE = REPO_ROOT / "src" / "comm" / "CMakeLists.txt" +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" + + +class TileXRCcuSignalWaitProgramTest(unittest.TestCase): + def compile_and_run(self, code: str): + compiler = shutil.which("g++") or shutil.which("clang++") or shutil.which("c++") + if compiler is None: + self.skipTest("no local C++ compiler found; remote CANN compile covers signal/wait C++ syntax") + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + test_cpp = temp_path / "signal_wait_program_test.cpp" + test_bin = temp_path / "signal_wait_program_test" + test_cpp.write_text(code, encoding="utf-8") + subprocess.run( + [ + compiler, + "-std=c++14", + "-I", + str(INCLUDE_DIR), + "-I", + str(COMM_DIR), + str(test_cpp), + str(SIGNAL_WAIT_SOURCE), + str(BARRIER_SOURCE), + str(MICROCODE_SOURCE), + "-o", + str(test_bin), + ], + cwd=REPO_ROOT, + check=True, + text=True, + capture_output=True, + ) + return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) + + def test_signal_wait_program_builds_signal_only_hcomm_style_synccke_post(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_signal_wait_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuSignalWaitProgramSpec spec; + spec.role = TileXRCcuSignalWaitProgramRole::Signal; + spec.channelId = 2; + spec.remoteXn = 2361; + spec.localXn = 1961; + spec.remoteNotifyCke = 364; + spec.remoteNotifyMask = 1; + spec.sourceCke = 0x101; + spec.sourceCkeMask = 0xffff; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildSignalWaitProgram(spec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "signal program build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 1 || report.postInstructionCount != 1 || + report.waitInstructionCount != 0 || report.totalInstructionCount != 1) { + std::cerr << "unexpected signal report\n"; + return 2; + } + if (program[0].words[0] != 0xffff0101016c100bULL || + program[0].words[1] != 0x0000000000000002ULL || + program[0].words[2] != 0x0001000000000000ULL) { + std::cerr << "unexpected signal instructions\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_signal_wait_program_builds_wait_only_clear_cke_wait(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_signal_wait_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuSignalWaitProgramSpec spec; + spec.role = TileXRCcuSignalWaitProgramRole::Wait; + spec.localXn = 0x7a9; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildSignalWaitProgram(spec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "wait program build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 1 || report.postInstructionCount != 0 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 1) { + std::cerr << "unexpected wait report\n"; + return 2; + } + if (program[0].words[0] != 0x0000000000010804ULL || + program[0].words[1] != 0x0000000000010220ULL) { + std::cerr << "unexpected wait instruction\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_signal_wait_program_builds_signal_and_wait_for_barrier(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_signal_wait_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuSignalWaitProgramSpec spec; + spec.role = TileXRCcuSignalWaitProgramRole::SignalAndWait; + spec.channelId = 2; + spec.remoteXn = 2361; + spec.localXn = 1961; + spec.remoteNotifyCke = 364; + spec.remoteNotifyMask = 1; + spec.localWaitCke = 0x220; + spec.localWaitMask = 1; + spec.sourceCke = 0x101; + spec.sourceCkeMask = 0xffff; + + std::vector program; + TileXRCcuBarrierProgramReport report; + if (TileXRCcuBuildSignalWaitProgram(spec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "barrier program build failed: " << report.message << "\n"; + return 1; + } + if (program.size() != 2 || report.postInstructionCount != 1 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 2) { + std::cerr << "unexpected signal_and_wait report\n"; + return 2; + } + if (program[0].words[0] != 0xffff0101016c100bULL || + program[1].words[0] != 0x0000000000010804ULL || + program[1].words[1] != 0x0000000000010220ULL) { + std::cerr << "unexpected signal_and_wait instructions\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_signal_wait_program_rejects_missing_role_resources(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_signal_wait_program.h" + + #include + #include + + using namespace TileXR; + + int main() + { + std::vector program(1); + TileXRCcuBarrierProgramReport report; + TileXRCcuSignalWaitProgramSpec spec; + spec.role = TileXRCcuSignalWaitProgramRole::Signal; + if (TileXRCcuBuildSignalWaitProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing signal resources accepted\n"; + return 1; + } + if (!program.empty() || report.message.find("signal") == std::string::npos) { + std::cerr << "weak signal diagnostic: " << report.message << "\n"; + return 2; + } + spec.role = TileXRCcuSignalWaitProgramRole::Wait; + if (TileXRCcuBuildSignalWaitProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL) { + std::cerr << "missing wait resources accepted\n"; + return 3; + } + if (!program.empty() || report.message.find("wait") == std::string::npos) { + std::cerr << "weak wait diagnostic: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_signal_wait_program_is_wired(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = SIGNAL_WAIT_HEADER.read_text(encoding="utf-8") + source = SIGNAL_WAIT_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_signal_wait_program.h", cmake) + self.assertIn("ccu/tilexr_ccu_signal_wait_program.cpp", cmake) + self.assertIn("enum class TileXRCcuSignalWaitProgramRole", header) + self.assertIn("struct TileXRCcuSignalWaitProgramSpec", header) + self.assertIn("TileXRCcuBuildSignalWaitProgram", header) + self.assertIn("TileXRCcuEncodeLoadImdToXn", source) + self.assertIn("TileXRCcuEncodeSetCke", source) + self.assertIn("TileXRCcuEncodeSyncCke", source) + self.assertIn("TileXRCcuEncodeClearCke", source) + + +if __name__ == "__main__": + unittest.main() From 81f2210a3cb49cd3834a658b610b29bdb7fcbd8c Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Tue, 14 Jul 2026 09:19:45 +0800 Subject: [PATCH 20/41] fix(ccu): initialize signal wait source cke Add the source CKE prelude before SyncCke in direct CCU signal/wait programs so wait-only peers observe the posted CKE instead of timing out. Update signal/wait program coverage to assert the hcomm-style source CKE initialization sequence for signal-only and signal-and-wait roles. --- .../ccu/tilexr_ccu_signal_wait_program.cpp | 6 +++++ .../test_tilexr_ccu_signal_wait_program.py | 24 ++++++++++--------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp b/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp index 1ed253d6..fb00e38a 100644 --- a/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp +++ b/src/comm/ccu/tilexr_ccu_signal_wait_program.cpp @@ -196,6 +196,9 @@ int TileXRCcuBuildSignalWaitProgram( return Fail(program, report, "missing wait CKE resource for direct CCU signal/wait program"); } + if (AppendCommonPrelude(spec, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } if (AppendSignalInstruction(spec, program, report) != TILEXR_SUCCESS) { return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -203,6 +206,9 @@ int TileXRCcuBuildSignalWaitProgram( AppendWaitInstruction(spec, program, report, false) != TILEXR_SUCCESS) { return TILEXR_ERROR_PARA_CHECK_FAIL; } + if (AppendFinish(spec, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } if (report != nullptr) { report->postInstructionCount = 1; report->waitInstructionCount = spec.role == TileXRCcuSignalWaitProgramRole::SignalAndWait ? 1U : 0U; diff --git a/tests/ccu/test_tilexr_ccu_signal_wait_program.py b/tests/ccu/test_tilexr_ccu_signal_wait_program.py index 88ec594a..f008749b 100644 --- a/tests/ccu/test_tilexr_ccu_signal_wait_program.py +++ b/tests/ccu/test_tilexr_ccu_signal_wait_program.py @@ -53,7 +53,7 @@ def compile_and_run(self, code: str): ) return subprocess.run([str(test_bin)], cwd=REPO_ROOT, check=False, text=True, capture_output=True) - def test_signal_wait_program_builds_signal_only_hcomm_style_synccke_post(self): + def test_signal_wait_program_initializes_source_cke_before_signal_only_post(self): code = textwrap.dedent( r''' #include "ccu/tilexr_ccu_signal_wait_program.h" @@ -81,14 +81,15 @@ def test_signal_wait_program_builds_signal_only_hcomm_style_synccke_post(self): std::cerr << "signal program build failed: " << report.message << "\n"; return 1; } - if (program.size() != 1 || report.postInstructionCount != 1 || - report.waitInstructionCount != 0 || report.totalInstructionCount != 1) { + if (program.size() != 5 || report.postInstructionCount != 1 || + report.waitInstructionCount != 0 || report.totalInstructionCount != 5) { std::cerr << "unexpected signal report\n"; return 2; } - if (program[0].words[0] != 0xffff0101016c100bULL || - program[0].words[1] != 0x0000000000000002ULL || - program[0].words[2] != 0x0001000000000000ULL) { + if (program[2].words[0] != 0xffff010100010802ULL || + program[3].words[0] != 0xffff0101016c100bULL || + program[3].words[1] != 0x0000000000000002ULL || + program[3].words[2] != 0x0001000000000000ULL) { std::cerr << "unexpected signal instructions\n"; return 3; } @@ -176,14 +177,15 @@ def test_signal_wait_program_builds_signal_and_wait_for_barrier(self): std::cerr << "barrier program build failed: " << report.message << "\n"; return 1; } - if (program.size() != 2 || report.postInstructionCount != 1 || - report.waitInstructionCount != 1 || report.totalInstructionCount != 2) { + if (program.size() != 6 || report.postInstructionCount != 1 || + report.waitInstructionCount != 1 || report.totalInstructionCount != 6) { std::cerr << "unexpected signal_and_wait report\n"; return 2; } - if (program[0].words[0] != 0xffff0101016c100bULL || - program[1].words[0] != 0x0000000000010804ULL || - program[1].words[1] != 0x0000000000010220ULL) { + if (program[2].words[0] != 0xffff010100010802ULL || + program[3].words[0] != 0xffff0101016c100bULL || + program[4].words[0] != 0x0000000000010804ULL || + program[4].words[1] != 0x0000000000010220ULL) { std::cerr << "unexpected signal_and_wait instructions\n"; return 3; } From d192cbb74eeb67589799ec2acc4f67c6b948a98f Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Tue, 14 Jul 2026 14:56:17 +0800 Subject: [PATCH 21/41] docs: add ccu alltoall mvp design --- .../2026-07-14-ccu-alltoall-mvp-design.md | 281 ++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md diff --git a/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md b/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md new file mode 100644 index 00000000..b8ed5de1 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md @@ -0,0 +1,281 @@ +# TileXR Direct CCU 2-Rank AllToAll MVP Design + +## Scope + +Build a TileXR-owned direct CCU 2-rank AllToAll smoke path for a fixed P2P +payload of 2 MB per rank. The MVP follows the HCCL CCU AllToAll algorithm +shape, but it must not include or call any HCCL or hcomm interface. + +The MVP validates: + +- Rank 0 sends 2 MB to rank 1's receive buffer. +- Rank 1 sends 2 MB to rank 0's receive buffer. +- The CCU mission performs PreSync, data movement, copy completion wait, and + PostSync. +- Host-side validation reads local receive buffers and checks peer-specific + data patterns. + +This is a hardware data-plane and synchronization smoke. It is not yet the +LoopEngine validation target. + +## Non-Goals + +- Do not include HCCL/hcomm headers. +- Do not call HCCL/hcomm launch, channel, resource, or CCU wrapper APIs. +- Do not link new HCCL/hcomm libraries. +- Do not claim LoopEngine is validated by the MVP. +- Do not add a broad public collective API before the smoke path proves the + direct CCU sequence. + +## Reference Model + +The local HCCL source under `.tmp/hccl-gitcode` uses this AllToAll shape: + +1. `PreSync`: publish output address and token to peers and wait for peer + readiness. +2. `DoAlltoAll`: write local source slices into peer output through CCU + channels and wait for copy events. +3. `PostSync`: notify and wait for peers after data movement. + +TileXR will use the same algorithm shape and size model, but express every +operation through TileXR-owned direct CCU program builders, resources, and +submit tasks. + +## Data Model + +The MVP is fixed-size and two-rank: + +```text +rankSize = 2 +totalBytesPerRank = 2 * 1024 * 1024 +memorySliceBytes = 4096 +memSlicePerLoop = 8 +bytesPerBlock = memorySliceBytes * memSlicePerLoop = 32768 +blockCount = totalBytesPerRank / bytesPerBlock = 64 +``` + +`memSlicePerLoop` is capped at 8. The default is 8. The MVP should reject +non-4KB-aligned sizes and any block configuration that would produce a partial +MemorySlice. + +The data direction is LocalToRemote: + +```text +rank0: rank0.send -> rank1.recv +rank1: rank1.send -> rank0.recv +``` + +This matches the HCCL-style AllToAll write model and keeps the operation +semantically close to send-to-peer communication. + +## Mission Flow + +Each rank submits one CCU mission. The mission contains four phases. + +### 1. PreSync + +The rank confirms that peer output address/token resources are ready before +remote writes begin. + +The MVP can implement this with TileXR-owned CKE/checklist operations: + +```text +SetCke(local source CKE) +SyncCke(peer pre-sync wait CKE, local source CKE) +ClearCke(local pre-sync wait CKE) +``` + +PreSync is intentionally separate from final PostSync. This makes readiness +and completion failures distinguishable in logs and mission traces. + +### 2. Data Move + +The MVP expands the 2 MB transfer into 64 fixed 32 KB blocks at host program +build time: + +```text +for block in 0..63: + localAddr = sendBase + block * 32768 + remoteAddr = peerRecvBase + block * 32768 + length = 32768 + TransLocMemToRmtMem(localAddr, remoteAddr, length) + ClearCke(copy completion CKE) +``` + +This represents the HCCL-style `memSlicePerLoop=8` work unit without relying +on LoopEngine support. The block size is deliberately the same as eight 4 KB +MemorySlices. + +### 3. Copy Completion Wait + +Each block waits for its own transfer completion using the existing memory +copy completion CKE semantics. The same completion CKE can be reused because +the blocks are emitted serially and each block consumes completion before the +next block begins. + +### 4. PostSync + +After all 64 blocks complete, the rank performs a final two-way completion +barrier: + +```text +SetCke(local source CKE) +SyncCke(peer post-sync wait CKE, local source CKE) +ClearCke(local post-sync wait CKE) +``` + +This is the TileXR-owned `SignalAndWait` completion barrier. + +## Program Builder + +Add a new builder under `src/comm/ccu`: + +```text +tilexr_ccu_alltoall_program.h +tilexr_ccu_alltoall_program.cpp +``` + +Main types: + +```cpp +struct TileXRCcuAllToAll2RankProgramSpec { + uint64_t localSendAddr; + uint64_t localSendToken; + uint64_t remoteRecvAddr; + uint64_t remoteRecvToken; + uint64_t bytes; + uint32_t memorySliceBytes; + uint32_t memSlicePerLoop; + uint16_t localGsa; + uint16_t remoteGsa; + uint16_t localXn; + uint16_t remoteXn; + uint16_t lengthXn; + uint16_t channelId; + uint16_t copyCompletionCke; + uint16_t preSyncLocalWaitCke; + uint16_t preSyncRemoteNotifyCke; + uint16_t postSyncLocalWaitCke; + uint16_t postSyncRemoteNotifyCke; + uint16_t sourceCke; +}; +``` + +The builder validates: + +- `bytes == 2 MB` for the initial smoke. +- `memorySliceBytes == 4096`. +- `1 <= memSlicePerLoop <= 8`. +- `bytes % (memorySliceBytes * memSlicePerLoop) == 0`. +- all XN/GSA/CKE/channel/token/address resources are non-zero. + +The builder emits one instruction stream: + +```text +PreSync instructions +64 * 32KB LocalToRemote copy block instructions +PostSync instructions +Finish instruction +``` + +## Planner and Orchestrator + +Add a testing/private direct CCU path first: + +```text +TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt +TileXRCcuRunDirectAllToAll2RankInstallAttempt +BuildDirectAllToAll2RankLaunchPackage +``` + +The planner reuses the existing direct CCU lower-layer path: + +- direct runtime session +- driver adapter +- endpoint route/channel install +- repository install +- mission/task build +- prepared task submit + +It should allocate enough CKE resources for separate pre-sync, copy-completion, +post-sync, and source CKE use. If the first implementation must reuse the +existing one-resource allocator shape, the resource mapping must be explicit in +trace output and unit tests. + +## Smoke Test + +Extend `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` with an opt-in mode: + +```text +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 +TILEXR_CCU_ALLTOALL_BYTES=2097152 +TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 +``` + +The runner should: + +1. allocate local send and receive buffers on each rank; +2. fill send with a rank-specific byte pattern; +3. fill receive with a sentinel; +4. prepare the alltoall mission; +5. wait until both ranks are submit-ready; +6. submit the prepared task and synchronize the stream; +7. wait for both ranks to report done; +8. copy local receive buffer back to host; +9. verify it equals the peer rank pattern. + +Expected passing evidence: + +```text +tilexr_ccu_alltoall config rank= bytes=2097152 memSlicePerLoop=8 blockCount=64 +tilexr_ccu_alltoall submit ... submitRet=0 syncRet=0 +tilexr_ccu_alltoall result passed=1 rank= mismatches=0 +``` + +All hardware runs must use an outer `timeout`. + +## Unit Tests + +Add focused tests before hardware validation: + +- program builder rejects invalid `memSlicePerLoop > 8`; +- program builder rejects non-4KB-aligned size; +- 2 MB generates exactly 64 copy blocks; +- PreSync instructions appear before the first copy block; +- PostSync instructions appear after the last copy block; +- no HCCL/hcomm symbols are introduced into `src/comm`. + +The dependency guard remains mandatory: + +```bash +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh build/src/comm/libtile-comm.so +``` + +## LoopGroup Follow-Up + +After the MVP passes, replace host-expanded 64-block emission with a TileXR-owned +LoopGroup implementation: + +- add TileXR LoopCtx/LoopGroup encoding; +- add LoopEngine allocation and lifecycle tracking; +- add GoSize calculation equivalent to the HCCL model: + - `addrOffset` + - `loopParam` + - `parallelParam` + - `residual` +- keep the smoke API and validation unchanged; +- add evidence that LoopEngine, not host expansion, performed the 2 MB + MemorySlice traversal. + +The follow-up must be a separate change so the MVP result cannot be mistaken +for LoopEngine validation. + +## Open Risks + +- TileXR currently lacks a first-class LoopGroup/LoopEngine encoder. +- Current direct memory-copy builder is single-copy oriented; the alltoall + builder must avoid accidental resource overlap across 64 copy blocks. +- PreSync and PostSync require enough independent CKE/checklist resources to + keep readiness and completion distinguishable. +- LocalToRemote validation writes peer memory, so host-side checks must read + each rank's local receive buffer after both ranks complete. From b95c63b2da0b72b0f16e217f7f569b82b197046a Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Wed, 15 Jul 2026 09:41:59 +0800 Subject: [PATCH 22/41] feat(ccu): add host-phased direct alltoall MVP Add a TileXR-owned direct CCU alltoall MVP for two-rank validation without introducing HCCL or hcomm private APIs. Implement the alltoall program builder with 4KB memory slices, 32KB blocks, 64-block 2MB transfers, rank-aware copy phases, and validation for invalid slice or rank inputs. Wire the direct alltoall install path through the CCU planner and orchestrator, including route-scoped remote memory overrides so the data copy route targets the peer buffer while sync routes keep using the CCU resource window. Harden lower-layer planning for multi-route endpoint snapshots by expanding the PFE jetty window and preserving distinct channel, CKE, and jetty ownership for pre-sync, data, and post-sync resources. Add a host-phased alltoall smoke mode that reuses the proven P2P CCU RemoteToLocal copy path: both ranks prepare each phase, only the active rank submits the CCU copy mission, and host ready/done files serialize phase0 and phase1 with bounded stream synchronization. Cover the new behavior with alltoall program tests, orchestrator and planner source checks, backend boundary tests, lower-layer plan regressions, and smoke-runner assertions for timeout-safe execution. Verified on the remote CCU environment with unit tests, tile-comm rebuild, hcomm/HCCL dependency guard, and a device 0/1 2MB alltoall smoke run passing on both ranks with zero mismatches. --- src/comm/CMakeLists.txt | 2 + src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 300 ++++++++++++++ src/comm/ccu/tilexr_ccu_alltoall_program.h | 67 ++++ .../ccu/tilexr_ccu_collective_planner.cpp | 210 +++++++++- src/comm/ccu/tilexr_ccu_collective_planner.h | 18 + .../ccu/tilexr_ccu_direct_orchestrator.cpp | 224 ++++++++++- src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 18 + .../tilexr_ccu_lower_layer_plan_builder.cpp | 119 ++++-- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 299 ++++++++++++++ tests/ccu/run_tilexr_ccu_direct_smoke.sh | 119 +++++- tests/ccu/test_tilexr_ccu_alltoall_program.py | 372 ++++++++++++++++++ tests/ccu/test_tilexr_ccu_backend_boundary.py | 18 + .../test_tilexr_ccu_direct_orchestrator.py | 42 ++ .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 49 +++ ...est_tilexr_ccu_lower_layer_plan_builder.py | 113 ++++++ 15 files changed, 1931 insertions(+), 39 deletions(-) create mode 100644 src/comm/ccu/tilexr_ccu_alltoall_program.cpp create mode 100644 src/comm/ccu/tilexr_ccu_alltoall_program.h create mode 100644 tests/ccu/test_tilexr_ccu_alltoall_program.py diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index a635c42b..1a881156 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -106,6 +106,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp tilexr_internal.h tilexr_log.h ccu/tilexr_ccu_abi_constants.h + ccu/tilexr_ccu_alltoall_program.h + ccu/tilexr_ccu_alltoall_program.cpp ccu/tilexr_ccu_barrier_program.h ccu/tilexr_ccu_barrier_program.cpp ccu/tilexr_ccu_backend.h diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp new file mode 100644 index 00000000..e77379f4 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -0,0 +1,300 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_alltoall_program.h" + +namespace TileXR { +namespace { + +void ResetReport(TileXRCcuAllToAllProgramReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuAllToAllProgramReport{}; + } +} + +int Fail( + std::vector* program, + TileXRCcuAllToAllProgramReport* report, + const std::string& message) +{ + if (program != nullptr) { + program->clear(); + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int ValidateSpec( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (program == nullptr) { + return Fail(program, report, "missing output direct CCU alltoall program"); + } + if (spec.localRecvAddr == 0 || spec.remoteSendAddr == 0) { + return Fail(program, report, "missing direct CCU alltoall address"); + } + if (spec.localRecvToken == 0 || spec.remoteSendToken == 0) { + return Fail(program, report, "missing direct CCU alltoall token"); + } + if (spec.bytes == 0 || spec.bytes % TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES != 0) { + return Fail(program, report, "direct CCU alltoall bytes must be nonzero and 4KB aligned"); + } + if (spec.localRank > 1U) { + return Fail(program, report, "direct CCU alltoall localRank must be 0 or 1"); + } + if (spec.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES) { + return Fail(program, report, "direct CCU alltoall memorySliceBytes must be 4096"); + } + if (spec.memSlicePerBlock == 0 || spec.memSlicePerBlock > TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK) { + return Fail(program, report, "direct CCU alltoall memSlicePerBlock must be in [1, 8]"); + } + const uint64_t bytesPerBlock = static_cast(spec.memorySliceBytes) * spec.memSlicePerBlock; + if (spec.bytes % bytesPerBlock != 0) { + return Fail(program, report, "direct CCU alltoall bytes must align to memSlicePerBlock"); + } + if (spec.localGsa == 0 || spec.remoteGsa == 0 || spec.localXn == 0 || spec.remoteXn == 0 || + spec.lengthXn == 0) { + return Fail(program, report, "missing direct CCU alltoall GSA/XN resource"); + } + const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; + const uint16_t copyChannelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; + if (preSyncChannelId == 0 || copyChannelId == 0 || postSyncChannelId == 0 || + spec.copyCompletionCke == 0 || spec.preSyncLocalWaitCke == 0 || + spec.preSyncRemoteNotifyCke == 0 || spec.postSyncLocalWaitCke == 0 || + spec.postSyncRemoteNotifyCke == 0 || spec.sourceCke == 0 || spec.ckeMask == 0) { + return Fail(program, report, "missing direct CCU alltoall CKE/channel resource"); + } + return TILEXR_SUCCESS; +} + +int AppendSetSourceCke( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuCkeSpec set; + set.ckeId = spec.sourceCke; + set.mask = spec.ckeMask; + set.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSetCke(set, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall source CKE set"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendSyncPair( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuSyncCkeSpec post; + post.remoteCke = remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = spec.ckeMask; + post.channelId = channelId; + post.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall SyncCke"); + } + program->push_back(instr); + + TileXRCcuCkeSpec wait; + wait.waitCkeId = localWaitCke; + wait.waitMask = spec.ckeMask; + wait.clearWait = true; + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall ClearCke wait"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendSyncPhase( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall sync prelude XN"); + } + program->push_back(instr); + + if (AppendSetSourceCke(spec, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return AppendSyncPair(remoteNotifyCke, localWaitCke, channelId, spec, program, report); +} + +int AppendCopyBlock( + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuMemoryCopySpec copy; + copy.direction = TileXRCcuMemoryCopyDirection::RemoteToLocal; + copy.localGsa = spec.localGsa; + copy.localXn = spec.localXn; + copy.remoteGsa = spec.remoteGsa; + copy.remoteXn = spec.remoteXn; + copy.lengthXn = spec.lengthXn; + copy.localAddr = spec.localRecvAddr + offset; + copy.localToken = spec.localRecvToken; + copy.remoteAddr = spec.remoteSendAddr + offset; + copy.remoteToken = spec.remoteSendToken; + copy.lengthBytes = bytesPerBlock; + copy.channelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + copy.completionCke = spec.copyCompletionCke; + copy.completionMask = spec.ckeMask; + + std::vector block; + TileXRCcuMemoryProgramReport memoryReport; + if (TileXRCcuBuildMemoryCopyProgram(copy, &block, &memoryReport) != TILEXR_SUCCESS) { + return Fail(program, report, memoryReport.message); + } + program->insert(program->end(), block.begin(), block.end()); + return TILEXR_SUCCESS; +} + +int AppendFinish( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall finish instruction"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +void FillReport( + const TileXRCcuAllToAll2RankProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + if (report == nullptr) { + return; + } + const uint32_t bytesPerBlock = spec.memorySliceBytes * spec.memSlicePerBlock; + report->preSyncInstructionCount = 4; + report->blockCount = static_cast(spec.bytes / bytesPerBlock); + report->bytesPerBlock = bytesPerBlock; + report->copyInstructionCount = report->blockCount * 7U; + report->postSyncInstructionCount = 4; + report->finishInstructionCount = 1; + report->totalInstructionCount = static_cast(program.size()); + report->message = "ok"; +} + +} // namespace + +int TileXRCcuBuildAllToAll2RankProgram( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + ResetReport(report); + if (program != nullptr) { + program->clear(); + } + int ret = ValidateSpec(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + const uint64_t bytesPerBlock = static_cast(spec.memorySliceBytes) * spec.memSlicePerBlock; + const uint32_t blockCount = static_cast(spec.bytes / bytesPerBlock); + const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; + const uint16_t copyChannelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; + program->reserve(4U + 4U + blockCount * 7U + 4U + 4U + 4U + 4U + 1U); + + ret = AppendSyncPhase( + spec.preSyncRemoteNotifyCke, + spec.preSyncLocalWaitCke, + preSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + for (uint32_t phase = 0; phase < 2U; ++phase) { + ret = AppendSyncPhase( + spec.preSyncRemoteNotifyCke, + spec.preSyncLocalWaitCke, + preSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + // append copy only for the local rank's active phase. + if (phase == spec.localRank) { + for (uint32_t block = 0; block < blockCount; ++block) { + const uint64_t offset = static_cast(block) * bytesPerBlock; + ret = AppendCopyBlock(spec, offset, bytesPerBlock, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + } + + ret = AppendSyncPhase( + spec.postSyncRemoteNotifyCke, + spec.postSyncLocalWaitCke, + postSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + ret = AppendSyncPhase( + spec.postSyncRemoteNotifyCke, + spec.postSyncLocalWaitCke, + postSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = AppendFinish(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + FillReport(spec, *program, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h new file mode 100644 index 00000000..c2f466cf --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_ALLTOALL_PROGRAM_H +#define TILEXR_CCU_ALLTOALL_PROGRAM_H + +#include "ccu/tilexr_ccu_memory_program.h" + +#include +#include +#include + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES = 4096U; +constexpr uint32_t TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK = 8U; +constexpr uint32_t TILEXR_CCU_ALLTOALL_BLOCK_BYTES = + TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES * TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + +struct TileXRCcuAllToAll2RankProgramSpec { + uint32_t localRank = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t remoteSendAddr = 0; + uint64_t remoteSendToken = 0; + uint64_t bytes = 0; + uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + uint16_t localGsa = 0; + uint16_t remoteGsa = 0; + uint16_t localXn = 0; + uint16_t remoteXn = 0; + uint16_t lengthXn = 0; + uint16_t channelId = 0; + uint16_t preSyncChannelId = 0; + uint16_t copyChannelId = 0; + uint16_t postSyncChannelId = 0; + uint16_t copyCompletionCke = 0; + uint16_t preSyncLocalWaitCke = 0; + uint16_t preSyncRemoteNotifyCke = 0; + uint16_t postSyncLocalWaitCke = 0; + uint16_t postSyncRemoteNotifyCke = 0; + uint16_t sourceCke = 0; + uint16_t ckeMask = 1; +}; + +struct TileXRCcuAllToAllProgramReport { + uint32_t preSyncInstructionCount = 0; + uint32_t blockCount = 0; + uint32_t bytesPerBlock = 0; + uint32_t copyInstructionCount = 0; + uint32_t postSyncInstructionCount = 0; + uint32_t finishInstructionCount = 0; + uint32_t totalInstructionCount = 0; + std::string message; +}; + +int TileXRCcuBuildAllToAll2RankProgram( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_ALLTOALL_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 7991522b..763f5376 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -25,6 +25,8 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = + 4U + 4U + 64U * 7U + 4U + 4U + 4U + 4U + 1U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; @@ -672,6 +674,24 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( uint32_t memoryTokenId, uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) +{ + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 0, + peerRank, + remoteCcuVa, + memoryTokenId, + rawMemoryTokenId, + memoryTokenValue); + directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; +} + +void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + uint32_t syncRouteIndex, + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue) { directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; directCcuRemoteRouteMemoryOverride_.peerRank = peerRank; @@ -681,12 +701,16 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( directCcuRemoteRouteMemoryOverride_.memoryTokenValue = memoryTokenValue; directCcuRemoteRouteMemoryOverrideValid_ = remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; + directCcuRemoteRouteMemoryOverrideAllRoutes_ = false; + directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = syncRouteIndex; } void TileXRCcuCollectivePlanner::ClearDirectCcuRemoteRouteMemoryOverride() { directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; directCcuRemoteRouteMemoryOverrideValid_ = false; + directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; } void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( @@ -695,14 +719,19 @@ void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( if (!directCcuRemoteRouteMemoryOverrideValid_ || remoteCcuBuffers == nullptr) { return; } + uint32_t routeIndex = 0; for (auto &remoteCcuBuffer : *remoteCcuBuffers) { - if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank) { + if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank || + (!directCcuRemoteRouteMemoryOverrideAllRoutes_ && + routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_)) { + ++routeIndex; continue; } remoteCcuBuffer.remoteCcuVa = directCcuRemoteRouteMemoryOverride_.remoteCcuVa; remoteCcuBuffer.memoryTokenId = directCcuRemoteRouteMemoryOverride_.memoryTokenId; remoteCcuBuffer.rawMemoryTokenId = directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId; remoteCcuBuffer.memoryTokenValue = directCcuRemoteRouteMemoryOverride_.memoryTokenValue; + ++routeIndex; } } #endif @@ -1143,6 +1172,185 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( ClearDirectCcuRemoteRouteMemoryOverride(); return ret; } + +int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU alltoall install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for alltoall install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before alltoall buffer registration"; + } + return ret; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU alltoall local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU alltoall peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.sourceRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU alltoall remote source buffer"; + } + return ret; + } + + TileXRCcuDirectAllToAll2RankSpec alltoall; + alltoall.localRank = static_cast(rank); + alltoall.localRecvAddr = localEndpoint.destinationAddr; + alltoall.localRecvToken = localEndpoint.destinationToken; + alltoall.remoteSendAddr = remoteImportRequest.addr; + alltoall.remoteSendToken = + TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); + alltoall.bytes = bytes; + alltoall.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + alltoall.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + + if (TraceDirectCcuMemoryCopy()) { + std::cerr << "TileXRDirectCcuTrace alltoallEndpoint" + << " rank=" << rank + << " peerRank=" << peerRank + << " localRecvAddr=0x" << std::hex << alltoall.localRecvAddr + << " localRecvToken=0x" << alltoall.localRecvToken + << " remoteSendAddr=0x" << alltoall.remoteSendAddr + << " remoteSendToken=0x" << alltoall.remoteSendToken + << " bytes=0x" << alltoall.bytes + << std::dec << std::endl; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 3; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-alltoall"; + } + + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 1U, + peerRank, + importedRemoteBuffer.targetSegVa, + remoteImportRequest.tokenId, + remoteImportRequest.rawTokenId, + remoteImportRequest.tokenValue); + ret = TileXRCcuRunDirectAllToAll2RankInstallAttempt(next, alltoall, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} #endif int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index 9c512711..551e98d8 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -63,6 +63,15 @@ class TileXRCcuCollectivePlanner { TileXRCcuMemoryCopyDirection direction, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuAllToAll2RankInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); #endif int RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session); bool HasDirectCcuLowerLayerPlan() const; @@ -94,6 +103,13 @@ class TileXRCcuCollectivePlanner { uint32_t memoryTokenId, uint32_t rawMemoryTokenId, uint32_t memoryTokenValue); + void SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + uint32_t syncRouteIndex, + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue); void ClearDirectCcuRemoteRouteMemoryOverride(); void ApplyDirectCcuRemoteRouteMemoryOverride( std::vector *remoteCcuBuffers) const; @@ -117,6 +133,8 @@ class TileXRCcuCollectivePlanner { #ifdef TILEXR_CCU_TESTING TileXRCcuRemoteCcuBufferInfo directCcuRemoteRouteMemoryOverride_ = {}; bool directCcuRemoteRouteMemoryOverrideValid_ = false; + bool directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + uint32_t directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; #endif }; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index 3f0fc29c..db61a428 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -5,6 +5,7 @@ #include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_alltoall_program.h" #include "ccu/tilexr_ccu_runtime.h" #include @@ -35,6 +36,9 @@ constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALUE_MASK = 0xffffffffULL; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = + 4U + 4U + 64U * 7U + 4U + 4U + 4U + 4U + 1U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; @@ -753,6 +757,52 @@ int ConfigureDirectMemoryCopyResources( return TILEXR_SUCCESS; } +int ConfigureDirectAllToAll2RankResources( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT || + attempt->plan.taskWindows.size() != 1) { + if (report != nullptr) { + report->message = "alltoall direct CCU plan requires three sync resources and one task"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (options.gsaStartId == 0 || attempt->resourceSpec.gsaCount < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "alltoall direct CCU requires a kernel-local GSA resource window"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint8_t dieId = attempt->specInfo.dieId; + const uint16_t localXnStart = attempt->allocation.localXn.startId; + if (!ContainsRange( + attempt->resourceSpec.xnStartId, + attempt->resourceSpec.xnCount, + localXnStart, + TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)) { + if (report != nullptr) { + report->message = "alltoall direct CCU local XN window is too small"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->allocation.localXn = + MakeRange(dieId, localXnStart, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)); + attempt->allocation.localGsa = + MakeRange(dieId, options.gsaStartId, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT)); + attempt->plan.kernelLocalXn = attempt->allocation.localXn; + attempt->plan.kernelLocalGsa = attempt->allocation.localGsa; + attempt->plan.taskWindows[0].instCnt = + static_cast(std::max( + attempt->plan.taskWindows[0].instCnt, + TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT)); + return TILEXR_SUCCESS; +} + int BuildDirectMemoryCopyLaunchPackage( const TileXRCcuDirectMemoryCopySpec& memoryCopy, TileXRCcuDirectInstallAttempt* attempt, @@ -853,6 +903,123 @@ int BuildDirectMemoryCopyLaunchPackage( return TILEXR_SUCCESS; } +int BuildDirectAllToAll2RankLaunchPackage( + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT || + attempt->plan.kernelLocalXn.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall GSA/XN resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[0]; + const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[1]; + const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]; + + TileXRCcuAllToAll2RankProgramSpec alltoallSpec; + alltoallSpec.localRank = alltoall.localRank; + alltoallSpec.localRecvAddr = alltoall.localRecvAddr; + alltoallSpec.localRecvToken = alltoall.localRecvToken; + alltoallSpec.remoteSendAddr = alltoall.remoteSendAddr; + alltoallSpec.remoteSendToken = alltoall.remoteSendToken; + alltoallSpec.bytes = alltoall.bytes; + alltoallSpec.memorySliceBytes = alltoall.memorySliceBytes; + alltoallSpec.memSlicePerBlock = alltoall.memSlicePerBlock; + alltoallSpec.localGsa = attempt->plan.kernelLocalGsa.startId; + alltoallSpec.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + alltoallSpec.localXn = attempt->plan.kernelLocalXn.startId; + alltoallSpec.remoteXn = static_cast(attempt->plan.kernelLocalXn.startId + 1U); + alltoallSpec.lengthXn = static_cast(attempt->plan.kernelLocalXn.startId + 2U); + alltoallSpec.channelId = copyResource.channelId; + alltoallSpec.preSyncChannelId = preResource.channelId; + alltoallSpec.copyChannelId = copyResource.channelId; + alltoallSpec.postSyncChannelId = postResource.channelId; + alltoallSpec.preSyncRemoteNotifyCke = preResource.notifyCke; + alltoallSpec.preSyncLocalWaitCke = + preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; + alltoallSpec.copyCompletionCke = + copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke; + alltoallSpec.postSyncRemoteNotifyCke = postResource.notifyCke; + alltoallSpec.postSyncLocalWaitCke = + postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; + alltoallSpec.sourceCke = preResource.sourceCke; + alltoallSpec.ckeMask = preResource.remoteNotifyMask == 0 ? 1U : preResource.remoteNotifyMask; + + if (DirectTraceEnabled()) { + std::cerr << "TileXRDirectCcuTrace alltoallSpec" + << " direction=RemoteToLocal" + << " localRank=" << alltoallSpec.localRank + << " localGsa=" << alltoallSpec.localGsa + << " localXn=" << alltoallSpec.localXn + << " remoteGsa=" << alltoallSpec.remoteGsa + << " remoteXn=" << alltoallSpec.remoteXn + << " lengthXn=" << alltoallSpec.lengthXn + << " preChannelId=" << alltoallSpec.preSyncChannelId + << " copyChannelId=" << alltoallSpec.copyChannelId + << " postChannelId=" << alltoallSpec.postSyncChannelId + << " preNotifyCke=" << alltoallSpec.preSyncRemoteNotifyCke + << " copyCompletionCke=" << alltoallSpec.copyCompletionCke + << " postNotifyCke=" << alltoallSpec.postSyncRemoteNotifyCke + << " localRecvAddr=0x" << std::hex << alltoallSpec.localRecvAddr + << " remoteSendAddr=0x" << alltoallSpec.remoteSendAddr + << " bytes=0x" << alltoallSpec.bytes + << std::dec << std::endl; + } + + TileXRCcuProgram program; + TileXRCcuAllToAllProgramReport alltoallReport; + if (TileXRCcuBuildAllToAll2RankProgram(alltoallSpec, &program.sync, &alltoallReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = alltoallReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (program.sync.empty() || program.sync.size() > std::numeric_limits::max()) { + if (report != nullptr) { + report->message = "invalid direct CCU alltoall instruction count"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + uint32_t SignalWaitInstructionCount(TileXRCcuSignalWaitProgramRole role) { if (role == TileXRCcuSignalWaitProgramRole::Wait) { @@ -1118,6 +1285,7 @@ int TileXRCcuSubmitPreparedTasks( int RunDirectInstallAttemptImpl( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectMemoryCopySpec* memoryCopy, + const TileXRCcuDirectAllToAll2RankSpec* alltoall, const TileXRCcuDirectSignalWaitSpec* signalWait, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) @@ -1163,17 +1331,22 @@ int RunDirectInstallAttemptImpl( ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); - attempt->resourceRequest.sqeArgCount = memoryCopy == nullptr && signalWait == nullptr ? options.sqeArgCount : 0U; + const bool customProgram = memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr; + attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = - memoryCopy == nullptr && signalWait == nullptr ? options.syncResourceCount : 1U; + alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : + customProgram ? 1U : options.syncResourceCount; attempt->resourceRequest.syncInstructionCount = memoryCopy != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : + alltoall != nullptr ? + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : signalWait != nullptr ? std::max(options.syncInstructionCount, SignalWaitInstructionCount(signalWait->role)) : options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; attempt->resourceRequest.barrierMode = + alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? options.barrierMode : SignalWaitBarrierMode(signalWait->role); TileXRCcuResourceAllocator allocator; @@ -1201,8 +1374,19 @@ int RunDirectInstallAttemptImpl( "failed to configure direct CCU memory copy resources" : report->message); } + } else if (alltoall != nullptr) { + ret = ConfigureDirectAllToAll2RankResources(options, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to configure direct CCU alltoall resources" : + report->message); + } } attempt->plan.barrierMode = + alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? attempt->plan.barrierMode : SignalWaitBarrierMode(signalWait->role); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); @@ -1224,7 +1408,7 @@ int RunDirectInstallAttemptImpl( "failed to reconcile direct CCU lower-layer peer resources" : report->message); } - if (memoryCopy == nullptr) { + if (!customProgram) { ret = PopulateHcommStyleSqeTaskArgs(attempt, report); if (ret != TILEXR_SUCCESS) { return Fail(attempt, report, "failed to populate direct CCU SQE task arguments"); @@ -1234,6 +1418,8 @@ int RunDirectInstallAttemptImpl( TileXRCcuLaunchPackageReport packageReport; ret = memoryCopy != nullptr ? BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report) : + alltoall != nullptr ? + BuildDirectAllToAll2RankLaunchPackage(*alltoall, attempt, report) : signalWait != nullptr ? BuildDirectSignalWaitLaunchPackage(*signalWait, attempt, report) : TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport); @@ -1241,10 +1427,12 @@ int RunDirectInstallAttemptImpl( return Fail( attempt, report, - memoryCopy == nullptr && signalWait == nullptr ? packageReport.message : + !customProgram ? packageReport.message : (report == nullptr || report->message.empty() ? (memoryCopy != nullptr ? "failed to build direct CCU memory copy launch package" : + alltoall != nullptr ? + "failed to build direct CCU alltoall launch package" : "failed to build direct CCU signal/wait launch package") : report->message)); } @@ -1308,7 +1496,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1327,7 +1515,29 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, attempt, report); +} + +int TileXRCcuRunDirectAllToAll2RankInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (alltoall.localRank > 1U || + alltoall.localRecvAddr == 0 || alltoall.localRecvToken == 0 || + alltoall.remoteSendAddr == 0 || alltoall.remoteSendToken == 0 || + alltoall.bytes == 0 || alltoall.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES || + alltoall.memSlicePerBlock == 0 || + alltoall.memSlicePerBlock > TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK) { + ResetReport(report); + ClearAttempt(attempt); + if (report != nullptr) { + report->message = "invalid direct CCU alltoall address/token/slice inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, attempt, report); } int TileXRCcuRunDirectSignalWaitInstallAttempt( @@ -1336,7 +1546,7 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, &signalWait, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index 7a7271cf..eb63680f 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -8,6 +8,7 @@ #include "ccu/tilexr_ccu_install_provider.h" #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" +#include "ccu/tilexr_ccu_alltoall_program.h" #include "ccu/tilexr_ccu_memory_program.h" #include "ccu/tilexr_ccu_signal_wait_program.h" #include "ccu/tilexr_ccu_specs.h" @@ -69,6 +70,17 @@ struct TileXRCcuDirectMemoryCopySpec { uint64_t lengthBytes = 0; }; +struct TileXRCcuDirectAllToAll2RankSpec { + uint32_t localRank = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t remoteSendAddr = 0; + uint64_t remoteSendToken = 0; + uint64_t bytes = 0; + uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; +}; + struct TileXRCcuDirectSignalWaitSpec { TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; }; @@ -129,6 +141,12 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectAllToAll2RankInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuRunDirectSignalWaitInstallAttempt( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectSignalWaitSpec& signalWait, diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index 054e9272..04590eea 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -118,6 +118,65 @@ bool RangeContains(uint32_t start, uint32_t count, uint32_t value) return count != 0 && value >= start && value < start + count; } +void NormalizeVerifiedEndpointRouteJettyWindow(TileXRCcuLowerLayerTransportSnapshot* snapshot) +{ + if (snapshot == nullptr || snapshot->routes.empty()) { + return; + } + + uint16_t firstExplicitStart = 0; + uint16_t minExplicitStart = 0; + uint32_t explicitEnd = 0; + uint32_t explicitStartCount = 0; + bool allExplicitStartsEqual = true; + for (const auto& route : snapshot->routes) { + if (route.startJettyId == 0) { + continue; + } + if (firstExplicitStart == 0) { + firstExplicitStart = route.startJettyId; + minExplicitStart = route.startJettyId; + } else if (route.startJettyId != firstExplicitStart) { + allExplicitStartsEqual = false; + minExplicitStart = std::min(minExplicitStart, route.startJettyId); + } + explicitEnd = std::max(explicitEnd, static_cast(route.startJettyId) + 1U); + ++explicitStartCount; + } + + if (explicitStartCount != 0 && allExplicitStartsEqual && snapshot->routes.size() > 1) { + const uint32_t routeCount = static_cast(snapshot->routes.size()); + for (uint32_t i = 0; i < routeCount; ++i) { + snapshot->routes[i].startJettyId = CheckedU16(static_cast(firstExplicitStart) + i); + } + minExplicitStart = firstExplicitStart; + explicitEnd = static_cast(firstExplicitStart) + routeCount; + } + + if (minExplicitStart != 0) { + snapshot->startJettyId = minExplicitStart; + } + + uint32_t requiredJettyCount = static_cast(snapshot->routes.size()); + if (snapshot->startJettyId != 0) { + uint32_t end = static_cast(snapshot->startJettyId) + snapshot->routes.size(); + for (uint32_t i = 0; i < snapshot->routes.size(); ++i) { + const uint32_t start = snapshot->routes[i].startJettyId == 0 ? + static_cast(snapshot->startJettyId) + i : + snapshot->routes[i].startJettyId; + end = std::max(end, start + 1U); + } + requiredJettyCount = std::max( + requiredJettyCount, + end - static_cast(snapshot->startJettyId)); + } else if (explicitEnd != 0 && minExplicitStart != 0) { + requiredJettyCount = std::max( + requiredJettyCount, + explicitEnd - static_cast(minExplicitStart)); + } + snapshot->pfeJettyCount = CheckedU16(std::max(snapshot->pfeJettyCount, requiredJettyCount)); +} + uint32_t SelectLowerLayerPfeOffset(uint8_t dieId, uint32_t pfeId) { if (LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE", "hcomm_die")) { @@ -531,6 +590,7 @@ int TileXRCcuBuildLowerLayerTransportTemplate( const uint32_t requiredJettyCount = verifiedJettyEnd - verifiedStartJettyId; result.pfeJettyCount = CheckedU16(std::max(result.pfeJettyCount, requiredJettyCount)); } + NormalizeVerifiedEndpointRouteJettyWindow(&result); *snapshot = result; FillTemplateReport(*snapshot, report); @@ -575,10 +635,12 @@ int TileXRCcuOverlayVerifiedEndpointRoutes( routeIt->localDoorbellTokenId = verified.localDoorbellTokenId; routeIt->localDoorbellTokenValue = verified.localDoorbellTokenValue; routeIt->localSqDepth = verified.localSqDepth; + routeIt->startJettyId = verified.startJettyId; routeIt->endpointRouteVerified = true; routeIt->channelResourceOwnerVerified = verified.channelResourceOwnerVerified; routeIt->transportResourceExchangeVerified = verified.transportResourceExchangeVerified; } + NormalizeVerifiedEndpointRouteJettyWindow(snapshot); FillTemplateReport(*snapshot, report); return TILEXR_SUCCESS; @@ -602,28 +664,31 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( return Fail(plan, report, "too many lower-layer CCU transport routes"); } + TileXRCcuLowerLayerTransportSnapshot normalized = snapshot; + NormalizeVerifiedEndpointRouteJettyWindow(&normalized); + TileXRCcuLowerLayerPlanSpec spec; - spec.msidToken = snapshot.msidToken; - spec.pfe.dieId = snapshot.dieId; - spec.pfe.pfeOffset = snapshot.pfeOffset; - spec.pfe.startJettyId = snapshot.startJettyId; - spec.pfe.jettyCount = snapshot.pfeJettyCount; - spec.pfe.startLocalJettyCtxId = snapshot.startLocalJettyCtxId; - spec.xnClear.dieId = snapshot.dieId; - spec.xnClear.startXnId = snapshot.xnStartId; - spec.xnClear.count = snapshot.xnCount; - spec.xnClear.valid = snapshot.xnCount != 0; - spec.ckeClear.dieId = snapshot.dieId; - spec.ckeClear.startCkeId = snapshot.ckeStartId; - spec.ckeClear.count = snapshot.ckeCount; - spec.ckeClear.valid = snapshot.ckeCount != 0; + spec.msidToken = normalized.msidToken; + spec.pfe.dieId = normalized.dieId; + spec.pfe.pfeOffset = normalized.pfeOffset; + spec.pfe.startJettyId = normalized.startJettyId; + spec.pfe.jettyCount = normalized.pfeJettyCount; + spec.pfe.startLocalJettyCtxId = normalized.startLocalJettyCtxId; + spec.xnClear.dieId = normalized.dieId; + spec.xnClear.startXnId = normalized.xnStartId; + spec.xnClear.count = normalized.xnCount; + spec.xnClear.valid = normalized.xnCount != 0; + spec.ckeClear.dieId = normalized.dieId; + spec.ckeClear.startCkeId = normalized.ckeStartId; + spec.ckeClear.count = normalized.ckeCount; + spec.ckeClear.valid = normalized.ckeCount != 0; uint32_t routeIndex = 0; - for (const auto& route : snapshot.routes) { + for (const auto& route : normalized.routes) { TileXRCcuLowerLayerJettySpec jetty; - jetty.dieId = snapshot.dieId; - jetty.pfeId = snapshot.pfeId; - jetty.startJettyCtxId = static_cast(snapshot.startLocalJettyCtxId + routeIndex); + jetty.dieId = normalized.dieId; + jetty.pfeId = normalized.pfeId; + jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + routeIndex); jetty.doorbellVa = route.localDoorbellVa == 0 ? route.doorbellVa : route.localDoorbellVa; jetty.doorbellTokenId = route.localDoorbellTokenId == 0 ? route.doorbellTokenId : @@ -636,13 +701,13 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( spec.jettys.push_back(jetty); TileXRCcuLowerLayerChannelSpec channel; - channel.dieId = snapshot.dieId; + channel.dieId = normalized.dieId; channel.channelId = route.channelId; channel.remoteEid = route.remoteEid; channel.tpn = route.tpn; - channel.sourcePfeId = snapshot.pfeId; + channel.sourcePfeId = normalized.pfeId; channel.startJettyId = route.startJettyId == 0 ? - static_cast(snapshot.startJettyId + routeIndex) : + static_cast(normalized.startJettyId + routeIndex) : route.startJettyId; channel.jettyCount = 1; channel.memoryTokenId = route.memoryTokenId; @@ -659,16 +724,16 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( } TileXRCcuRemoteXnBindingProof remoteXn; - remoteXn.dieId = snapshot.dieId; + remoteXn.dieId = normalized.dieId; remoteXn.channelId = static_cast(route.channelId); - remoteXn.localXn = static_cast(snapshot.xnStartId + routeIndex); + remoteXn.localXn = static_cast(normalized.xnStartId + routeIndex); remoteXn.remoteXn = route.remoteXnId; remoteXn.notifyCke = route.remoteNotifyCke == 0 ? - static_cast(snapshot.ckeStartId + routeIndex) : + static_cast(normalized.ckeStartId + routeIndex) : route.remoteNotifyCke; remoteXn.peerRank = route.peerRank; remoteXn.peerExchangeObserved = route.remoteXnId != 0; - remoteXn.localWaitCke = static_cast(snapshot.ckeStartId + routeIndex); + remoteXn.localWaitCke = static_cast(normalized.ckeStartId + routeIndex); remoteXn.endpointRouteVerified = route.endpointRouteVerified; remoteXn.channelResourceOwnerVerified = route.channelResourceOwnerVerified; remoteXn.transportResourceExchangeVerified = route.transportResourceExchangeVerified; @@ -680,8 +745,8 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( if (ret != TILEXR_SUCCESS) { return ret; } - AppendRemoteXnClears(snapshot, plan); - AppendRemoteNotifyCkeClears(snapshot, plan); + AppendRemoteXnClears(normalized, plan); + AppendRemoteNotifyCkeClears(normalized, plan); FillReport(*plan, report); return TILEXR_SUCCESS; } diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 979798fc..86b65118 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -77,6 +77,9 @@ constexpr const char* kP2pCcuCopyBytesEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COP constexpr const char* kP2pCcuCopyActiveRankEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK"; constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION"; constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; +constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; +constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; +constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; @@ -127,6 +130,18 @@ struct P2pCcuCopyState { bool passed = false; }; +struct AllToAllState { + DeviceBuffer source; + DeviceBuffer destination; + std::vector expected; + std::vector observed; + size_t bytes = 0; + int initRet = ACL_SUCCESS; + int readRet = ACL_SUCCESS; + uint32_t mismatchCount = 0; + bool passed = false; +}; + bool EnvFlag(const char* name) { const char* value = std::getenv(name); @@ -142,6 +157,11 @@ bool SignalWaitSmokeEnabled() return EnvFlag(kSignalWaitEnv); } +bool AllToAllSmokeEnabled() +{ + return EnvFlag(kAllToAllEnv); +} + bool BarrierSmokeEnabled() { return EnvFlag(kSignalWaitBarrierEnv); @@ -518,6 +538,85 @@ int CheckP2pCcuCopyState(P2pCcuCopyState* state) return state->passed ? ACL_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; } +size_t AllToAllBytesFromEnv() +{ + return static_cast(EnvInt(kAllToAllBytesEnv, 2 * 1024 * 1024)); +} + +int AllToAllMemSlicePerLoopFromEnv() +{ + return EnvInt(kAllToAllMemSlicePerLoopEnv, 8); +} + +int InitAllToAllState(int rank, int peer, AllToAllState* state) +{ + if (state == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->bytes = AllToAllBytesFromEnv(); + if (state->bytes != 2U * 1024U * 1024U || AllToAllMemSlicePerLoopFromEnv() != 8) { + state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + return state->initRet; + } + state->expected = BuildP2pCcuCopyPattern(peer, state->bytes); + state->observed.assign(state->bytes, 0); + const std::vector source = BuildP2pCcuCopyPattern(rank, state->bytes); + std::vector destination(state->bytes, 0xa5U); + + int ret = state->source.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = state->destination.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->source.ptr, + state->bytes, + source.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + state->initRet = ret; + return ret; +} + +int CheckAllToAllState(AllToAllState* state) +{ + if (state == nullptr || state->destination.ptr == nullptr || state->bytes == 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->readRet = aclrtMemcpy( + state->observed.data(), + state->observed.size(), + state->destination.ptr, + state->bytes, + ACL_MEMCPY_DEVICE_TO_HOST); + if (state->readRet != ACL_SUCCESS) { + return state->readRet; + } + state->mismatchCount = 0; + for (size_t i = 0; i < state->bytes; ++i) { + if (state->observed[i] != state->expected[i]) { + ++state->mismatchCount; + } + } + state->passed = state->mismatchCount == 0; + return state->passed ? ACL_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + void PrintP2pCcuCopy( int rank, int peer, @@ -1356,6 +1455,203 @@ int RunInactiveP2pCcuCopyRank(int rank, int peer, int rankSize, P2pCcuCopyState* return checkRet == ACL_SUCCESS ? 0 : 14; } +int RunAllToAllCopyPhase( + DirectCcuSmokeContext* context, + int rank, + int rankSize, + int device, + int phase, + AllToAllState* alltoall) +{ + if (context == nullptr || alltoall == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 7; + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + const int peer = 1 - rank; + const bool active = rank == phase; + const bool submitRequested = EnvFlag(kSubmitEnv); + + std::cout << "tilexr_ccu_alltoall phase" + << " rank=" << rank + << " phase=" << phase + << " direction=RemoteToLocal" + << std::endl; + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall->initRet != ACL_SUCCESS ? + alltoall->initRet : + context->planner.PrepareDirectCcuMemoryCopyInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall->source.ptr), + reinterpret_cast(alltoall->destination.ptr), + alltoall->bytes, + static_cast(peer), + TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + if (!active) { + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested) { + const bool phaseReady = WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady); + finalRet = phaseReady ? 0 : 13; + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; + } + + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before alltoall aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after alltoall aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " phase=" << phase + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + if (!attempt.submitTasks.empty()) { + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + } + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + aclrtDestroyStream(stream); + } + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + +int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + AllToAllState alltoall; + alltoall.initRet = InitAllToAllState(rank, peer, &alltoall); + + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 7; + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << alltoall.bytes + << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() + << " blockCount=64" + << " hostPhases=2" + << std::endl; + PrintConfig(options, rankSize); + + int finalRet = alltoall.initRet == ACL_SUCCESS ? 0 : alltoall.initRet; + for (int phase = 0; phase < 2 && finalRet == 0; ++phase) { + finalRet = RunAllToAllCopyPhase(context, rank, rankSize, device, phase, &alltoall); + } + if (finalRet == 0) { + const int checkRet = CheckAllToAllState(&alltoall); + if (checkRet != ACL_SUCCESS) { + finalRet = 14; + } + } + + if (finalRet == 0) { + std::cout << "tilexr_ccu_alltoall result passed=1" + << " rank=" << rank + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << std::endl; + } else { + std::cout << "tilexr_ccu_alltoall result passed=0" + << " rank=" << rank + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << std::endl; + } + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_alltoall fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + return finalRet; +} + int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { if (context == nullptr) { @@ -1489,6 +1785,9 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { + if (AllToAllSmokeEnabled()) { + return RunAllToAllSmokeForRank(context, rank, rankSize, device); + } if (SignalWaitSmokeEnabled() || BarrierSmokeEnabled()) { return RunSignalWaitSmokeForRank(context, rank, rankSize, device); } diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index e8fac065..8e9380e8 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -87,6 +87,11 @@ signal_wait_mode_enabled() [ "${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0}" = "1" ] } +alltoall_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0}" = "1" ] +} + apply_p2p_ccu_copy_defaults() { if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" != "1" ]; then @@ -154,8 +159,46 @@ apply_signal_wait_defaults() export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" } +apply_alltoall_defaults() +{ + if ! alltoall_mode_enabled; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_GSA_START="${TILEXR_CCU_PROBE_GSA_START:-510}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_BARRIER_MODE="${TILEXR_CCU_DIRECT_BARRIER_MODE:-sync_cke}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" + export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}" +} + apply_p2p_ccu_copy_defaults apply_signal_wait_defaults +apply_alltoall_defaults if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" @@ -170,6 +213,9 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT \ TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK \ TILEXR_CCU_DIRECT_SMOKE_BARRIER \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL \ + TILEXR_CCU_ALLTOALL_BYTES \ + TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do diagnostic_value="${!diagnostic_var:-}" @@ -428,6 +474,15 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL}") +fi +if [ "${TILEXR_CCU_ALLTOALL_BYTES:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_BYTES=${TILEXR_CCU_ALLTOALL_BYTES}") +fi +if [ "${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN}") fi @@ -559,7 +614,7 @@ for token_field in "${resource_window_token_fields[@]}"; do fi done -echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" +echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then thread_log="${work_dir}/ccu_thread.log" @@ -574,7 +629,34 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then echo "thread log: ${thread_log}" >&2 exit 4 fi - if signal_wait_mode_enabled; then + if alltoall_mode_enabled; then + if [ "$(grep -c "tilexr_ccu_alltoall prepare ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode prepare did not return success for both ranks" >&2 + exit 5 + fi + if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode prepare did not complete install attempt for both ranks" >&2 + exit 6 + fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode submit requested but prepare did not reach submitReady=1" >&2 + exit 6 + fi + if [ "$(grep -c "tilexr_ccu_alltoall submit ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode submit did not return success for both ranks" >&2 + exit 7 + fi + if [ "$(grep -c "tilexr_ccu_alltoall timing" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode timing was not reported for both ranks" >&2 + exit 8 + fi + fi + if [ "$(grep -c "tilexr_ccu_alltoall result passed=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode result did not pass for both ranks" >&2 + exit 8 + fi + elif signal_wait_mode_enabled; then if [ "$(grep -c "tilexr_ccu_signal_wait prepare ret=0" "${thread_log}")" -lt 2 ]; then echo "ERROR: direct CCU signal/wait thread-mode prepare did not return success for both ranks" >&2 exit 5 @@ -666,7 +748,18 @@ if [ "${rank0_status}" -ne 0 ] || [ "${rank1_status}" -ne 0 ]; then exit 4 fi -if signal_wait_mode_enabled; then +if alltoall_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_alltoall prepare ret=0" "${log}"; then + echo "ERROR: direct CCU alltoall prepare did not return success in ${log}" >&2 + exit 5 + fi + if ! grep -q "installSucceeded=1" "${log}"; then + echo "ERROR: direct CCU alltoall prepare did not complete install attempt in ${log}" >&2 + exit 6 + fi + done +elif signal_wait_mode_enabled; then for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_signal_wait prepare ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait prepare did not return success in ${log}" >&2 @@ -705,6 +798,17 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then fi done for log in "${rank0_log}" "${rank1_log}"; do + if alltoall_mode_enabled; then + if ! grep -q "tilexr_ccu_alltoall submit ret=0" "${log}"; then + echo "ERROR: direct CCU alltoall submit did not return success in ${log}" >&2 + exit 7 + fi + if ! grep -q "tilexr_ccu_alltoall timing" "${log}"; then + echo "ERROR: direct CCU alltoall timing was not reported in ${log}" >&2 + exit 8 + fi + continue + fi if signal_wait_mode_enabled; then if ! grep -q "tilexr_ccu_signal_wait submit ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait submit did not return success in ${log}" >&2 @@ -733,7 +837,14 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then done fi -if signal_wait_mode_enabled; then +if alltoall_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_alltoall result passed=1" "${log}"; then + echo "ERROR: direct CCU alltoall result did not pass in ${log}" >&2 + exit 9 + fi + done +elif signal_wait_mode_enabled; then for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then echo "ERROR: direct CCU signal/wait result did not pass in ${log}" >&2 diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py new file mode 100644 index 00000000..2049c528 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -0,0 +1,372 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" +ALLTOALL_HEADER = COMM_DIR / "ccu" / "tilexr_ccu_alltoall_program.h" +ALLTOALL_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_alltoall_program.cpp" +MICROCODE_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_microcode.cpp" +MEMORY_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_memory_program.cpp" +COMM_CMAKE = COMM_DIR / "CMakeLists.txt" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + #include + + using namespace TileXR; + + int main() + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localRecvAddr = 0x10000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 0x101; + spec.remoteGsa = 0x102; + spec.localXn = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x203; + spec.channelId = 0x12; + spec.copyCompletionCke = 0x301; + spec.preSyncLocalWaitCke = 0x302; + spec.preSyncRemoteNotifyCke = 0x303; + spec.postSyncLocalWaitCke = 0x304; + spec.postSyncRemoteNotifyCke = 0x305; + spec.sourceCke = 0x306; + spec.ckeMask = 1; + + std::vector program; + TileXRCcuAllToAllProgramReport report; + int ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS) { + std::cerr << "builder failed: " << report.message << "\n"; + return 1; + } + const uint32_t expectedBlocks = 64; + const uint32_t expectedInstructions = 4 + 4 + expectedBlocks * 7 + 4 + 4 + 4 + 4 + 1; + if (report.blockCount != expectedBlocks || + report.copyInstructionCount != expectedBlocks * 7 || + report.preSyncInstructionCount != 4 || + report.postSyncInstructionCount != 4 || + report.finishInstructionCount != 1 || + report.totalInstructionCount != expectedInstructions || + program.size() != expectedInstructions) { + std::cerr << "unexpected report counts" + << " blocks=" << report.blockCount + << " copyInst=" << report.copyInstructionCount + << " pre=" << report.preSyncInstructionCount + << " post=" << report.postSyncInstructionCount + << " finish=" << report.finishInstructionCount + << " total=" << report.totalInstructionCount + << " size=" << program.size() << "\n"; + return 2; + } + if (report.bytesPerBlock != 32768 || report.message != "ok") { + std::cerr << "unexpected block size or message\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_two_rank_program_serializes_copy_phases_by_local_rank(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + + using namespace TileXR; + + constexpr uint16_t kTransRmtMemToLocMemHeader = 0x1008U; + + uint16_t Header(const TileXRCcuInstr& instr) + { + return static_cast(instr.words[0] & 0xffffU); + } + + TileXRCcuAllToAll2RankProgramSpec ValidSpec(uint32_t localRank) + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localRank = localRank; + spec.localRecvAddr = 0x10000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 0x101; + spec.remoteGsa = 0x102; + spec.localXn = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x203; + spec.channelId = 0x12; + spec.copyCompletionCke = 0x301; + spec.preSyncLocalWaitCke = 0x302; + spec.preSyncRemoteNotifyCke = 0x303; + spec.postSyncLocalWaitCke = 0x304; + spec.postSyncRemoteNotifyCke = 0x305; + spec.sourceCke = 0x306; + spec.ckeMask = 1; + return spec; + } + + size_t FirstCopyIndex(const std::vector& program) + { + for (size_t i = 0; i < program.size(); ++i) { + if (Header(program[i]) == kTransRmtMemToLocMemHeader) { + return i; + } + } + return program.size(); + } + + uint32_t CopyInstructionCount(const std::vector& program) + { + uint32_t count = 0; + for (const auto& instr : program) { + if (Header(instr) == kTransRmtMemToLocMemHeader) { + ++count; + } + } + return count; + } + + int main() + { + std::vector rank0; + std::vector rank1; + TileXRCcuAllToAllProgramReport report0; + TileXRCcuAllToAllProgramReport report1; + int ret0 = TileXRCcuBuildAllToAll2RankProgram(ValidSpec(0), &rank0, &report0); + int ret1 = TileXRCcuBuildAllToAll2RankProgram(ValidSpec(1), &rank1, &report1); + if (ret0 != TILEXR_SUCCESS || ret1 != TILEXR_SUCCESS) { + std::cerr << "builder failed rank0=" << report0.message + << " rank1=" << report1.message << "\n"; + return 1; + } + if (report0.totalInstructionCount != 473 || report1.totalInstructionCount != 473 || + rank0.size() != 473 || rank1.size() != 473) { + std::cerr << "unexpected two-phase instruction count" + << " rank0=" << rank0.size() + << " rank1=" << rank1.size() + << " report0=" << report0.totalInstructionCount + << " report1=" << report1.totalInstructionCount << "\n"; + return 2; + } + if (CopyInstructionCount(rank0) != 64 || CopyInstructionCount(rank1) != 64) { + std::cerr << "each rank should issue exactly 64 remote-to-local transfers\n"; + return 3; + } + const size_t rank0FirstCopy = FirstCopyIndex(rank0); + const size_t rank1FirstCopy = FirstCopyIndex(rank1); + if (rank0FirstCopy != 13 || rank1FirstCopy != 21) { + std::cerr << "copy phases overlap or moved unexpectedly" + << " rank0FirstCopy=" << rank0FirstCopy + << " rank1FirstCopy=" << rank1FirstCopy << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builder_rejects_invalid_slice_configuration(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + + using namespace TileXR; + + TileXRCcuAllToAll2RankProgramSpec ValidSpec() + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localRecvAddr = 0x10000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(3, 4, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 1; + spec.remoteGsa = 2; + spec.localXn = 3; + spec.remoteXn = 4; + spec.lengthXn = 5; + spec.channelId = 6; + spec.copyCompletionCke = 7; + spec.preSyncLocalWaitCke = 8; + spec.preSyncRemoteNotifyCke = 9; + spec.postSyncLocalWaitCke = 10; + spec.postSyncRemoteNotifyCke = 11; + spec.sourceCke = 12; + spec.ckeMask = 1; + return spec; + } + + int main() + { + std::vector program; + TileXRCcuAllToAllProgramReport report; + + auto spec = ValidSpec(); + spec.memSlicePerBlock = 9; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("memSlicePerBlock") == std::string::npos) { + std::cerr << "memSlicePerBlock > 8 accepted: " << report.message << "\n"; + return 1; + } + + spec = ValidSpec(); + spec.bytes = 4097; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("4KB") == std::string::npos) { + std::cerr << "non-4KB size accepted: " << report.message << "\n"; + return 2; + } + + spec = ValidSpec(); + spec.remoteSendToken = 0; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("token") == std::string::npos) { + std::cerr << "missing token accepted: " << report.message << "\n"; + return 3; + } + + spec = ValidSpec(); + spec.localRank = 2; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("localRank") == std::string::npos) { + std::cerr << "invalid localRank accepted: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builder_is_wired_and_has_no_hccl_dependency_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = ALLTOALL_HEADER.read_text(encoding="utf-8") + source = ALLTOALL_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_alltoall_program.h", cmake) + self.assertIn("ccu/tilexr_ccu_alltoall_program.cpp", cmake) + self.assertIn("TileXRCcuBuildAllToAll2RankProgram", header) + self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) + self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_two_phase_rank_role_is_part_of_program_contract(self): + header = ALLTOALL_HEADER.read_text(encoding="utf-8") + source = ALLTOALL_SOURCE.read_text(encoding="utf-8") + orchestrator = (COMM_DIR / "ccu" / "tilexr_ccu_direct_orchestrator.cpp").read_text(encoding="utf-8") + planner = (COMM_DIR / "ccu" / "tilexr_ccu_collective_planner.cpp").read_text(encoding="utf-8") + + self.assertIn("uint32_t localRank = 0", header) + self.assertIn("append copy only for the local rank's active phase", source) + self.assertIn("alltoallSpec.localRank = alltoall.localRank", orchestrator) + self.assertIn("alltoall.localRank = static_cast(rank)", planner) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 55e645b8..47aefa71 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -223,6 +223,24 @@ def test_p2p_ccu_copy_uses_original_va_for_microcode_and_imported_segva_for_rout self.assertNotIn("QueryDirectCcuProcessMemoryToken(sourceAddr", endpoint_builder) self.assertNotIn("QueryDirectCcuProcessMemoryToken(destinationAddr", endpoint_builder) + def test_alltoall_overrides_only_copy_route_memory_not_sync_routes(self): + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + prepare_alltoall = planner[ + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt"): + planner.index("#endif", planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt")) + ] + override_apply = planner[ + planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride"): + planner.index("#endif", planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride")) + ] + + self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(", prepare_alltoall) + self.assertIn("1U", prepare_alltoall) + self.assertIn("uint32_t routeIndex = 0", override_apply) + self.assertIn("routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_", override_apply) + self.assertIn("++routeIndex", override_apply) + self.assertIn("directCcuRemoteRouteMemoryOverrideAllRoutes_", override_apply) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index b2bb6587..b9d83657 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -16,6 +16,8 @@ REPO_ROOT = Path(__file__).resolve().parents[2] DIRECT_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_orchestrator.h" DIRECT_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_orchestrator.cpp" +PLANNER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.h" +PLANNER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp" INSTALL_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_install_provider.cpp" PROVIDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_provider.cpp" PACKAGE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.cpp" @@ -27,6 +29,7 @@ BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" MEMORY_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_memory_program.cpp" +ALLTOALL_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_alltoall_program.cpp" SIGNAL_WAIT_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_signal_wait_program.cpp" RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime.cpp" LOWER_LAYER_PLAN_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_plan_builder.cpp" @@ -138,6 +141,7 @@ def compile_and_run(self, code: str): str(BARRIER_SOURCE), str(MICROCODE_SOURCE), str(MEMORY_PROGRAM_SOURCE), + str(ALLTOALL_PROGRAM_SOURCE), str(SIGNAL_WAIT_PROGRAM_SOURCE), str(RUNTIME_SOURCE), str(LOWER_LAYER_PLAN_SOURCE), @@ -1689,6 +1693,7 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("ccu/tilexr_ccu_direct_orchestrator.cpp", cmake) self.assertIn("struct TileXRCcuDirectInstallOptions", header) self.assertIn("struct TileXRCcuDirectMemoryCopySpec", header) + self.assertIn("struct TileXRCcuDirectAllToAll2RankSpec", header) self.assertIn("struct TileXRCcuDirectSignalWaitSpec", header) self.assertIn("struct TileXRCcuDirectInstallAttempt", header) self.assertIn("struct TileXRCcuDirectInstallReport", header) @@ -1703,6 +1708,7 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TileXRCcuSubmitPreparedTasks", header) self.assertIn("TileXRCcuRunDirectInstallAttempt", header) self.assertIn("TileXRCcuRunDirectMemoryCopyInstallAttempt", header) + self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", header) self.assertIn("TileXRCcuRunDirectSignalWaitInstallAttempt", header) self.assertIn("TileXRCcuDecodeBasicInfo", source) self.assertIn("TileXRCcuBuildResourceSpec", source) @@ -1710,6 +1716,8 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TileXRCcuBuildLaunchPackage", source) self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) self.assertIn("BuildDirectMemoryCopyLaunchPackage", source) + self.assertIn("TileXRCcuBuildAllToAll2RankProgram", source) + self.assertIn("BuildDirectAllToAll2RankLaunchPackage", source) self.assertIn("TileXRCcuBuildSignalWaitProgram", source) self.assertIn("BuildDirectSignalWaitLaunchPackage", source) self.assertIn("spec.localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke", source) @@ -1735,6 +1743,24 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER", source) + def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) + self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) + self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke", source) + self.assertIn("preSyncRemoteNotifyCke = preResource.notifyCke", source) + self.assertIn("alltoallSpec.copyCompletionCke =", source) + self.assertIn("copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke", source) + self.assertIn("postSyncRemoteNotifyCke = postResource.notifyCke", source) + self.assertIn("preSyncChannelId = preResource.channelId", source) + self.assertIn("copyChannelId = copyResource.channelId", source) + self.assertIn("postSyncChannelId = postResource.channelId", source) + self.assertIn("RemoteToLocal", source) + self.assertIn("uint32_t memSlicePerBlock", header) + combined = header + "\n" + source for needle in PRIVATE_CCU_PRODUCER_NEEDLES: with self.subTest(needle=needle): @@ -1763,6 +1789,22 @@ def test_direct_memory_copy_submit_tasks_keep_planned_runtime_payload(self): self.assertNotIn("NormalizeDirectMemoryCopySubmitTasks", source) self.assertNotIn("task.argSize = 1", source) + def test_collective_planner_has_private_alltoall_prepare_path(self): + header = PLANNER_HEADER.read_text(encoding="utf-8") + source = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", header) + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", source) + self.assertIn("rankSize != 2", source) + self.assertIn("alltoall.localRecvAddr = localEndpoint.destinationAddr", source) + self.assertIn("alltoall.localRecvToken = localEndpoint.destinationToken", source) + self.assertIn("remoteImportRequest = peerEndpoint.sourceRemoteImport", source) + self.assertIn("alltoall.remoteSendAddr = remoteImportRequest.addr", source) + self.assertIn("alltoall.remoteSendToken", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) + self.assertIn("tilexr-comm-direct-ccu-alltoall", source) + self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 960cdadc..e07bb012 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -574,6 +574,30 @@ def test_probe_default_sync_instruction_count_includes_hcomm_style_task1_prelude default_body, ) + def test_alltoall_submit_uses_bounded_stream_synchronize(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + alltoall_body = source[ + source.index("int RunAllToAllCopyPhase"): + source.index("int RunSignalWaitSmokeForRank") + ] + + self.assertIn("aclrtSynchronizeStreamWithTimeout", alltoall_body) + self.assertIn("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", alltoall_body) + self.assertIn("PrintMissionContext(context, attempt.submitTasks.front(), \"tilexr_ccu_alltoall\")", alltoall_body) + self.assertNotIn("const int syncRet = aclrtSynchronizeStream(stream);", alltoall_body) + + def test_alltoall_inactive_rank_still_prepares_before_host_phase_wait(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + alltoall_body = source[ + source.index("int RunAllToAllCopyPhase"): + source.index("int RunAllToAllSmokeForRank") + ] + + prepare_index = alltoall_body.index("PrepareDirectCcuMemoryCopyInstallAttempt") + inactive_index = alltoall_body.index("if (!active)") + self.assertLess(prepare_index, inactive_index) + self.assertIn("const bool phaseReady =", alltoall_body) + def test_thread_mode_rank_specific_resource_env_overrides_common_prepare_options(self): source = PROBE_SOURCE.read_text(encoding="utf-8") prepare_options_body = source[ @@ -622,6 +646,31 @@ def test_probe_keeps_hcomm_hccl_and_runtime_launch_out_of_source(self): self.assertNotIn("runtime/kernel.h", source) self.assertNotIn("rtCCULaunch", source) + def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn('kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"', source) + self.assertIn('kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"', source) + self.assertIn('kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"', source) + self.assertIn("struct AllToAllState", source) + self.assertIn("AllToAllSmokeEnabled", source) + self.assertIn("InitAllToAllState", source) + self.assertIn("RunAllToAllCopyPhase", source) + self.assertIn("RunAllToAllSmokeForRank", source) + self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) + self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + self.assertIn("tilexr_ccu_alltoall config", source) + self.assertIn("tilexr_ccu_alltoall result passed=1", source) + self.assertIn("BuildP2pCcuCopyPattern(peer", source) + + def test_smoke_runner_forwards_alltoall_env(self): + runner = (REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh").read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7", runner) + def test_probe_compiles_and_default_run_skips_without_touching_hardware(self): temp_dir, probe_bin, tile_comm_dir, cann_lib_dir, driver_lib_dir = self.compile_probe() try: diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 09b4ea6c..09cfd586 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -1563,6 +1563,119 @@ def test_overlay_verified_endpoint_routes_updates_only_matching_endpoint_fields( self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_overlay_verified_endpoint_route_expands_pfe_window_for_multi_route_snapshot(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + uint16_t DecodeChannelStartJettyId(const TileXRCcuChannelCtxDataV1& ctx) + { + const uint16_t word18 = Read16(ctx.raw, 18); + const uint16_t word20 = Read16(ctx.raw, 20); + return static_cast(((word18 >> 12U) & 0xfU) | ((word20 & 0xfffU) << 4U)); + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 3}; + allocation.localXn = {1, 0x1a0, 3}; + allocation.remoteXn = {1, 0x2a0, 3}; + allocation.notifyCke = {1, 0x360, 3}; + allocation.localWaitCke = {1, 0x220, 3}; + allocation.remoteNotifyCke = {1, 0x360, 3}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.memoryTokenValue = 0x5678; + remote.remoteXnId = 0x2a0; + remote.remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + remote.remoteEid[i] = static_cast(0x40 + i); + } + remote.tpn = 0x010203; + remote.doorbellVa = 0x1111222233334444ULL; + remote.doorbellTokenId = 0x12345; + remote.doorbellTokenValue = 0; + remote.sqDepth = 8; + remote.startJettyId = 0x400; + remote.endpointRouteVerified = true; + + std::vector remoteCcuBuffers {remote, remote, remote}; + for (uint32_t i = 0; i < remoteCcuBuffers.size(); ++i) { + remoteCcuBuffers[i].remoteCcuVa += i * 0x1000ULL; + remoteCcuBuffers[i].memoryTokenValue += i; + remoteCcuBuffers[i].remoteXnId = static_cast(0x2a0 + i); + remoteCcuBuffers[i].remoteNotifyCke = static_cast(0x360 + i); + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + + TileXRCcuLowerLayerTransportRoute verified = snapshot.routes[0]; + std::vector verifiedRoutes {verified}; + if (TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "overlay failed: " << report.message << "\n"; + return 2; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "install plan build failed: " << report.message + << " pfeJettyCount=" << snapshot.pfeJettyCount + << " routeCount=" << snapshot.routes.size() << "\n"; + return 3; + } + if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 3 || plan.pfes.empty()) { + std::cerr << "install plan shape mismatch\n"; + return 4; + } + if (plan.channels.size() != 3 || + plan.remoteXnBindings.size() != 3 || + snapshot.pfeJettyCount < snapshot.routes.size()) { + std::cerr << "multi-route lower-layer plan did not preserve all routes\n"; + return 5; + } + if (DecodeChannelStartJettyId(plan.channels[0].ctx) != 0x400 || + DecodeChannelStartJettyId(plan.channels[1].ctx) != 0x401 || + DecodeChannelStartJettyId(plan.channels[2].ctx) != 0x402) { + std::cerr << "multi-route channel jetty IDs were not expanded from verified start\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_transport_template_uses_peer_exchanged_remote_xn_ids_when_present(self): code = textwrap.dedent( r''' From 82cb4eb1a6a31724bdea58ea0557720e1d37f8cb Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Thu, 16 Jul 2026 18:01:12 +0800 Subject: [PATCH 23/41] feat(ccu): harden direct alltoall validation Add TileXR-owned Direct CCU alltoall refinements for the 2-rank P2P smoke path, including HCCL-style PreSync/PostSync masks, planner/orchestrator prepare plumbing, bounded stream synchronization, SyncXn ping smoke coverage, and phase-scoped submit ready/done gates for multi-phase host submission. Document the code changes, server environment requirements, and validated 141.62.24.62 smoke results in docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md. Verification: git diff --check; python -m unittest tests.ccu.test_tilexr_ccu_alltoall_program tests.ccu.test_tilexr_ccu_direct_orchestrator tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_lower_layer_plan_builder. --- docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md | 185 +++++++++ src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 203 +++++++--- src/comm/ccu/tilexr_ccu_alltoall_program.h | 20 + src/comm/ccu/tilexr_ccu_backend.h | 2 + .../ccu/tilexr_ccu_collective_planner.cpp | 200 +++++++++- src/comm/ccu/tilexr_ccu_collective_planner.h | 9 + .../ccu/tilexr_ccu_direct_orchestrator.cpp | 266 ++++++++++++- src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 20 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 365 ++++++++++++++++-- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 71 +++- tests/ccu/test_tilexr_ccu_alltoall_program.py | 129 +++++-- .../test_tilexr_ccu_direct_orchestrator.py | 44 ++- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 32 ++ .../test_tilexr_ccu_direct_smoke_runner.py | 17 + ...est_tilexr_ccu_lower_layer_plan_builder.py | 17 +- 15 files changed, 1422 insertions(+), 158 deletions(-) create mode 100644 docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md diff --git a/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md b/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md new file mode 100644 index 00000000..23f499ed --- /dev/null +++ b/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md @@ -0,0 +1,185 @@ +# CCU Direct AllToAll Change Record + +## Scope + +This change records the TileXR-owned Direct CCU alltoall work on the +`direct-ccu-rebased` branch. The implementation keeps HCCL and hcomm as +reference-only inputs: TileXR does not include HCCL/hcomm headers, call their +private CCU interfaces, or link new HCCL/hcomm dependencies. + +The validated target is the 2-rank P2P alltoall smoke path with a 2 MB payload. + +## Code Changes + +### AllToAll program builder + +- Added a TileXR-owned 2-rank Direct CCU alltoall program flow. +- Uses fixed 4 KB memory slices, `memSlicePerLoop <= 8`, 32 KB blocks, and + 64 blocks for the 2 MB smoke scale. +- Encodes the HCCL-style phase structure in TileXR microcode: + - PreSync: remote notification and local wait before data movement. + - Copy: CCU memory copy blocks over the existing P2P copy route. + - PostSync: completion notification and optional peer wait. +- Split synchronization masks for PreSync and PostSync so the phases no longer + reuse one ambiguous CKE bit. +- Added instruction-count reporting for pre-sync, copy, post-sync, and finish + sections. + +### Planner and orchestrator + +- Added planner/orchestrator entry points for preparing a 2-rank alltoall + launch package. +- Added direct resource configuration for alltoall sync resources, channels, + XNs, CKEs, GSA addresses, and remote endpoint routes. +- Kept the production backend guarded: Direct CCU prepare/submit helpers remain + test-only under `TILEXR_CCU_TESTING`. + +### Smoke probe and runner + +- Added alltoall smoke switches: + - `TILEXR_CCU_DIRECT_SMOKE_ALLTOALL` + - `TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION` + - `TILEXR_CCU_ALLTOALL_BYTES` + - `TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP` +- Added `TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING` for a minimal SyncXn route + sanity check. +- Added bounded stream synchronization through + `aclrtSynchronizeStreamWithTimeout` for signal/wait, SyncXn ping, and + alltoall submit paths. +- Fixed multi-phase host submit coordination by making the submit-ready and + submit-done files phase-scoped: + - `rank0.phase0.ready` + - `rank0.phase0.done` + - `rank1.phase1.ready` + - `rank1.phase1.done` +- Extended the P2P copy direction parser to accept `local_to_remote`, + `LocalToRemote`, and `1`. + +### Tests + +- Added and updated unit tests for: + - alltoall program instruction layout and resource usage, + - planner/orchestrator alltoall prepare paths, + - smoke runner environment forwarding, + - phase-scoped collective submit gates, + - bounded synchronization in smoke modes, + - lower-layer resource planning expectations. + +## Validated Commands + +Local structural tests: + +```bash +python -m unittest \ + tests.ccu.test_tilexr_ccu_alltoall_program \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_lower_layer_plan_builder +``` + +Result: + +```text +Ran 105 tests in 3.728s +OK (skipped=52) +``` + +Remote build and dependency guard on `141.62.24.62`: + +```bash +cd /home/tileXR +source scripts/common_env.sh +timeout 180s cmake -S . -B build_ccu_direct \ + -DTILEXR_CCU_TESTING=1 \ + -DCMAKE_INSTALL_PREFIX=/home/tileXR/install +timeout 240s cmake --build build_ccu_direct --target tile-comm -j2 +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh \ + build_ccu_direct/src/comm/libtile-comm.so +``` + +Result: build succeeded and the dependency guard reported no hcomm/HCCL private +CCU dependency or symbol reference. + +Remote P2P positive-control smoke on devices `6,7`: + +```bash +env ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 \ + TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ + TILEXR_TILE_COMM_LIB=/home/tileXR/build_ccu_direct/src/comm/libtile-comm.so \ + TILEXR_CCU_DIRECT_TRACE=1 \ + TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=1 \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 \ + TILEXR_CCU_SMOKE_DEVICES=6,7 \ + TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1 \ + TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=remote_to_local \ + TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ + TILEXR_CCU_SMOKE_WORK_DIR=build_ccu_direct/diag62_p2p_r2l_67_current \ + TILEXR_CCU_SMOKE_TIMEOUT=120 \ + TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS=30000 \ + TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ + TILEXR_CCU_DIRECT_READBACK_INSTRUCTIONS=1 \ + timeout 150s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +Result: rank0 and rank1 exited with status 0. + +Remote 2-rank 2 MB alltoall smoke on devices `6,7`: + +```bash +env ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 \ + TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ + TILEXR_TILE_COMM_LIB=/home/tileXR/build_ccu_direct/src/comm/libtile-comm.so \ + TILEXR_CCU_DIRECT_TRACE=1 \ + TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=1 \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 \ + TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 \ + TILEXR_CCU_SMOKE_DEVICES=6,7 \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ + TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ + TILEXR_CCU_ALLTOALL_BYTES=2097152 \ + TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 \ + TILEXR_CCU_SMOKE_WORK_DIR=build_ccu_direct/diag62_alltoall_host_phased_67_current \ + TILEXR_CCU_SMOKE_TIMEOUT=180 \ + TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS=30000 \ + TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ + TILEXR_CCU_DIRECT_READBACK_INSTRUCTIONS=1 \ + timeout 220s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +Key log evidence: + +```text +tilexr_ccu_alltoall config rank=0 peer=1 bytes=2097152 memSlicePerLoop=8 blockCount=64 hostPhases=2 +tilexr_ccu_direct_smoke collectiveSubmitReady rank=0 phase=0 localReady=1 allRanksReady=1 +tilexr_ccu_direct_smoke collectiveSubmitDone rank=0 phase=0 localResult=0 allRanksDone=1 allRanksSucceeded=1 +tilexr_ccu_direct_smoke collectiveSubmitReady rank=0 phase=1 localReady=1 allRanksReady=1 +tilexr_ccu_direct_smoke collectiveSubmitDone rank=0 phase=1 localResult=0 allRanksDone=1 allRanksSucceeded=1 +tilexr_ccu_alltoall result passed=1 rank=0 ret=0 readRet=0 mismatches=0 +tilexr_ccu_alltoall config rank=1 peer=0 bytes=2097152 memSlicePerLoop=8 blockCount=64 hostPhases=2 +tilexr_ccu_direct_smoke collectiveSubmitReady rank=1 phase=0 localReady=1 allRanksReady=1 +tilexr_ccu_direct_smoke collectiveSubmitDone rank=1 phase=0 localResult=0 allRanksDone=1 allRanksSucceeded=1 +tilexr_ccu_direct_smoke collectiveSubmitReady rank=1 phase=1 localReady=1 allRanksReady=1 +tilexr_ccu_direct_smoke collectiveSubmitDone rank=1 phase=1 localResult=0 allRanksDone=1 allRanksSucceeded=1 +tilexr_ccu_alltoall result passed=1 rank=1 ret=0 readRet=0 mismatches=0 +``` + +## Environment Notes + +- File transfer to the server used mutagen, as required by the project + instructions. +- On `141.62.24.62`, the default system CANN `libra.so` does not export + `RaCustomChannel`. Hardware CCU smoke tests must set: + +```bash +ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 +``` + +- Hardware test commands use an outer `timeout` and also pass + `TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT` for stream-level timeout control. + +## Current Limitations + +- The validated alltoall path is 2-rank P2P with host-phased submission. +- The formal LoopGroup version is not claimed as validated by this record. +- The validated route uses devices `6,7` on `141.62.24.62`. diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index e77379f4..85316ffa 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -8,6 +8,18 @@ namespace TileXR { namespace { +uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); +} + +uint16_t PostSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); +} + void ResetReport(TileXRCcuAllToAllProgramReport* report) { if (report != nullptr) { @@ -37,18 +49,18 @@ int ValidateSpec( if (program == nullptr) { return Fail(program, report, "missing output direct CCU alltoall program"); } - if (spec.localRecvAddr == 0 || spec.remoteSendAddr == 0) { + if (spec.localRank > 1U) { + return Fail(program, report, "direct CCU alltoall localRank must be 0 or 1"); + } + if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || spec.remoteSendAddr == 0) { return Fail(program, report, "missing direct CCU alltoall address"); } - if (spec.localRecvToken == 0 || spec.remoteSendToken == 0) { + if (spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.remoteSendToken == 0) { return Fail(program, report, "missing direct CCU alltoall token"); } if (spec.bytes == 0 || spec.bytes % TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES != 0) { return Fail(program, report, "direct CCU alltoall bytes must be nonzero and 4KB aligned"); } - if (spec.localRank > 1U) { - return Fail(program, report, "direct CCU alltoall localRank must be 0 or 1"); - } if (spec.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES) { return Fail(program, report, "direct CCU alltoall memorySliceBytes must be 4096"); } @@ -77,12 +89,13 @@ int ValidateSpec( int AppendSetSourceCke( const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t mask, std::vector* program, TileXRCcuAllToAllProgramReport* report) { TileXRCcuCkeSpec set; set.ckeId = spec.sourceCke; - set.mask = spec.ckeMask; + set.mask = mask; set.clearWait = true; TileXRCcuInstr instr; @@ -93,18 +106,44 @@ int AppendSetSourceCke( return TILEXR_SUCCESS; } +int AppendNotifyWait( + uint16_t localWaitCke, + uint16_t mask, + const char* phase, + bool clearCkeWait, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuCkeSpec wait; + wait.waitCkeId = localWaitCke; + wait.waitMask = mask; + wait.clearWait = true; + + TileXRCcuInstr instr; + const int ret = clearCkeWait ? + TileXRCcuEncodeClearCke(wait, &instr) : + TileXRCcuEncodeSetCke(wait, &instr); + if (ret != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " NotifyWait"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + int AppendSyncPair( uint16_t remoteNotifyCke, uint16_t localWaitCke, uint16_t channelId, const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t localMask, + uint16_t waitMask, std::vector* program, TileXRCcuAllToAllProgramReport* report) { TileXRCcuSyncCkeSpec post; post.remoteCke = remoteNotifyCke; post.localCke = spec.sourceCke; - post.localCkeMask = spec.ckeMask; + post.localCkeMask = localMask; post.channelId = channelId; post.clearWait = true; @@ -114,35 +153,98 @@ int AppendSyncPair( } program->push_back(instr); - TileXRCcuCkeSpec wait; - wait.waitCkeId = localWaitCke; - wait.waitMask = spec.ckeMask; - wait.clearWait = true; - if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { - return Fail(program, report, "failed to encode direct CCU alltoall ClearCke wait"); + return AppendNotifyWait(localWaitCke, waitMask, "PostSync", true, program, report); +} + +int AppendRemoteNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t mask, + const char* phase, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (AppendSetSourceCke(spec, mask, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuSyncCkeSpec post; + post.remoteCke = remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = mask; + post.channelId = channelId; + post.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " SyncCke notify"); } program->push_back(instr); return TILEXR_SUCCESS; } -int AppendSyncPhase( +int AppendPreSyncPhase( uint16_t remoteNotifyCke, uint16_t localWaitCke, - uint16_t channelId, + uint16_t outputChannelId, const TileXRCcuAllToAll2RankProgramSpec& spec, std::vector* program, TileXRCcuAllToAllProgramReport* report) { - TileXRCcuInstr instr; - if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { - return Fail(program, report, "failed to encode direct CCU alltoall sync prelude XN"); + const uint16_t notifyMask = PreSyncSignalMask(spec); + const uint16_t waitMask = PreSyncSignalMask(spec); + if (AppendRemoteNotify( + remoteNotifyCke, + outputChannelId, + spec, + notifyMask, + "PreSync output", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; } - program->push_back(instr); - if (AppendSetSourceCke(spec, program, report) != TILEXR_SUCCESS) { + return AppendNotifyWait(localWaitCke, waitMask, "PreSync output", false, program, report); +} + +int AppendPostSyncPhase( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + const uint16_t notifyMask = PostSyncSignalMask(spec); + const uint16_t waitMask = PostSyncSignalMask(spec); + if (AppendSetSourceCke(spec, notifyMask, program, report) != TILEXR_SUCCESS) { return TILEXR_ERROR_PARA_CHECK_FAIL; } - return AppendSyncPair(remoteNotifyCke, localWaitCke, channelId, spec, program, report); + if (!spec.postSyncWait) { + TileXRCcuSyncCkeSpec post; + post.remoteCke = remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = notifyMask; + post.channelId = channelId; + post.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PostSync notify-only SyncCke"); + } + program->push_back(instr); + return TILEXR_SUCCESS; + } + return AppendSyncPair( + remoteNotifyCke, + localWaitCke, + channelId, + spec, + notifyMask, + waitMask, + program, + report); } int AppendCopyBlock( @@ -199,12 +301,12 @@ void FillReport( return; } const uint32_t bytesPerBlock = spec.memorySliceBytes * spec.memSlicePerBlock; - report->preSyncInstructionCount = 4; + report->preSyncInstructionCount = 3; report->blockCount = static_cast(spec.bytes / bytesPerBlock); report->bytesPerBlock = bytesPerBlock; report->copyInstructionCount = report->blockCount * 7U; - report->postSyncInstructionCount = 4; - report->finishInstructionCount = 1; + report->postSyncInstructionCount = !spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U); + report->finishInstructionCount = spec.emitFinish ? 1U : 0U; report->totalInstructionCount = static_cast(program.size()); report->message = "ok"; } @@ -228,11 +330,13 @@ int TileXRCcuBuildAllToAll2RankProgram( const uint64_t bytesPerBlock = static_cast(spec.memorySliceBytes) * spec.memSlicePerBlock; const uint32_t blockCount = static_cast(spec.bytes / bytesPerBlock); const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; - const uint16_t copyChannelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; - program->reserve(4U + 4U + blockCount * 7U + 4U + 4U + 4U + 4U + 1U); + program->reserve( + 3U + blockCount * 7U + + (!spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U)) + + (spec.emitFinish ? 1U : 0U)); - ret = AppendSyncPhase( + ret = AppendPreSyncPhase( spec.preSyncRemoteNotifyCke, spec.preSyncLocalWaitCke, preSyncChannelId, @@ -243,30 +347,16 @@ int TileXRCcuBuildAllToAll2RankProgram( return ret; } - for (uint32_t phase = 0; phase < 2U; ++phase) { - ret = AppendSyncPhase( - spec.preSyncRemoteNotifyCke, - spec.preSyncLocalWaitCke, - preSyncChannelId, - spec, - program, - report); + for (uint32_t block = 0; block < blockCount; ++block) { + const uint64_t offset = static_cast(block) * bytesPerBlock; + ret = AppendCopyBlock(spec, offset, bytesPerBlock, program, report); if (ret != TILEXR_SUCCESS) { return ret; } + } - // append copy only for the local rank's active phase. - if (phase == spec.localRank) { - for (uint32_t block = 0; block < blockCount; ++block) { - const uint64_t offset = static_cast(block) * bytesPerBlock; - ret = AppendCopyBlock(spec, offset, bytesPerBlock, program, report); - if (ret != TILEXR_SUCCESS) { - return ret; - } - } - } - - ret = AppendSyncPhase( + if (spec.postSyncNotify) { + ret = AppendPostSyncPhase( spec.postSyncRemoteNotifyCke, spec.postSyncLocalWaitCke, postSyncChannelId, @@ -277,20 +367,11 @@ int TileXRCcuBuildAllToAll2RankProgram( return ret; } } - - ret = AppendSyncPhase( - spec.postSyncRemoteNotifyCke, - spec.postSyncLocalWaitCke, - postSyncChannelId, - spec, - program, - report); - if (ret != TILEXR_SUCCESS) { - return ret; - } - ret = AppendFinish(spec, program, report); - if (ret != TILEXR_SUCCESS) { - return ret; + if (spec.emitFinish) { + ret = AppendFinish(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } } FillReport(spec, *program, report); diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index c2f466cf..88cf45e8 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -18,13 +18,23 @@ constexpr uint32_t TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES = 4096U; constexpr uint32_t TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK = 8U; constexpr uint32_t TILEXR_CCU_ALLTOALL_BLOCK_BYTES = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES * TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; +constexpr uint16_t TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_TOKEN_XN_ID = 2U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_ID = 3U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_SIGNAL_MASK = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK = 2U; struct TileXRCcuAllToAll2RankProgramSpec { uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; uint64_t localRecvAddr = 0; uint64_t localRecvToken = 0; uint64_t remoteSendAddr = 0; uint64_t remoteSendToken = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; uint64_t bytes = 0; uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -33,17 +43,27 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t localXn = 0; uint16_t remoteXn = 0; uint16_t lengthXn = 0; + uint16_t preSyncLocalAddrXn = 0; + uint16_t preSyncLocalTokenXn = 0; uint16_t channelId = 0; uint16_t preSyncChannelId = 0; + uint16_t preSyncTokenChannelId = 0; uint16_t copyChannelId = 0; uint16_t postSyncChannelId = 0; uint16_t copyCompletionCke = 0; + uint16_t preSyncRemoteAddrXn = 0; + uint16_t preSyncRemoteTokenXn = 0; uint16_t preSyncLocalWaitCke = 0; uint16_t preSyncRemoteNotifyCke = 0; + uint16_t preSyncTokenLocalWaitCke = 0; + uint16_t preSyncRemoteTokenNotifyCke = 0; uint16_t postSyncLocalWaitCke = 0; uint16_t postSyncRemoteNotifyCke = 0; uint16_t sourceCke = 0; uint16_t ckeMask = 1; + bool postSyncNotify = true; + bool postSyncWait = true; + bool emitFinish = true; }; struct TileXRCcuAllToAllProgramReport { diff --git a/src/comm/ccu/tilexr_ccu_backend.h b/src/comm/ccu/tilexr_ccu_backend.h index 65ebfd82..8ede81c9 100644 --- a/src/comm/ccu/tilexr_ccu_backend.h +++ b/src/comm/ccu/tilexr_ccu_backend.h @@ -53,6 +53,8 @@ enum class TileXRCcuSignalWaitRole { struct TileXRCcuSignalWaitRequest { int peerRank = -1; TileXRCcuSignalWaitRole role = TileXRCcuSignalWaitRole::Signal; + bool overrideBarrierMode = false; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncCke; uint32_t syncInstructionCount = 0; uint16_t missionStartId = 0; uint16_t instructionStartId = 0; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 763f5376..862d0868 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -26,7 +26,8 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 4U + 4U + 64U * 7U + 4U + 4U + 4U + 4U + 1U; + 3U + 64U * 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; @@ -126,6 +127,20 @@ bool UseCcuResourceWindowForMemoryCopy() return value != nullptr && value[0] != '\0' && value[0] != '0'; } +uint16_t SelectSyncXnPingMask(const char *envName) +{ + const char *text = std::getenv(envName); + if (text == nullptr || text[0] == '\0') { + return 0; + } + char *end = nullptr; + const unsigned long parsed = std::strtoul(text, &end, 0); + if (end == text || *end != '\0' || parsed == 0 || parsed > 0xffffUL) { + return 0; + } + return static_cast(parsed); +} + void TraceDirectCcuProcessMemoryToken( uint64_t addr, uint64_t bytes, @@ -615,11 +630,13 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( if (peerResources.localXnCount == 0 || peerResources.remoteXnCount == 0 || peerResources.localWaitCkeCount == 0 || + peerResources.remoteNotifyCkeCount == 0 || peerResources.channelCount == 0 || peerLocalXnOffset >= peerResources.localXnCount || selectedRemoteXnOffset >= peerResources.remoteXnCount || peerLocalIndex >= peerResources.channelCount || - peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { + peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount || + peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount) { if (report != nullptr) { report->message = "peer direct CCU local XN/CKE resources are incomplete"; } @@ -928,6 +945,8 @@ int TileXRCcuCollectivePlanner::PrepareSignalWait( TileXRCcuDirectSignalWaitSpec signalWait; signalWait.role = ToDirectSignalWaitProgramRole(request.role); + signalWait.overrideBarrierMode = request.overrideBarrierMode; + signalWait.barrierMode = request.barrierMode; ret = TileXRCcuRunDirectSignalWaitInstallAttempt(options, signalWait, &plan->attempt, report); if (ret != TILEXR_SUCCESS) { *plan = TileXRCcuSignalWaitPlan {}; @@ -1288,11 +1307,15 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( TileXRCcuDirectAllToAll2RankSpec alltoall; alltoall.localRank = static_cast(rank); + alltoall.localSendAddr = localEndpoint.sourceAddr; + alltoall.localSendToken = localEndpoint.sourceToken; alltoall.localRecvAddr = localEndpoint.destinationAddr; alltoall.localRecvToken = localEndpoint.destinationToken; alltoall.remoteSendAddr = remoteImportRequest.addr; alltoall.remoteSendToken = TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); + alltoall.remoteRecvAddr = peerEndpoint.destinationAddr; + alltoall.remoteRecvToken = peerEndpoint.destinationToken; alltoall.bytes = bytes; alltoall.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; alltoall.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -1301,10 +1324,14 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( std::cerr << "TileXRDirectCcuTrace alltoallEndpoint" << " rank=" << rank << " peerRank=" << peerRank - << " localRecvAddr=0x" << std::hex << alltoall.localRecvAddr + << " localSendAddr=0x" << std::hex << alltoall.localSendAddr + << " localSendToken=0x" << alltoall.localSendToken + << " localRecvAddr=0x" << alltoall.localRecvAddr << " localRecvToken=0x" << alltoall.localRecvToken << " remoteSendAddr=0x" << alltoall.remoteSendAddr << " remoteSendToken=0x" << alltoall.remoteSendToken + << " remoteRecvAddr=0x" << alltoall.remoteRecvAddr + << " remoteRecvToken=0x" << alltoall.remoteRecvToken << " bytes=0x" << alltoall.bytes << std::dec << std::endl; } @@ -1341,7 +1368,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( } SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - 1U, + 0U, peerRank, importedRemoteBuffer.targetSegVa, remoteImportRequest.tokenId, @@ -1351,6 +1378,171 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( ClearDirectCcuRemoteRouteMemoryOverride(); return ret; } + +int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU SyncXn ping install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU SyncXn ping endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for SyncXn ping install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before SyncXn ping"; + } + return ret; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU SyncXn ping local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU SyncXn ping peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU SyncXn ping peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU SyncXn ping remote endpoint buffer"; + } + return ret; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 1; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-sync-xn-ping"; + } + + TileXRCcuDirectSyncXnPingSpec syncXnPing; + syncXnPing.localRank = static_cast(rank); + syncXnPing.peerRank = peerRank; + syncXnPing.payload = 0x54585253594e0000ULL | static_cast(rank & 0xffff); + syncXnPing.remoteNotifyMask = + SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK"); + syncXnPing.localWaitMask = + SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK"); + + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 0U, + peerRank, + importedRemoteBuffer.targetSegVa, + remoteImportRequest.tokenId, + remoteImportRequest.rawTokenId, + remoteImportRequest.tokenValue); + ret = TileXRCcuRunDirectSyncXnPingInstallAttempt(next, syncXnPing, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} #endif int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index 551e98d8..f09489ca 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -72,6 +72,15 @@ class TileXRCcuCollectivePlanner { uint32_t peerRank, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuSyncXnPingInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); #endif int RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session); bool HasDirectCcuLowerLayerPlan() const; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index db61a428..3f05b61f 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -38,10 +38,11 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 4U + 4U + 64U * 7U + 4U + 4U + 4U + 4U + 1U; + 3U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; void ResetReport(TileXRCcuDirectInstallReport* report) { @@ -923,16 +924,20 @@ int BuildDirectAllToAll2RankLaunchPackage( return TILEXR_ERROR_PARA_CHECK_FAIL; } - const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[0]; - const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[1]; + const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]; + const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]; const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]; TileXRCcuAllToAll2RankProgramSpec alltoallSpec; alltoallSpec.localRank = alltoall.localRank; + alltoallSpec.localSendAddr = alltoall.localSendAddr; + alltoallSpec.localSendToken = alltoall.localSendToken; alltoallSpec.localRecvAddr = alltoall.localRecvAddr; alltoallSpec.localRecvToken = alltoall.localRecvToken; alltoallSpec.remoteSendAddr = alltoall.remoteSendAddr; alltoallSpec.remoteSendToken = alltoall.remoteSendToken; + alltoallSpec.remoteRecvAddr = alltoall.remoteRecvAddr; + alltoallSpec.remoteRecvToken = alltoall.remoteRecvToken; alltoallSpec.bytes = alltoall.bytes; alltoallSpec.memorySliceBytes = alltoall.memorySliceBytes; alltoallSpec.memSlicePerBlock = alltoall.memSlicePerBlock; @@ -941,20 +946,31 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.localXn = attempt->plan.kernelLocalXn.startId; alltoallSpec.remoteXn = static_cast(attempt->plan.kernelLocalXn.startId + 1U); alltoallSpec.lengthXn = static_cast(attempt->plan.kernelLocalXn.startId + 2U); + alltoallSpec.preSyncLocalAddrXn = preResource.localXn; + alltoallSpec.preSyncLocalTokenXn = postResource.localXn; alltoallSpec.channelId = copyResource.channelId; alltoallSpec.preSyncChannelId = preResource.channelId; + alltoallSpec.preSyncTokenChannelId = postResource.channelId; alltoallSpec.copyChannelId = copyResource.channelId; - alltoallSpec.postSyncChannelId = postResource.channelId; + alltoallSpec.postSyncChannelId = preResource.channelId; + alltoallSpec.preSyncRemoteAddrXn = preResource.remoteXn; + alltoallSpec.preSyncRemoteTokenXn = postResource.remoteXn; alltoallSpec.preSyncRemoteNotifyCke = preResource.notifyCke; alltoallSpec.preSyncLocalWaitCke = preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; + alltoallSpec.preSyncRemoteTokenNotifyCke = postResource.notifyCke; + alltoallSpec.preSyncTokenLocalWaitCke = + postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; alltoallSpec.copyCompletionCke = copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke; - alltoallSpec.postSyncRemoteNotifyCke = postResource.notifyCke; + alltoallSpec.postSyncRemoteNotifyCke = preResource.notifyCke; alltoallSpec.postSyncLocalWaitCke = - postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; + preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; alltoallSpec.sourceCke = preResource.sourceCke; alltoallSpec.ckeMask = preResource.remoteNotifyMask == 0 ? 1U : preResource.remoteNotifyMask; + alltoallSpec.postSyncNotify = false; + alltoallSpec.postSyncWait = false; + alltoallSpec.emitFinish = false; if (DirectTraceEnabled()) { std::cerr << "TileXRDirectCcuTrace alltoallSpec" @@ -965,14 +981,26 @@ int BuildDirectAllToAll2RankLaunchPackage( << " remoteGsa=" << alltoallSpec.remoteGsa << " remoteXn=" << alltoallSpec.remoteXn << " lengthXn=" << alltoallSpec.lengthXn + << " preLocalAddrXn=" << alltoallSpec.preSyncLocalAddrXn + << " preLocalTokenXn=" << alltoallSpec.preSyncLocalTokenXn << " preChannelId=" << alltoallSpec.preSyncChannelId + << " preTokenChannelId=" << alltoallSpec.preSyncTokenChannelId << " copyChannelId=" << alltoallSpec.copyChannelId << " postChannelId=" << alltoallSpec.postSyncChannelId << " preNotifyCke=" << alltoallSpec.preSyncRemoteNotifyCke + << " preTokenNotifyCke=" << alltoallSpec.preSyncRemoteTokenNotifyCke + << " preTokenWaitCke=" << alltoallSpec.preSyncTokenLocalWaitCke + << " preRemoteAddrXn=" << alltoallSpec.preSyncRemoteAddrXn + << " preRemoteTokenXn=" << alltoallSpec.preSyncRemoteTokenXn << " copyCompletionCke=" << alltoallSpec.copyCompletionCke << " postNotifyCke=" << alltoallSpec.postSyncRemoteNotifyCke - << " localRecvAddr=0x" << std::hex << alltoallSpec.localRecvAddr + << " postSyncNotify=" << (alltoallSpec.postSyncNotify ? 1 : 0) + << " postSyncWait=" << (alltoallSpec.postSyncWait ? 1 : 0) + << " emitFinish=" << (alltoallSpec.emitFinish ? 1 : 0) + << " localSendAddr=0x" << std::hex << alltoallSpec.localSendAddr + << " localRecvAddr=0x" << alltoallSpec.localRecvAddr << " remoteSendAddr=0x" << alltoallSpec.remoteSendAddr + << " remoteRecvAddr=0x" << alltoallSpec.remoteRecvAddr << " bytes=0x" << alltoallSpec.bytes << std::dec << std::endl; } @@ -1041,6 +1069,39 @@ TileXRCcuBarrierMode SignalWaitBarrierMode(TileXRCcuSignalWaitProgramRole role) TileXRCcuBarrierMode::SyncCke; } +TileXRCcuBarrierMode EffectiveSignalWaitBarrierMode(const TileXRCcuDirectSignalWaitSpec& signalWait) +{ + return signalWait.overrideBarrierMode ? signalWait.barrierMode : SignalWaitBarrierMode(signalWait.role); +} + +uint32_t BarrierInstructionCount(TileXRCcuBarrierMode mode) +{ + switch (mode) { + case TileXRCcuBarrierMode::SyncCke: + case TileXRCcuBarrierMode::SyncCkeSetWait: + return 3U; + case TileXRCcuBarrierMode::SyncXnLoadPostOnly: + return 2U; + case TileXRCcuBarrierMode::SyncXn: + case TileXRCcuBarrierMode::LocalCke: + return 2U; + case TileXRCcuBarrierMode::SyncXnPostOnly: + case TileXRCcuBarrierMode::SyncCkePostOnly: + case TileXRCcuBarrierMode::LocalCkePostOnly: + return 1U; + default: + return 2U; + } +} + +uint32_t SignalWaitInstructionCount(const TileXRCcuDirectSignalWaitSpec& signalWait) +{ + if (signalWait.overrideBarrierMode) { + return BarrierInstructionCount(signalWait.barrierMode); + } + return SignalWaitInstructionCount(signalWait.role); +} + int BuildDirectSignalWaitLaunchPackage( const TileXRCcuDirectSignalWaitSpec& signalWait, TileXRCcuDirectInstallAttempt* attempt, @@ -1069,7 +1130,31 @@ int BuildDirectSignalWaitLaunchPackage( TileXRCcuProgram program; TileXRCcuBarrierProgramReport signalWaitReport; - if (TileXRCcuBuildSignalWaitProgram(spec, &program.sync, &signalWaitReport) != TILEXR_SUCCESS) { + if (signalWait.overrideBarrierMode) { + TileXRCcuBarrierSyncSpec barrier; + barrier.channelId = spec.channelId; + barrier.remoteXn = spec.remoteXn; + barrier.localXn = spec.localXn; + barrier.remoteNotifyCke = spec.remoteNotifyCke; + barrier.remoteNotifyMask = spec.remoteNotifyMask; + barrier.localWaitCke = spec.localWaitCke; + barrier.localWaitMask = spec.localWaitMask; + barrier.sourceCke = spec.sourceCke; + barrier.sourceCkeMask = spec.sourceCkeMask; + barrier.clearLocalWait = spec.clearLocalWait; + std::vector barriers; + barriers.push_back(barrier); + if (TileXRCcuBuildBarrierProgram( + barriers, + &program.sync, + &signalWaitReport, + signalWait.barrierMode) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = signalWaitReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } else if (TileXRCcuBuildSignalWaitProgram(spec, &program.sync, &signalWaitReport) != TILEXR_SUCCESS) { if (report != nullptr) { report->message = signalWaitReport.message; } @@ -1110,6 +1195,132 @@ int BuildDirectSignalWaitLaunchPackage( return TILEXR_SUCCESS; } +int BuildDirectSyncXnPingLaunchPackage( + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->plan.syncResources.size() != 1 || attempt->plan.taskWindows.size() != 1) { + if (report != nullptr) { + report->message = "missing direct CCU SyncXn ping producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (syncXnPing.localRank > 1U || syncXnPing.peerRank > 1U || syncXnPing.localRank == syncXnPing.peerRank) { + if (report != nullptr) { + report->message = "direct CCU SyncXn ping requires two distinct rank ids"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; + const uint16_t localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; + const uint16_t defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank); + const uint16_t defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank); + const uint16_t remoteNotifyMask = + syncXnPing.remoteNotifyMask == 0 ? defaultRemoteNotifyMask : syncXnPing.remoteNotifyMask; + const uint16_t localWaitMask = + syncXnPing.localWaitMask == 0 ? defaultLocalWaitMask : syncXnPing.localWaitMask; + if (resource.localXn == 0 || resource.remoteXn == 0 || resource.channelId == 0 || + resource.notifyCke == 0 || localWaitCke == 0 || resource.sourceCke == 0) { + if (report != nullptr) { + report->message = "missing direct CCU SyncXn ping XN/CKE/channel resource"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuProgram program; + program.sync.reserve(TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(resource.localXn, syncXnPing.payload, 0, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping payload load"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuSyncXnSpec notify; + notify.remoteXn = resource.remoteXn; + notify.localXn = resource.localXn; + notify.channelId = resource.channelId; + notify.notifyCke = resource.notifyCke; + notify.notifyMask = remoteNotifyMask; + notify.clearWait = true; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping notify"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuCkeSpec source; + source.ckeId = resource.sourceCke; + source.mask = remoteNotifyMask; + source.clearWait = true; + if (TileXRCcuEncodeSetCke(source, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping source CKE set"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuSyncCkeSpec syncCke; + syncCke.remoteCke = resource.notifyCke; + syncCke.localCke = resource.sourceCke; + syncCke.localCkeMask = remoteNotifyMask; + syncCke.channelId = resource.channelId; + syncCke.clearWait = true; + if (TileXRCcuEncodeSyncCke(syncCke, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping SyncCke notify"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuCkeSpec wait; + wait.waitCkeId = localWaitCke; + wait.waitMask = localWaitMask; + wait.clearWait = true; + if (TileXRCcuEncodeSetCke(wait, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping wait"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + void FillReportFromAttempt(const TileXRCcuDirectInstallAttempt& attempt, TileXRCcuDirectInstallReport* report) { if (report == nullptr) { @@ -1287,6 +1498,7 @@ int RunDirectInstallAttemptImpl( const TileXRCcuDirectMemoryCopySpec* memoryCopy, const TileXRCcuDirectAllToAll2RankSpec* alltoall, const TileXRCcuDirectSignalWaitSpec* signalWait, + const TileXRCcuDirectSyncXnPingSpec* syncXnPing, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { @@ -1331,7 +1543,8 @@ int RunDirectInstallAttemptImpl( ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); - const bool customProgram = memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr; + const bool customProgram = + memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr || syncXnPing != nullptr; attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : @@ -1342,12 +1555,15 @@ int RunDirectInstallAttemptImpl( alltoall != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : signalWait != nullptr ? - std::max(options.syncInstructionCount, SignalWaitInstructionCount(signalWait->role)) : + std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : + syncXnPing != nullptr ? + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT) : options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.barrierMode = + attempt->resourceRequest.barrierMode = alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : - signalWait == nullptr ? options.barrierMode : SignalWaitBarrierMode(signalWait->role); + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); TileXRCcuResourceAllocator allocator; if (allocator.Init(attempt->resourceSpec) != TILEXR_SUCCESS) { @@ -1387,7 +1603,8 @@ int RunDirectInstallAttemptImpl( } attempt->plan.barrierMode = alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : - signalWait == nullptr ? attempt->plan.barrierMode : SignalWaitBarrierMode(signalWait->role); + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); if (ret != TILEXR_SUCCESS) { @@ -1422,6 +1639,8 @@ int RunDirectInstallAttemptImpl( BuildDirectAllToAll2RankLaunchPackage(*alltoall, attempt, report) : signalWait != nullptr ? BuildDirectSignalWaitLaunchPackage(*signalWait, attempt, report) : + syncXnPing != nullptr ? + BuildDirectSyncXnPingLaunchPackage(*syncXnPing, attempt, report) : TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport); if (ret != TILEXR_SUCCESS) { return Fail( @@ -1433,6 +1652,8 @@ int RunDirectInstallAttemptImpl( "failed to build direct CCU memory copy launch package" : alltoall != nullptr ? "failed to build direct CCU alltoall launch package" : + syncXnPing != nullptr ? + "failed to build direct CCU SyncXn ping launch package" : "failed to build direct CCU signal/wait launch package") : report->message)); } @@ -1496,7 +1717,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1515,7 +1736,7 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectAllToAll2RankInstallAttempt( @@ -1525,8 +1746,10 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( TileXRCcuDirectInstallReport* report) { if (alltoall.localRank > 1U || + alltoall.localSendAddr == 0 || alltoall.localSendToken == 0 || alltoall.localRecvAddr == 0 || alltoall.localRecvToken == 0 || alltoall.remoteSendAddr == 0 || alltoall.remoteSendToken == 0 || + alltoall.remoteRecvAddr == 0 || alltoall.remoteRecvToken == 0 || alltoall.bytes == 0 || alltoall.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES || alltoall.memSlicePerBlock == 0 || alltoall.memSlicePerBlock > TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK) { @@ -1537,7 +1760,7 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectSignalWaitInstallAttempt( @@ -1546,7 +1769,16 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, nullptr, attempt, report); +} + +int TileXRCcuRunDirectSyncXnPingInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index eb63680f..dd5a2ee5 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -72,10 +72,14 @@ struct TileXRCcuDirectMemoryCopySpec { struct TileXRCcuDirectAllToAll2RankSpec { uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; uint64_t localRecvAddr = 0; uint64_t localRecvToken = 0; uint64_t remoteSendAddr = 0; uint64_t remoteSendToken = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; uint64_t bytes = 0; uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -83,6 +87,16 @@ struct TileXRCcuDirectAllToAll2RankSpec { struct TileXRCcuDirectSignalWaitSpec { TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; + bool overrideBarrierMode = false; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncCke; +}; + +struct TileXRCcuDirectSyncXnPingSpec { + uint32_t localRank = 0; + uint32_t peerRank = 1; + uint64_t payload = 0; + uint16_t remoteNotifyMask = 0; + uint16_t localWaitMask = 0; }; struct TileXRCcuDirectInstallAttempt { @@ -153,6 +167,12 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectSyncXnPingInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt); int TileXRCcuSubmitPreparedTasks( diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 86b65118..6fce2c50 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -78,8 +78,10 @@ constexpr const char* kP2pCcuCopyActiveRankEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CC constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION"; constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; +constexpr const char* kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"; constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; +constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; @@ -162,6 +164,16 @@ bool AllToAllSmokeEnabled() return EnvFlag(kAllToAllEnv); } +bool AllToAllLongMissionEnabled() +{ + return EnvFlag(kAllToAllLongMissionEnv); +} + +bool SyncXnPingSmokeEnabled() +{ + return EnvFlag(kSyncXnPingEnv); +} + bool BarrierSmokeEnabled() { return EnvFlag(kSignalWaitBarrierEnv); @@ -229,7 +241,8 @@ bool IsP2pCcuCopyActiveRank(int rank) TileXR::TileXRCcuMemoryCopyDirection P2pCcuCopyDirectionFromEnv() { const char* value = std::getenv(kP2pCcuCopyDirectionEnv); - if (value != nullptr && std::string(value) == "local_to_remote") { + const std::string direction = value == nullptr ? "" : std::string(value); + if (direction == "local_to_remote" || direction == "LocalToRemote" || direction == "1") { return TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote; } return TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal; @@ -839,6 +852,8 @@ TileXR::TileXRCcuSignalWaitRequest MakeSignalWaitRequest( TileXR::TileXRCcuSignalWaitRequest request {}; request.peerRank = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; request.role = SignalWaitRoleForRank(rank); + request.overrideBarrierMode = BarrierSmokeEnabled(); + request.barrierMode = options.barrierMode; request.syncInstructionCount = std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr ? 0U : options.syncInstructionCount; @@ -1255,18 +1270,27 @@ bool CollectiveSubmitReadyGateConfigured() return readyDir != nullptr && readyDir[0] != '\0'; } -std::string SubmitReadinessPath(int rank) +std::string RankPhaseFileStem(int rank, int phase) +{ + std::string stem = "/rank" + std::to_string(rank); + if (phase >= 0) { + stem += ".phase" + std::to_string(phase); + } + return stem; +} + +std::string SubmitReadinessPath(int rank, int phase = -1) { const char* readyDir = std::getenv(kReadyDirEnv); if (readyDir == nullptr || readyDir[0] == '\0') { return {}; } - return std::string(readyDir) + "/rank" + std::to_string(rank) + ".ready"; + return std::string(readyDir) + RankPhaseFileStem(rank, phase) + ".ready"; } -bool WriteSubmitReadiness(int rank, bool ready) +bool WriteSubmitReadiness(int rank, bool ready, int phase = -1) { - const std::string path = SubmitReadinessPath(rank); + const std::string path = SubmitReadinessPath(rank, phase); if (path.empty()) { return true; } @@ -1278,12 +1302,12 @@ bool WriteSubmitReadiness(int rank, bool ready) return static_cast(out); } -bool ReadSubmitReadiness(int rank, bool* ready) +bool ReadSubmitReadiness(int rank, bool* ready, int phase = -1) { if (ready == nullptr) { return false; } - const std::string path = SubmitReadinessPath(rank); + const std::string path = SubmitReadinessPath(rank, phase); if (path.empty()) { return false; } @@ -1300,12 +1324,12 @@ bool ReadSubmitReadiness(int rank, bool* ready) return true; } -bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) +bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady, int phase = -1) { if (!CollectiveSubmitReadyGateConfigured()) { return localReady; } - const bool wrote = WriteSubmitReadiness(rank, localReady); + const bool wrote = WriteSubmitReadiness(rank, localReady, phase); const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); const auto start = std::chrono::steady_clock::now(); for (;;) { @@ -1313,7 +1337,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) bool allReady = wrote && localReady; for (int peer = 0; peer < rankSize; ++peer) { bool peerReady = false; - if (!ReadSubmitReadiness(peer, &peerReady)) { + if (!ReadSubmitReadiness(peer, &peerReady, phase)) { allSeen = false; allReady = false; break; @@ -1323,6 +1347,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) if (allSeen) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" << " rank=" << rank + << " phase=" << phase << " localReady=" << (localReady ? 1 : 0) << " allRanksReady=" << (allReady ? 1 : 0) << std::endl; @@ -1333,6 +1358,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) if (elapsedMs >= timeoutMs) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" << " rank=" << rank + << " phase=" << phase << " localReady=" << (localReady ? 1 : 0) << " allRanksReady=0" << " timeoutMs=" << timeoutMs @@ -1349,18 +1375,18 @@ bool CollectiveSubmitDoneGateConfigured() return doneDir != nullptr && doneDir[0] != '\0'; } -std::string SubmitDonePath(int rank) +std::string SubmitDonePath(int rank, int phase = -1) { const char* doneDir = std::getenv(kDoneDirEnv); if (doneDir == nullptr || doneDir[0] == '\0') { return {}; } - return std::string(doneDir) + "/rank" + std::to_string(rank) + ".done"; + return std::string(doneDir) + RankPhaseFileStem(rank, phase) + ".done"; } -bool WriteSubmitDone(int rank, int result) +bool WriteSubmitDone(int rank, int result, int phase = -1) { - const std::string path = SubmitDonePath(rank); + const std::string path = SubmitDonePath(rank, phase); if (path.empty()) { return true; } @@ -1372,12 +1398,12 @@ bool WriteSubmitDone(int rank, int result) return static_cast(out); } -bool ReadSubmitDone(int rank, int* result) +bool ReadSubmitDone(int rank, int* result, int phase = -1) { if (result == nullptr) { return false; } - const std::string path = SubmitDonePath(rank); + const std::string path = SubmitDonePath(rank, phase); if (path.empty()) { return false; } @@ -1389,12 +1415,12 @@ bool ReadSubmitDone(int rank, int* result) return static_cast(in); } -bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) +bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult, int phase = -1) { if (!CollectiveSubmitDoneGateConfigured()) { return true; } - const bool wrote = WriteSubmitDone(rank, localResult); + const bool wrote = WriteSubmitDone(rank, localResult, phase); const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); const auto start = std::chrono::steady_clock::now(); for (;;) { @@ -1402,7 +1428,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) bool allSucceeded = wrote && localResult == 0; for (int peer = 0; peer < rankSize; ++peer) { int peerResult = 0; - if (!ReadSubmitDone(peer, &peerResult)) { + if (!ReadSubmitDone(peer, &peerResult, phase)) { allSeen = false; allSucceeded = false; break; @@ -1412,6 +1438,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) if (allSeen) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" << " rank=" << rank + << " phase=" << phase << " localResult=" << localResult << " allRanksDone=1" << " allRanksSucceeded=" << (allSucceeded ? 1 : 0) @@ -1423,6 +1450,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) if (elapsedMs >= timeoutMs) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" << " rank=" << rank + << " phase=" << phase << " localResult=" << localResult << " allRanksDone=0" << " timeoutMs=" << timeoutMs @@ -1508,9 +1536,9 @@ int RunAllToAllCopyPhase( if (prepareRet != TileXR::TILEXR_SUCCESS) { finalRet = 6; } else if (submitRequested) { - const bool phaseReady = WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady); + const bool phaseReady = WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady, phase); finalRet = phaseReady ? 0 : 13; - if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase) && finalRet == 0) { finalRet = 13; } } @@ -1522,7 +1550,8 @@ int RunAllToAllCopyPhase( WaitForCollectiveSubmitReadiness( rank, rankSize, - prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady, + phase) : false; if (prepareRet != TileXR::TILEXR_SUCCESS) { finalRet = 6; @@ -1571,12 +1600,166 @@ int RunAllToAllCopyPhase( } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase) && finalRet == 0) { + finalRet = 13; + } + aclrtDestroyStream(stream); + } + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + +void PrintAllToAllResultAndMaybeFastExit(int rank, int finalRet, const AllToAllState& alltoall) +{ + if (finalRet == 0) { + std::cout << "tilexr_ccu_alltoall result passed=1" + << " rank=" << rank + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << std::endl; + } else { + std::cout << "tilexr_ccu_alltoall result passed=0" + << " rank=" << rank + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << std::endl; + } + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_alltoall fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } +} + +int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + AllToAllState alltoall; + alltoall.initRet = InitAllToAllState(rank, peer, &alltoall); + + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 3; + options.sqeArgCount = 0; + options.syncInstructionCount = 3 + 64 * 7 + 3 + 1; + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << alltoall.bytes + << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() + << " blockCount=64" + << " longMission=1" + << " preSync=1" + << " postSync=1" + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall.initRet != ACL_SUCCESS ? + alltoall.initRet : + context->planner.PrepareDirectCcuAllToAll2RankInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall.source.ptr), + reinterpret_cast(alltoall.destination.ptr), + alltoall.bytes, + static_cast(peer), + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " longMission=1" + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + if (!attempt.submitTasks.empty()) { + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + } + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { finalRet = 13; } aclrtDestroyStream(stream); } } + + if (finalRet == 0) { + const int checkRet = CheckAllToAllState(&alltoall); + if (checkRet != ACL_SUCCESS) { + finalRet = 14; + } + } + PrintAllToAllResultAndMaybeFastExit(rank, finalRet, alltoall); const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; } @@ -1586,6 +1769,9 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } + if (AllToAllLongMissionEnabled()) { + return RunAllToAllLongMissionSmokeForRank(context, rank, rankSize, device); + } if (rankSize != 2) { std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; @@ -1625,23 +1811,127 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi } } + PrintAllToAllResultAndMaybeFastExit(rank, finalRet, alltoall); + return finalRet; +} + +int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_sync_xn_ping skipped rankSize=" << rankSize + << " reason=\"direct CCU SyncXn ping requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + AllToAllState routeState; + routeState.initRet = InitAllToAllState(rank, peer, &routeState); + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 3; + + std::cout << "tilexr_ccu_sync_xn_ping config" + << " rank=" << rank + << " peer=" << peer + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = routeState.initRet != ACL_SUCCESS ? + routeState.initRet : + context->planner.PrepareDirectCcuSyncXnPingInstallAttempt( + context->session, + options, + reinterpret_cast(routeState.source.ptr), + reinterpret_cast(routeState.destination.ptr), + routeState.bytes, + static_cast(peer), + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_sync_xn_ping prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_sync_xn_ping submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_sync_xn_ping submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_sync_xn_ping aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_sync_xn_ping submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before SyncXn ping aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after SyncXn ping aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_sync_xn_ping timing" + << " rank=" << rank + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_sync_xn_ping aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + if (!attempt.submitTasks.empty()) { + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_sync_xn_ping"); + } + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + aclrtDestroyStream(stream); + } + } + if (finalRet == 0) { - std::cout << "tilexr_ccu_alltoall result passed=1" + std::cout << "tilexr_ccu_sync_xn_ping result passed=1" << " rank=" << rank << " ret=" << finalRet - << " readRet=" << alltoall.readRet - << " mismatches=" << alltoall.mismatchCount << std::endl; } else { - std::cout << "tilexr_ccu_alltoall result passed=0" + std::cout << "tilexr_ccu_sync_xn_ping result passed=0" << " rank=" << rank << " ret=" << finalRet - << " readRet=" << alltoall.readRet - << " mismatches=" << alltoall.mismatchCount << std::endl; } if (ShouldFastExitAfterRun()) { - std::cout << "tilexr_ccu_alltoall fastExitAfterRun=1" + std::cout << "tilexr_ccu_sync_xn_ping fastExitAfterRun=1" << " ret=" << finalRet << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" << std::endl; @@ -1649,7 +1939,8 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi std::fflush(stderr); std::_Exit(finalRet); } - return finalRet; + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; } int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) @@ -1721,9 +2012,10 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank const auto submitEnd = std::chrono::steady_clock::now(); PrintSubmitReport("tilexr_ccu_signal_wait submit", submitRet, submitReport); const auto syncBegin = std::chrono::steady_clock::now(); - TraceLifecycle("before signal/wait aclrtSynchronizeStream"); - const int syncRet = aclrtSynchronizeStream(stream); - TraceLifecycle("after signal/wait aclrtSynchronizeStream"); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before signal/wait aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after signal/wait aclrtSynchronizeStreamWithTimeout"); const auto syncEnd = std::chrono::steady_clock::now(); std::cout << "tilexr_ccu_signal_wait timing" << " rank=" << rank @@ -1731,13 +2023,15 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank << " preSubmitDelayMs=" << effectiveDelayMs << " submitRet=" << submitRet << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs << " submitMs=" << std::chrono::duration_cast(submitEnd - submitBegin).count() << " syncMs=" << std::chrono::duration_cast(syncEnd - syncBegin).count() << std::endl; if (syncRet != ACL_SUCCESS) { - std::cerr << "tilexr_ccu_signal_wait aclrtSynchronizeStream ret=" << syncRet << std::endl; + std::cerr << "tilexr_ccu_signal_wait aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; if (!plan.submitTasks.empty()) { PrintMissionContext(context, plan.submitTasks.front(), "tilexr_ccu_signal_wait"); } @@ -1785,6 +2079,9 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { + if (SyncXnPingSmokeEnabled()) { + return RunSyncXnPingSmokeForRank(context, rank, rankSize, device); + } if (AllToAllSmokeEnabled()) { return RunAllToAllSmokeForRank(context, rank, rankSize, device); } diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 8e9380e8..7e68e66a 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -92,6 +92,16 @@ alltoall_mode_enabled() [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0}" = "1" ] } +alltoall_long_mission_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ] +} + +sync_xn_ping_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0}" = "1" ] +} + apply_p2p_ccu_copy_defaults() { if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" != "1" ]; then @@ -159,6 +169,41 @@ apply_signal_wait_defaults() export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" } +apply_sync_xn_ping_defaults() +{ + if ! sync_xn_ping_mode_enabled; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" + export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}" +} + apply_alltoall_defaults() { if ! alltoall_mode_enabled; then @@ -172,8 +217,13 @@ apply_alltoall_defaults() export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" - export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" + if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476}" + else + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" + fi export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" @@ -198,6 +248,7 @@ apply_alltoall_defaults() apply_p2p_ccu_copy_defaults apply_signal_wait_defaults +apply_sync_xn_ping_defaults apply_alltoall_defaults if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then @@ -213,7 +264,9 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT \ TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK \ TILEXR_CCU_DIRECT_SMOKE_BARRIER \ + TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING \ TILEXR_CCU_DIRECT_SMOKE_ALLTOALL \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION \ TILEXR_CCU_ALLTOALL_BYTES \ TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ @@ -474,9 +527,21 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION}") +fi if [ "${TILEXR_CCU_ALLTOALL_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_ALLTOALL_BYTES=${TILEXR_CCU_ALLTOALL_BYTES}") fi @@ -614,7 +679,7 @@ for token_field in "${resource_window_token_fields[@]}"; do fi done -echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" +echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} syncXnPing=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} alltoallLongMission=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then thread_log="${work_dir}/ccu_thread.log" diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index 2049c528..54409701 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -81,13 +81,27 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): using namespace TileXR; + constexpr uint16_t kSetCkeHeader = 0x0802U; + constexpr uint16_t kClearCkeHeader = 0x0804U; + + uint16_t Slot(const TileXRCcuInstr& instr, uint32_t slot) + { + const uint32_t word = slot / 4U; + const uint32_t shift = (slot % 4U) * 16U; + return static_cast((instr.words[word] >> shift) & 0xffffU); + } + int main() { TileXRCcuAllToAll2RankProgramSpec spec; - spec.localRecvAddr = 0x10000000ULL; - spec.localRecvToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12346, 0x22223333U, true); spec.remoteSendAddr = 0x20000000ULL; spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(0x23457, 0x44445555U, true); spec.bytes = 2ULL * 1024ULL * 1024ULL; spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -98,11 +112,15 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.lengthXn = 0x203; spec.channelId = 0x12; spec.copyCompletionCke = 0x301; + spec.preSyncRemoteAddrXn = 0x211; + spec.preSyncRemoteTokenXn = 0x212; spec.preSyncLocalWaitCke = 0x302; spec.preSyncRemoteNotifyCke = 0x303; - spec.postSyncLocalWaitCke = 0x304; - spec.postSyncRemoteNotifyCke = 0x305; - spec.sourceCke = 0x306; + spec.preSyncTokenLocalWaitCke = 0x304; + spec.preSyncRemoteTokenNotifyCke = 0x305; + spec.postSyncLocalWaitCke = 0x306; + spec.postSyncRemoteNotifyCke = 0x307; + spec.sourceCke = 0x308; spec.ckeMask = 1; std::vector program; @@ -113,11 +131,11 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): return 1; } const uint32_t expectedBlocks = 64; - const uint32_t expectedInstructions = 4 + 4 + expectedBlocks * 7 + 4 + 4 + 4 + 4 + 1; + const uint32_t expectedInstructions = 3 + expectedBlocks * 7 + 3 + 1; if (report.blockCount != expectedBlocks || report.copyInstructionCount != expectedBlocks * 7 || - report.preSyncInstructionCount != 4 || - report.postSyncInstructionCount != 4 || + report.preSyncInstructionCount != 3 || + report.postSyncInstructionCount != 3 || report.finishInstructionCount != 1 || report.totalInstructionCount != expectedInstructions || program.size() != expectedInstructions) { @@ -135,6 +153,35 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): std::cerr << "unexpected block size or message\n"; return 3; } + const uint16_t preSyncMask = static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); + const uint16_t postSyncMask = static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); + const uint32_t postSyncSetIndex = expectedInstructions - 4; + const uint32_t postSyncWaitIndex = expectedInstructions - 2; + if (program.size() < 3 || + Slot(program[2], 0) != kSetCkeHeader || + Slot(program[2], 4) != spec.preSyncLocalWaitCke || + Slot(program[2], 5) != preSyncMask) { + std::cerr << "PreSync should use the HCCL-style output barrier bit" + << " header=0x" << std::hex << Slot(program[2], 0) + << " waitCke=0x" << Slot(program[2], 4) + << " waitMask=0x" << Slot(program[2], 5) + << std::dec << "\n"; + return 4; + } + if (Slot(program[postSyncSetIndex], 0) != kSetCkeHeader || + Slot(program[postSyncSetIndex], 3) != postSyncMask || + Slot(program[postSyncWaitIndex], 0) != kClearCkeHeader || + Slot(program[postSyncWaitIndex], 4) != spec.postSyncLocalWaitCke || + Slot(program[postSyncWaitIndex], 5) != postSyncMask || + postSyncMask == preSyncMask) { + std::cerr << "PostSync should use a distinct HCCL-style post barrier bit" + << " postSetMask=0x" << std::hex << Slot(program[postSyncSetIndex], 3) + << " postWaitCke=0x" << Slot(program[postSyncWaitIndex], 4) + << " postWaitMask=0x" << Slot(program[postSyncWaitIndex], 5) + << " preMask=0x" << preSyncMask + << std::dec << "\n"; + return 5; + } return 0; } ''' @@ -145,7 +192,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) - def test_two_rank_program_serializes_copy_phases_by_local_rank(self): + def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): code = textwrap.dedent( r''' #include "ccu/tilexr_ccu_alltoall_program.h" @@ -166,10 +213,14 @@ def test_two_rank_program_serializes_copy_phases_by_local_rank(self): { TileXRCcuAllToAll2RankProgramSpec spec; spec.localRank = localRank; - spec.localRecvAddr = 0x10000000ULL; - spec.localRecvToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12346, 0x22223333U, true); spec.remoteSendAddr = 0x20000000ULL; spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(0x23457, 0x44445555U, true); spec.bytes = 2ULL * 1024ULL * 1024ULL; spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -180,11 +231,15 @@ def test_two_rank_program_serializes_copy_phases_by_local_rank(self): spec.lengthXn = 0x203; spec.channelId = 0x12; spec.copyCompletionCke = 0x301; + spec.preSyncRemoteAddrXn = 0x211; + spec.preSyncRemoteTokenXn = 0x212; spec.preSyncLocalWaitCke = 0x302; spec.preSyncRemoteNotifyCke = 0x303; - spec.postSyncLocalWaitCke = 0x304; - spec.postSyncRemoteNotifyCke = 0x305; - spec.sourceCke = 0x306; + spec.preSyncTokenLocalWaitCke = 0x304; + spec.preSyncRemoteTokenNotifyCke = 0x305; + spec.postSyncLocalWaitCke = 0x306; + spec.postSyncRemoteNotifyCke = 0x307; + spec.sourceCke = 0x308; spec.ckeMask = 1; return spec; } @@ -223,9 +278,12 @@ def test_two_rank_program_serializes_copy_phases_by_local_rank(self): << " rank1=" << report1.message << "\n"; return 1; } - if (report0.totalInstructionCount != 473 || report1.totalInstructionCount != 473 || - rank0.size() != 473 || rank1.size() != 473) { - std::cerr << "unexpected two-phase instruction count" + const uint32_t expectedInstructions = 3 + 64 * 7 + 3 + 1; + if (report0.totalInstructionCount != expectedInstructions || + report1.totalInstructionCount != expectedInstructions || + rank0.size() != expectedInstructions || + rank1.size() != expectedInstructions) { + std::cerr << "unexpected instruction count" << " rank0=" << rank0.size() << " rank1=" << rank1.size() << " report0=" << report0.totalInstructionCount @@ -233,13 +291,13 @@ def test_two_rank_program_serializes_copy_phases_by_local_rank(self): return 2; } if (CopyInstructionCount(rank0) != 64 || CopyInstructionCount(rank1) != 64) { - std::cerr << "each rank should issue exactly 64 remote-to-local transfers\n"; + std::cerr << "each rank should issue exactly 64 remote-to-local reads\n"; return 3; } const size_t rank0FirstCopy = FirstCopyIndex(rank0); const size_t rank1FirstCopy = FirstCopyIndex(rank1); - if (rank0FirstCopy != 13 || rank1FirstCopy != 21) { - std::cerr << "copy phases overlap or moved unexpectedly" + if (rank0FirstCopy != 8 || rank1FirstCopy != 8) { + std::cerr << "copy region should start at the same instruction after the single PreSync" << " rank0FirstCopy=" << rank0FirstCopy << " rank1FirstCopy=" << rank1FirstCopy << "\n"; return 4; @@ -267,10 +325,14 @@ def test_builder_rejects_invalid_slice_configuration(self): TileXRCcuAllToAll2RankProgramSpec ValidSpec() { TileXRCcuAllToAll2RankProgramSpec spec; - spec.localRecvAddr = 0x10000000ULL; - spec.localRecvToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); spec.remoteSendAddr = 0x20000000ULL; spec.remoteSendToken = TileXRCcuPackMemoryToken(3, 4, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(4, 5, true); spec.bytes = 2ULL * 1024ULL * 1024ULL; spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; @@ -281,11 +343,15 @@ def test_builder_rejects_invalid_slice_configuration(self): spec.lengthXn = 5; spec.channelId = 6; spec.copyCompletionCke = 7; + spec.preSyncRemoteAddrXn = 13; + spec.preSyncRemoteTokenXn = 14; spec.preSyncLocalWaitCke = 8; spec.preSyncRemoteNotifyCke = 9; - spec.postSyncLocalWaitCke = 10; - spec.postSyncRemoteNotifyCke = 11; - spec.sourceCke = 12; + spec.preSyncTokenLocalWaitCke = 10; + spec.preSyncRemoteTokenNotifyCke = 11; + spec.postSyncLocalWaitCke = 12; + spec.postSyncRemoteNotifyCke = 15; + spec.sourceCke = 16; spec.ckeMask = 1; return spec; } @@ -350,20 +416,29 @@ def test_builder_is_wired_and_has_no_hccl_dependency_surface(self): self.assertIn("TileXRCcuBuildAllToAll2RankProgram", header) self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + self.assertIn("preSyncRemoteTokenNotifyCke", header) + self.assertIn("preSyncTokenLocalWaitCke", header) + self.assertNotIn("tokenLocalWaitCke", source) + self.assertIn("PreSyncSignalMask", source) + self.assertIn("PostSyncSignalMask", source) + self.assertIn("TILEXR_CCU_ALLTOALL_SIGNAL_MASK", header) + self.assertIn("TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK", header) + self.assertIn("TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK", header) combined = header + "\n" + source for needle in PRIVATE_CCU_PRODUCER_NEEDLES: with self.subTest(needle=needle): self.assertNotIn(needle, combined) - def test_two_phase_rank_role_is_part_of_program_contract(self): + def test_local_rank_does_not_split_the_long_mission_into_copy_phases(self): header = ALLTOALL_HEADER.read_text(encoding="utf-8") source = ALLTOALL_SOURCE.read_text(encoding="utf-8") orchestrator = (COMM_DIR / "ccu" / "tilexr_ccu_direct_orchestrator.cpp").read_text(encoding="utf-8") planner = (COMM_DIR / "ccu" / "tilexr_ccu_collective_planner.cpp").read_text(encoding="utf-8") self.assertIn("uint32_t localRank = 0", header) - self.assertIn("append copy only for the local rank's active phase", source) + self.assertNotIn("append copy only for the local rank's active phase", source) + self.assertNotIn("for (uint32_t phase = 0; phase < 2U; ++phase)", source) self.assertIn("alltoallSpec.localRank = alltoall.localRank", orchestrator) self.assertIn("alltoall.localRank = static_cast(rank)", planner) diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index b9d83657..ef1db07b 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1688,6 +1688,7 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): cmake = COMM_CMAKE.read_text(encoding="utf-8") header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("ccu/tilexr_ccu_direct_orchestrator.h", cmake) self.assertIn("ccu/tilexr_ccu_direct_orchestrator.cpp", cmake) @@ -1746,18 +1747,29 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke", source) + self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) + self.assertIn("const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]", source) + self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(\n 0U,", planner) + self.assertIn("preSyncRemoteAddrXn = preResource.remoteXn", source) + self.assertIn("preSyncRemoteTokenXn = postResource.remoteXn", source) + self.assertIn("preSyncTokenChannelId = postResource.channelId", source) + self.assertNotIn("preSyncTokenChannelId = preResource.channelId", source) self.assertIn("preSyncRemoteNotifyCke = preResource.notifyCke", source) + self.assertIn("preSyncRemoteTokenNotifyCke = postResource.notifyCke", source) + self.assertIn("preSyncTokenLocalWaitCke =", source) + self.assertIn("preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke", source) self.assertIn("alltoallSpec.copyCompletionCke =", source) self.assertIn("copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke", source) - self.assertIn("postSyncRemoteNotifyCke = postResource.notifyCke", source) + self.assertIn("postSyncRemoteNotifyCke = preResource.notifyCke", source) self.assertIn("preSyncChannelId = preResource.channelId", source) self.assertIn("copyChannelId = copyResource.channelId", source) - self.assertIn("postSyncChannelId = postResource.channelId", source) + self.assertIn("postSyncChannelId = preResource.channelId", source) self.assertIn("RemoteToLocal", source) self.assertIn("uint32_t memSlicePerBlock", header) @@ -1796,15 +1808,37 @@ def test_collective_planner_has_private_alltoall_prepare_path(self): self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", header) self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", source) self.assertIn("rankSize != 2", source) - self.assertIn("alltoall.localRecvAddr = localEndpoint.destinationAddr", source) - self.assertIn("alltoall.localRecvToken = localEndpoint.destinationToken", source) - self.assertIn("remoteImportRequest = peerEndpoint.sourceRemoteImport", source) + self.assertIn("alltoall.localSendAddr = localEndpoint.sourceAddr", source) + self.assertIn("alltoall.localSendToken = localEndpoint.sourceToken", source) + self.assertIn("remoteImportRequest = peerEndpoint.destinationRemoteImport", source) self.assertIn("alltoall.remoteSendAddr = remoteImportRequest.addr", source) self.assertIn("alltoall.remoteSendToken", source) + self.assertIn("alltoall.remoteRecvAddr = peerEndpoint.destinationAddr", source) self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) self.assertIn("tilexr-comm-direct-ccu-alltoall", source) self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) + def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner_header = PLANNER_HEADER.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("struct TileXRCcuDirectSyncXnPingSpec", header) + self.assertIn("uint16_t remoteNotifyMask = 0", header) + self.assertIn("uint16_t localWaitMask = 0", header) + self.assertIn("TileXRCcuRunDirectSyncXnPingInstallAttempt", header) + self.assertIn("BuildDirectSyncXnPingLaunchPackage", source) + self.assertIn("TileXRCcuEncodeSyncXn", source) + self.assertIn("defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank)", source) + self.assertIn("defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank)", source) + self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U", source) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner_header) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK", planner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK", planner) + self.assertIn("RegisterCcuResourceRmaBuffer", planner) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index e07bb012..14abdd70 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -245,6 +245,8 @@ def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION", source) self.assertIn("P2pCcuCopyDirectionFromEnv", source) self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) + self.assertIn('"LocalToRemote"', source) + self.assertIn('"1"', source) self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) self.assertIn("p2pCcuCopy", source) @@ -597,6 +599,9 @@ def test_alltoall_inactive_rank_still_prepares_before_host_phase_wait(self): inactive_index = alltoall_body.index("if (!active)") self.assertLess(prepare_index, inactive_index) self.assertIn("const bool phaseReady =", alltoall_body) + self.assertIn("WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady, phase)", alltoall_body) + self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase)", alltoall_body) + self.assertIn("RankPhaseFileStem", source) def test_thread_mode_rank_specific_resource_env_overrides_common_prepare_options(self): source = PROBE_SOURCE.read_text(encoding="utf-8") @@ -650,25 +655,52 @@ def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): source = PROBE_SOURCE.read_text(encoding="utf-8") self.assertIn('kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"', source) + self.assertIn('kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"', source) self.assertIn('kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"', source) self.assertIn('kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"', source) self.assertIn("struct AllToAllState", source) self.assertIn("AllToAllSmokeEnabled", source) + self.assertIn("AllToAllLongMissionEnabled", source) self.assertIn("InitAllToAllState", source) self.assertIn("RunAllToAllCopyPhase", source) + self.assertIn("RunAllToAllLongMissionSmokeForRank", source) self.assertIn("RunAllToAllSmokeForRank", source) self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", source) self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) self.assertIn("tilexr_ccu_alltoall config", source) self.assertIn("tilexr_ccu_alltoall result passed=1", source) self.assertIn("BuildP2pCcuCopyPattern(peer", source) + def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn('kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"', source) + self.assertIn("SyncXnPingSmokeEnabled", source) + self.assertIn("RunSyncXnPingSmokeForRank", source) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", source) + sync_ping_body = source[ + source.index("int RunSyncXnPingSmokeForRank"): + source.index("int RunSignalWaitSmokeForRank") + ] + self.assertIn("AllToAllState routeState", sync_ping_body) + self.assertIn("InitAllToAllState(rank, peer, &routeState)", sync_ping_body) + self.assertIn("tilexr_ccu_sync_xn_ping prepare", source) + self.assertIn("tilexr_ccu_sync_xn_ping submit", source) + self.assertIn("tilexr_ccu_sync_xn_ping timing", source) + self.assertIn("aclrtSynchronizeStreamWithTimeout", source) + self.assertIn("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", source) + self.assertIn("tilexr_ccu_sync_xn_ping result passed=1", source) + def test_smoke_runner_forwards_alltoall_env(self): runner = (REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh").read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL", runner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION", runner) self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) + self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476", runner) self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7", runner) def test_probe_compiles_and_default_run_skips_without_touching_hardware(self): diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index c1248231..66b1b45c 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -34,6 +34,8 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION", source) self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) @@ -143,6 +145,8 @@ def test_runner_p2p_mode_applies_direct_ccu_resource_defaults(self): self.assertIn('TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}"', source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}"', source) self.assertIn('common_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION}")', source) self.assertIn("p2p_passed_count=0", source) self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) self.assertIn("direct CCU P2P CCU-copy produced no passing receiver result", source) @@ -167,6 +171,19 @@ def test_runner_signal_wait_mode_uses_internal_backend_defaults_and_asserts_resu self.assertIn('grep -q "tilexr_ccu_signal_wait submit ret=0"', source) self.assertIn('grep -q "tilexr_ccu_signal_wait result passed=1"', source) + def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("sync_xn_ping_mode_enabled", source) + self.assertIn("apply_sync_xn_ping_defaults", source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0', source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}"', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}"', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}"', source) + self.assertLess(source.index("apply_sync_xn_ping_defaults"), source.index("apply_alltoall_defaults")) + def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): source = RUNNER.read_text(encoding="utf-8") submit_check = source[ diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 09cfd586..969d9be2 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -2154,8 +2154,8 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): ] self.assertNotIn("SelectDirectCcuRemoteBindingOverride", exchange_body) self.assertIn("peerLocalWaitCkeOffset", exchange_body) - self.assertIn("peerResources.localWaitCkeStartId", exchange_body) self.assertIn("peerResources.localWaitCkeCount", exchange_body) + self.assertIn("peerResources.remoteNotifyCkeCount", exchange_body) self.assertNotIn("allocation.remoteNotifyCke.startId,\n routeIndex", exchange_body) self.assertIn("allocation.localXn.startId", planner_source) self.assertIn("remoteXnStartId", exchange_body) @@ -2225,7 +2225,7 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn("RefreshDirectCcuLowerLayerPlan();", register_memory_body) self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_memory_body) - def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): + def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): @@ -2233,13 +2233,12 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): ] compact_body = " ".join(exchange_body.split()) - self.assertIn( - "channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( peerResources.remoteXnStartId, peerLocalIndex, syncIndex, peerRouteCount)", - compact_body) self.assertIn( "peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset)", compact_body) self.assertIn("selectedRemoteXnOffset >= peerResources.remoteXnCount", compact_body) + self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertNotIn("SelectDirectCcuRemoteBindingOverride", compact_body) self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn("(*remoteCcuBuffers)[routeIndex].remoteCcuVa +=", compact_body) @@ -2255,7 +2254,7 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): "static_cast((*remoteCcuBuffers)[routeIndex].remoteXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) - def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): + def test_remote_notify_cke_targets_peer_local_wait_cke(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): @@ -2265,9 +2264,12 @@ def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): self.assertIn("peerLocalWaitCkeOffset", exchange_body) self.assertIn("peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount", compact_body) + self.assertIn("peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount", compact_body) self.assertIn( "remoteNotifyCke = static_cast(static_cast(peerResources.localWaitCkeStartId) + peerLocalWaitCkeOffset)", compact_body) + self.assertIn("remoteNotifyCke) >= peerResources.localWaitCkeStartId", compact_body) + self.assertIn("peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount", compact_body) self.assertNotIn( "remoteNotifyCke = SelectDirectCcuRemoteNotifyCkeId( allocation.remoteNotifyCke.startId, routeIndex)", compact_body) @@ -2291,7 +2293,8 @@ def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) self.assertIn("const size_t peerBufferIndex = syncIndex % peerRouteCount", compact_body) self.assertIn("(*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]", compact_body) - self.assertIn("channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self): From 2dc0f260776e6f10ba5d851127402e45c5ed9340 Mon Sep 17 00:00:00 2001 From: linzhen 00654177 Date: Fri, 17 Jul 2026 09:35:27 +0800 Subject: [PATCH 24/41] ccu alltoall bidirection loop0 error loop1-63 success --- src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 20 ++- .../ccu/tilexr_ccu_collective_planner.cpp | 12 +- .../ccu/tilexr_ccu_direct_orchestrator.cpp | 8 +- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 144 +++++++++++++++++- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 2 +- tests/ccu/test_tilexr_ccu_alltoall_program.py | 30 ++-- .../test_tilexr_ccu_direct_orchestrator.py | 25 ++- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 7 +- 8 files changed, 198 insertions(+), 50 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 85316ffa..2c31d6fa 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -10,14 +10,12 @@ namespace { uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) { - (void)spec; - return static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); + return spec.ckeMask; } uint16_t PostSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) { - (void)spec; - return static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); + return spec.ckeMask; } void ResetReport(TileXRCcuAllToAllProgramReport* report) @@ -52,10 +50,10 @@ int ValidateSpec( if (spec.localRank > 1U) { return Fail(program, report, "direct CCU alltoall localRank must be 0 or 1"); } - if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || spec.remoteSendAddr == 0) { + if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || spec.remoteRecvAddr == 0) { return Fail(program, report, "missing direct CCU alltoall address"); } - if (spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.remoteSendToken == 0) { + if (spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.remoteRecvToken == 0) { return Fail(program, report, "missing direct CCU alltoall token"); } if (spec.bytes == 0 || spec.bytes % TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES != 0) { @@ -255,16 +253,16 @@ int AppendCopyBlock( TileXRCcuAllToAllProgramReport* report) { TileXRCcuMemoryCopySpec copy; - copy.direction = TileXRCcuMemoryCopyDirection::RemoteToLocal; + copy.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; copy.localGsa = spec.localGsa; copy.localXn = spec.localXn; copy.remoteGsa = spec.remoteGsa; copy.remoteXn = spec.remoteXn; copy.lengthXn = spec.lengthXn; - copy.localAddr = spec.localRecvAddr + offset; - copy.localToken = spec.localRecvToken; - copy.remoteAddr = spec.remoteSendAddr + offset; - copy.remoteToken = spec.remoteSendToken; + copy.localAddr = spec.localSendAddr + offset; + copy.localToken = spec.localSendToken; + copy.remoteAddr = spec.remoteRecvAddr + offset; + copy.remoteToken = spec.remoteRecvToken; copy.lengthBytes = bytesPerBlock; copy.channelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; copy.completionCke = spec.copyCompletionCke; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 862d0868..8efb9ce3 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -1295,12 +1295,12 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( } TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; - TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.sourceRemoteImport; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); if (ret != TILEXR_SUCCESS) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to import direct CCU alltoall remote source buffer"; + report->message = "failed to import direct CCU alltoall remote destination buffer"; } return ret; } @@ -1311,11 +1311,11 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( alltoall.localSendToken = localEndpoint.sourceToken; alltoall.localRecvAddr = localEndpoint.destinationAddr; alltoall.localRecvToken = localEndpoint.destinationToken; - alltoall.remoteSendAddr = remoteImportRequest.addr; - alltoall.remoteSendToken = + alltoall.remoteSendAddr = peerEndpoint.sourceAddr; + alltoall.remoteSendToken = peerEndpoint.sourceToken; + alltoall.remoteRecvAddr = remoteImportRequest.addr; + alltoall.remoteRecvToken = TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); - alltoall.remoteRecvAddr = peerEndpoint.destinationAddr; - alltoall.remoteRecvToken = peerEndpoint.destinationToken; alltoall.bytes = bytes; alltoall.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; alltoall.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index 3f05b61f..8a282921 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -952,7 +952,7 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.preSyncChannelId = preResource.channelId; alltoallSpec.preSyncTokenChannelId = postResource.channelId; alltoallSpec.copyChannelId = copyResource.channelId; - alltoallSpec.postSyncChannelId = preResource.channelId; + alltoallSpec.postSyncChannelId = postResource.channelId; alltoallSpec.preSyncRemoteAddrXn = preResource.remoteXn; alltoallSpec.preSyncRemoteTokenXn = postResource.remoteXn; alltoallSpec.preSyncRemoteNotifyCke = preResource.notifyCke; @@ -963,9 +963,9 @@ int BuildDirectAllToAll2RankLaunchPackage( postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; alltoallSpec.copyCompletionCke = copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke; - alltoallSpec.postSyncRemoteNotifyCke = preResource.notifyCke; + alltoallSpec.postSyncRemoteNotifyCke = postResource.notifyCke; alltoallSpec.postSyncLocalWaitCke = - preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; + postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; alltoallSpec.sourceCke = preResource.sourceCke; alltoallSpec.ckeMask = preResource.remoteNotifyMask == 0 ? 1U : preResource.remoteNotifyMask; alltoallSpec.postSyncNotify = false; @@ -974,7 +974,7 @@ int BuildDirectAllToAll2RankLaunchPackage( if (DirectTraceEnabled()) { std::cerr << "TileXRDirectCcuTrace alltoallSpec" - << " direction=RemoteToLocal" + << " direction=LocalToRemote" << " localRank=" << alltoallSpec.localRank << " localGsa=" << alltoallSpec.localGsa << " localXn=" << alltoallSpec.localXn diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 6fce2c50..62db23bd 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -141,6 +141,13 @@ struct AllToAllState { int initRet = ACL_SUCCESS; int readRet = ACL_SUCCESS; uint32_t mismatchCount = 0; + size_t firstMismatchOffset = 0; + size_t lastMismatchOffset = 0; + uint32_t firstMismatchObserved = 0; + uint32_t firstMismatchExpected = 0; + uint32_t mismatchedBlockCount = 0; + uint32_t firstMismatchedBlock = 0; + uint32_t lastMismatchedBlock = 0; bool passed = false; }; @@ -621,8 +628,32 @@ int CheckAllToAllState(AllToAllState* state) return state->readRet; } state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + const size_t blockBytes = 8U * 4096U; + uint32_t currentBlock = UINT32_MAX; for (size_t i = 0; i < state->bytes; ++i) { if (state->observed[i] != state->expected[i]) { + if (state->mismatchCount == 0) { + state->firstMismatchOffset = i; + state->firstMismatchObserved = state->observed[i]; + state->firstMismatchExpected = state->expected[i]; + } + state->lastMismatchOffset = i; + const uint32_t block = static_cast(i / blockBytes); + if (block != currentBlock) { + if (state->mismatchedBlockCount == 0) { + state->firstMismatchedBlock = block; + } + state->lastMismatchedBlock = block; + currentBlock = block; + ++state->mismatchedBlockCount; + } ++state->mismatchCount; } } @@ -1135,6 +1166,54 @@ uint16_t LoadLe16(const uint8_t* data, size_t index) static_cast(static_cast(data[index * 2U + 1U]) << 8U); } +bool ReadMissionContextAtEnd( + DirectCcuSmokeContext* context, + const TileXRDirectCcuTaskInfo& task, + const char* label) +{ + if (context == nullptr || label == nullptr) { + return false; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport adapterReport; + int ret = context->session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return false; + } + + uint8_t raw[TileXR::TILEXR_CCU_DATA_ARRAY_SLOT_BYTES] = {}; + ret = adapter.ReadMissionContext( + task.dieId, + task.missionId, + raw, + sizeof(raw), + &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " opcode=" << adapterReport.opcode + << " driverRet=" << adapterReport.driverRet + << " opRet=" << adapterReport.opRet + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return false; + } + + const uint16_t part4 = LoadLe16(raw, 4); + const uint16_t part5 = LoadLe16(raw, 5); + const uint16_t part6 = LoadLe16(raw, 6); + const uint16_t currentIns = static_cast(((part5 & 0x1fU) << 11U) | ((part4 >> 5U) & 0x7ffU)); + const uint16_t endIns = static_cast(((part6 & 0x1fU) << 11U) | ((part5 >> 5U) & 0x7ffU)); + const bool atEnd = currentIns == endIns; + std::cerr << label << " missionCtxAtEnd=" << (atEnd ? 1 : 0) + << " currentIns=" << currentIns + << " endIns=" << endIns + << std::endl; + return atEnd; +} + void PrintMissionContext( DirectCcuSmokeContext* context, const TileXRDirectCcuTaskInfo& task, @@ -1568,6 +1647,7 @@ int RunAllToAllCopyPhase( std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; finalRet = 7; } else { + bool skipStreamDestroy = false; TileXRDirectCcuSubmitReport submitReport; const auto submitBegin = std::chrono::steady_clock::now(); const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); @@ -1593,17 +1673,37 @@ int RunAllToAllCopyPhase( if (syncRet != ACL_SUCCESS) { std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet << " timeoutMs=" << syncTimeoutMs << std::endl; + bool missionAtEnd = false; if (!attempt.submitTasks.empty()) { PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + missionAtEnd = ReadMissionContextAtEnd( + context, + attempt.submitTasks.front(), + "tilexr_ccu_alltoall"); + } + if (missionAtEnd) { + std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" + << " rank=" << rank + << " reason=\"continuing to device buffer validation\"" + << std::endl; + skipStreamDestroy = true; + } else { + finalRet = 8; } - finalRet = 8; } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; } if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase) && finalRet == 0) { finalRet = 13; } - aclrtDestroyStream(stream); + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" + << " rank=" << rank + << " reason=\"stream timeout after mission reached end\"" + << std::endl; + } else { + aclrtDestroyStream(stream); + } } } const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); @@ -1625,6 +1725,14 @@ void PrintAllToAllResultAndMaybeFastExit(int rank, int finalRet, const AllToAllS << " ret=" << finalRet << " readRet=" << alltoall.readRet << " mismatches=" << alltoall.mismatchCount + << " firstMismatchOffset=" << alltoall.firstMismatchOffset + << " lastMismatchOffset=" << alltoall.lastMismatchOffset + << " firstMismatchObserved=0x" << std::hex << alltoall.firstMismatchObserved + << " firstMismatchExpected=0x" << alltoall.firstMismatchExpected + << std::dec + << " mismatchedBlocks=" << alltoall.mismatchedBlockCount + << " firstMismatchedBlock=" << alltoall.firstMismatchedBlock + << " lastMismatchedBlock=" << alltoall.lastMismatchedBlock << std::endl; } if (ShouldFastExitAfterRun()) { @@ -1656,7 +1764,9 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); options.syncResourceCount = 3; options.sqeArgCount = 0; - options.syncInstructionCount = 3 + 64 * 7 + 3 + 1; + if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { + options.syncInstructionCount = 3 + 64 * 7; + } if (options.gsaStartId == 0) { options.gsaStartId = 1; } @@ -1669,7 +1779,8 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, << " blockCount=64" << " longMission=1" << " preSync=1" - << " postSync=1" + << " postSync=0" + << " finish=0" << std::endl; PrintConfig(options, rankSize); @@ -1714,6 +1825,7 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; finalRet = 7; } else { + bool skipStreamDestroy = false; TileXRDirectCcuSubmitReport submitReport; const auto submitBegin = std::chrono::steady_clock::now(); const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); @@ -1739,17 +1851,37 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, if (syncRet != ACL_SUCCESS) { std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet << " timeoutMs=" << syncTimeoutMs << std::endl; + bool missionAtEnd = false; if (!attempt.submitTasks.empty()) { PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + missionAtEnd = ReadMissionContextAtEnd( + context, + attempt.submitTasks.front(), + "tilexr_ccu_alltoall"); + } + if (missionAtEnd) { + std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" + << " rank=" << rank + << " reason=\"continuing to device buffer validation\"" + << std::endl; + skipStreamDestroy = true; + } else { + finalRet = 8; } - finalRet = 8; } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; } if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { finalRet = 13; } - aclrtDestroyStream(stream); + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" + << " rank=" << rank + << " reason=\"stream timeout after mission reached end\"" + << std::endl; + } else { + aclrtDestroyStream(stream); + } } } diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 7e68e66a..1ea5af0e 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -219,7 +219,7 @@ apply_alltoall_defaults() export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451}" else export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index 54409701..f608d7e4 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -153,15 +153,14 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): std::cerr << "unexpected block size or message\n"; return 3; } - const uint16_t preSyncMask = static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); - const uint16_t postSyncMask = static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); + const uint16_t syncMask = spec.ckeMask; const uint32_t postSyncSetIndex = expectedInstructions - 4; const uint32_t postSyncWaitIndex = expectedInstructions - 2; if (program.size() < 3 || Slot(program[2], 0) != kSetCkeHeader || Slot(program[2], 4) != spec.preSyncLocalWaitCke || - Slot(program[2], 5) != preSyncMask) { - std::cerr << "PreSync should use the HCCL-style output barrier bit" + Slot(program[2], 5) != syncMask) { + std::cerr << "PreSync should use the allocated CKE resource mask" << " header=0x" << std::hex << Slot(program[2], 0) << " waitCke=0x" << Slot(program[2], 4) << " waitMask=0x" << Slot(program[2], 5) @@ -169,16 +168,15 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): return 4; } if (Slot(program[postSyncSetIndex], 0) != kSetCkeHeader || - Slot(program[postSyncSetIndex], 3) != postSyncMask || + Slot(program[postSyncSetIndex], 3) != syncMask || Slot(program[postSyncWaitIndex], 0) != kClearCkeHeader || Slot(program[postSyncWaitIndex], 4) != spec.postSyncLocalWaitCke || - Slot(program[postSyncWaitIndex], 5) != postSyncMask || - postSyncMask == preSyncMask) { - std::cerr << "PostSync should use a distinct HCCL-style post barrier bit" + Slot(program[postSyncWaitIndex], 5) != syncMask) { + std::cerr << "PostSync should use the allocated post CKE resource with the resource mask" << " postSetMask=0x" << std::hex << Slot(program[postSyncSetIndex], 3) << " postWaitCke=0x" << Slot(program[postSyncWaitIndex], 4) << " postWaitMask=0x" << Slot(program[postSyncWaitIndex], 5) - << " preMask=0x" << preSyncMask + << " syncMask=0x" << syncMask << std::dec << "\n"; return 5; } @@ -202,7 +200,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): using namespace TileXR; - constexpr uint16_t kTransRmtMemToLocMemHeader = 0x1008U; + constexpr uint16_t kTransLocMemToRmtMemHeader = 0x1009U; uint16_t Header(const TileXRCcuInstr& instr) { @@ -247,7 +245,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): size_t FirstCopyIndex(const std::vector& program) { for (size_t i = 0; i < program.size(); ++i) { - if (Header(program[i]) == kTransRmtMemToLocMemHeader) { + if (Header(program[i]) == kTransLocMemToRmtMemHeader) { return i; } } @@ -258,7 +256,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): { uint32_t count = 0; for (const auto& instr : program) { - if (Header(instr) == kTransRmtMemToLocMemHeader) { + if (Header(instr) == kTransLocMemToRmtMemHeader) { ++count; } } @@ -291,7 +289,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): return 2; } if (CopyInstructionCount(rank0) != 64 || CopyInstructionCount(rank1) != 64) { - std::cerr << "each rank should issue exactly 64 remote-to-local reads\n"; + std::cerr << "each rank should issue exactly 64 local-to-remote writes\n"; return 3; } const size_t rank0FirstCopy = FirstCopyIndex(rank0); @@ -415,12 +413,16 @@ def test_builder_is_wired_and_has_no_hccl_dependency_surface(self): self.assertIn("ccu/tilexr_ccu_alltoall_program.cpp", cmake) self.assertIn("TileXRCcuBuildAllToAll2RankProgram", header) self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) - self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) self.assertIn("preSyncRemoteTokenNotifyCke", header) self.assertIn("preSyncTokenLocalWaitCke", header) self.assertNotIn("tokenLocalWaitCke", source) self.assertIn("PreSyncSignalMask", source) self.assertIn("PostSyncSignalMask", source) + self.assertIn("return spec.ckeMask;", source) + self.assertNotIn("1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID", source) + self.assertNotIn("1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID", source) + self.assertIn("post.clearWait = true;", source) self.assertIn("TILEXR_CCU_ALLTOALL_SIGNAL_MASK", header) self.assertIn("TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK", header) self.assertIn("TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK", header) diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index ef1db07b..20956afe 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1750,11 +1750,13 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 3U + 64U * 7U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 3U + 64U * 7U", planner) self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke", source) self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) self.assertIn("const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]", source) + self.assertIn("const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]", source) self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(\n 0U,", planner) self.assertIn("preSyncRemoteAddrXn = preResource.remoteXn", source) self.assertIn("preSyncRemoteTokenXn = postResource.remoteXn", source) @@ -1766,11 +1768,20 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): self.assertIn("preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke", source) self.assertIn("alltoallSpec.copyCompletionCke =", source) self.assertIn("copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke", source) - self.assertIn("postSyncRemoteNotifyCke = preResource.notifyCke", source) + self.assertIn("postSyncRemoteNotifyCke = postResource.notifyCke", source) + self.assertNotIn("postSyncRemoteNotifyCke = preResource.notifyCke", source) + self.assertIn("postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke", source) self.assertIn("preSyncChannelId = preResource.channelId", source) self.assertIn("copyChannelId = copyResource.channelId", source) - self.assertIn("postSyncChannelId = preResource.channelId", source) - self.assertIn("RemoteToLocal", source) + self.assertIn("postSyncChannelId = postResource.channelId", source) + self.assertNotIn("postSyncChannelId = preResource.channelId", source) + self.assertIn("postSyncNotify = false", source) + self.assertIn("postSyncWait = false", source) + self.assertIn("emitFinish = false", source) + self.assertNotIn("postSyncNotify = true", source) + self.assertNotIn("postSyncWait = true", source) + self.assertNotIn("emitFinish = true", source) + self.assertIn("LocalToRemote", source) self.assertIn("uint32_t memSlicePerBlock", header) combined = header + "\n" + source @@ -1811,9 +1822,9 @@ def test_collective_planner_has_private_alltoall_prepare_path(self): self.assertIn("alltoall.localSendAddr = localEndpoint.sourceAddr", source) self.assertIn("alltoall.localSendToken = localEndpoint.sourceToken", source) self.assertIn("remoteImportRequest = peerEndpoint.destinationRemoteImport", source) - self.assertIn("alltoall.remoteSendAddr = remoteImportRequest.addr", source) - self.assertIn("alltoall.remoteSendToken", source) - self.assertIn("alltoall.remoteRecvAddr = peerEndpoint.destinationAddr", source) + self.assertIn("alltoall.remoteRecvAddr = remoteImportRequest.addr", source) + self.assertIn("alltoall.remoteRecvToken", source) + self.assertNotIn("alltoall.remoteRecvAddr = peerEndpoint.destinationAddr", source) self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) self.assertIn("tilexr-comm-direct-ccu-alltoall", source) self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 14abdd70..e7a01a35 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -700,7 +700,12 @@ def test_smoke_runner_forwards_alltoall_env(self): self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) - self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476", runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-452", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-458", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-454", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-455", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476", runner) self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7", runner) def test_probe_compiles_and_default_run_skips_without_touching_hardware(self): From 8fabf183017c97c8f3c2ed17ecc94bb40e03430c Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 09:14:00 +0800 Subject: [PATCH 25/41] fix(ccu): complete direct all-to-all synchronization --- src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 102 +++++++++++++----- src/comm/ccu/tilexr_ccu_alltoall_program.h | 2 + .../ccu/tilexr_ccu_collective_planner.cpp | 5 +- .../ccu/tilexr_ccu_direct_orchestrator.cpp | 43 +++++--- src/comm/ccu/tilexr_ccu_driver_adapter.cpp | 59 ++++++++++ src/comm/ccu/tilexr_ccu_driver_adapter.h | 21 ++++ .../tilexr_ccu_lower_layer_plan_builder.cpp | 17 +-- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 82 +++++++++++++- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 12 +++ tests/ccu/test_tilexr_ccu_alltoall_program.py | 101 +++++++++++++---- .../test_tilexr_ccu_direct_orchestrator.py | 51 +++++++-- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 16 +++ tests/ccu/test_tilexr_ccu_driver_adapter.py | 64 +++++++++++ ...est_tilexr_ccu_lower_layer_plan_builder.py | 8 +- 14 files changed, 489 insertions(+), 94 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 2c31d6fa..7a48a691 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -10,7 +10,14 @@ namespace { uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) { - return spec.ckeMask; + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); +} + +uint16_t PreSyncTokenMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID); } uint16_t PostSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) @@ -79,7 +86,8 @@ int ValidateSpec( if (preSyncChannelId == 0 || copyChannelId == 0 || postSyncChannelId == 0 || spec.copyCompletionCke == 0 || spec.preSyncLocalWaitCke == 0 || spec.preSyncRemoteNotifyCke == 0 || spec.postSyncLocalWaitCke == 0 || - spec.postSyncRemoteNotifyCke == 0 || spec.sourceCke == 0 || spec.ckeMask == 0) { + spec.postSyncRemoteNotifyCke == 0 || + (spec.postSyncNotify && spec.sourceCke == 0) || spec.ckeMask == 0) { return Fail(program, report, "missing direct CCU alltoall CKE/channel resource"); } return TILEXR_SUCCESS; @@ -157,26 +165,31 @@ int AppendSyncPair( int AppendRemoteNotify( uint16_t remoteNotifyCke, uint16_t channelId, - const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t localXn, + uint16_t remoteXn, + uint64_t value, + uint16_t secFlag, uint16_t mask, const char* phase, std::vector* program, TileXRCcuAllToAllProgramReport* report) { - if (AppendSetSourceCke(spec, mask, program, report) != TILEXR_SUCCESS) { - return TILEXR_ERROR_PARA_CHECK_FAIL; + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(localXn, value, secFlag, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " variable load"); } + program->push_back(instr); - TileXRCcuSyncCkeSpec post; - post.remoteCke = remoteNotifyCke; - post.localCke = spec.sourceCke; - post.localCkeMask = mask; - post.channelId = channelId; - post.clearWait = true; + TileXRCcuSyncXnSpec notify; + notify.remoteXn = remoteXn; + notify.localXn = localXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = mask; + notify.clearWait = true; - TileXRCcuInstr instr; - if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { - return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " SyncCke notify"); + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " SyncXn notify"); } program->push_back(instr); return TILEXR_SUCCESS; @@ -190,18 +203,47 @@ int AppendPreSyncPhase( std::vector* program, TileXRCcuAllToAllProgramReport* report) { - const uint16_t notifyMask = PreSyncSignalMask(spec); - const uint16_t waitMask = PreSyncSignalMask(spec); + const uint16_t outputMask = PreSyncSignalMask(spec); + const uint16_t tokenMask = PreSyncTokenMask(spec); + const uint16_t waitMask = static_cast(outputMask | tokenMask); + const uint16_t localOutputXn = + spec.preSyncLocalAddrXn == 0 ? spec.localXn : spec.preSyncLocalAddrXn; + const uint16_t localTokenXn = + spec.preSyncLocalTokenXn == 0 ? spec.lengthXn : spec.preSyncLocalTokenXn; + const uint16_t tokenChannelId = + spec.preSyncTokenChannelId == 0 ? outputChannelId : spec.preSyncTokenChannelId; + const uint16_t tokenNotifyCke = + spec.preSyncRemoteTokenNotifyCke == 0 ? remoteNotifyCke : spec.preSyncRemoteTokenNotifyCke; if (AppendRemoteNotify( remoteNotifyCke, outputChannelId, - spec, - notifyMask, + localOutputXn, + spec.preSyncRemoteAddrXn, + spec.localRecvAddr, + 0, + outputMask, "PreSync output", program, report) != TILEXR_SUCCESS) { return TILEXR_ERROR_PARA_CHECK_FAIL; } + if (AppendRemoteNotify( + tokenNotifyCke, + tokenChannelId, + localTokenXn, + spec.preSyncRemoteTokenXn, + spec.localRecvToken, + 1, + tokenMask, + "PreSync token", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (!spec.preSyncWait) { + return TILEXR_SUCCESS; + } return AppendNotifyWait(localWaitCke, waitMask, "PreSync output", false, program, report); } @@ -299,7 +341,7 @@ void FillReport( return; } const uint32_t bytesPerBlock = spec.memorySliceBytes * spec.memSlicePerBlock; - report->preSyncInstructionCount = 3; + report->preSyncInstructionCount = spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) : 0U; report->blockCount = static_cast(spec.bytes / bytesPerBlock); report->bytesPerBlock = bytesPerBlock; report->copyInstructionCount = report->blockCount * 7U; @@ -330,19 +372,21 @@ int TileXRCcuBuildAllToAll2RankProgram( const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; program->reserve( - 3U + blockCount * 7U + + (spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) : 0U) + blockCount * 7U + (!spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U)) + (spec.emitFinish ? 1U : 0U)); - ret = AppendPreSyncPhase( - spec.preSyncRemoteNotifyCke, - spec.preSyncLocalWaitCke, - preSyncChannelId, - spec, - program, - report); - if (ret != TILEXR_SUCCESS) { - return ret; + if (spec.preSyncNotify) { + ret = AppendPreSyncPhase( + spec.preSyncRemoteNotifyCke, + spec.preSyncLocalWaitCke, + preSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } } for (uint32_t block = 0; block < blockCount; ++block) { diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index 88cf45e8..5d06471c 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -61,6 +61,8 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t postSyncRemoteNotifyCke = 0; uint16_t sourceCke = 0; uint16_t ckeMask = 1; + bool preSyncNotify = true; + bool preSyncWait = true; bool postSyncNotify = true; bool postSyncWait = true; bool emitFinish = true; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 8efb9ce3..4243b5b0 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -26,7 +26,7 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 3U + 64U * 7U; + 5U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; @@ -662,7 +662,8 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( const bool notifyCkeOwnerVerified = static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && static_cast(remoteNotifyCke) < - static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; + static_cast(peerResources.localWaitCkeStartId) + + peerResources.localWaitCkeCount; const bool localChannelOwnerVerified = allocation.channels.num != 0 && peerLocalXnOwnerVerified && diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index 8a282921..ea9fb2f4 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -38,7 +38,7 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 3U + 64U * 7U; + 5U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; @@ -927,6 +927,12 @@ int BuildDirectAllToAll2RankLaunchPackage( const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]; const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]; const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]; + const bool preSyncOnCopyRoute = + std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE") != nullptr; + const bool preSyncPeerLocalXn = + std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_PEER_LOCAL_XN") != nullptr; + const uint16_t preSyncRemoteAddrXn = + preSyncPeerLocalXn ? preResource.localXn : preResource.remoteXn; TileXRCcuAllToAll2RankProgramSpec alltoallSpec; alltoallSpec.localRank = alltoall.localRank; @@ -946,21 +952,28 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.localXn = attempt->plan.kernelLocalXn.startId; alltoallSpec.remoteXn = static_cast(attempt->plan.kernelLocalXn.startId + 1U); alltoallSpec.lengthXn = static_cast(attempt->plan.kernelLocalXn.startId + 2U); - alltoallSpec.preSyncLocalAddrXn = preResource.localXn; + alltoallSpec.preSyncLocalAddrXn = + preSyncOnCopyRoute ? copyResource.localXn : preResource.localXn; alltoallSpec.preSyncLocalTokenXn = postResource.localXn; alltoallSpec.channelId = copyResource.channelId; - alltoallSpec.preSyncChannelId = preResource.channelId; - alltoallSpec.preSyncTokenChannelId = postResource.channelId; + alltoallSpec.preSyncChannelId = + preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId; + alltoallSpec.preSyncTokenChannelId = preResource.channelId; alltoallSpec.copyChannelId = copyResource.channelId; alltoallSpec.postSyncChannelId = postResource.channelId; - alltoallSpec.preSyncRemoteAddrXn = preResource.remoteXn; - alltoallSpec.preSyncRemoteTokenXn = postResource.remoteXn; - alltoallSpec.preSyncRemoteNotifyCke = preResource.notifyCke; + alltoallSpec.preSyncRemoteAddrXn = + preSyncOnCopyRoute ? copyResource.localXn : preSyncRemoteAddrXn; + alltoallSpec.preSyncRemoteTokenXn = + preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn; + alltoallSpec.preSyncRemoteNotifyCke = + preSyncOnCopyRoute ? attempt->allocation.remoteNotifyCke.startId : preResource.notifyCke; alltoallSpec.preSyncLocalWaitCke = - preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; - alltoallSpec.preSyncRemoteTokenNotifyCke = postResource.notifyCke; + preSyncOnCopyRoute + ? (copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke) + : (preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke); + alltoallSpec.preSyncRemoteTokenNotifyCke = preResource.notifyCke; alltoallSpec.preSyncTokenLocalWaitCke = - postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; + preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; alltoallSpec.copyCompletionCke = copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke; alltoallSpec.postSyncRemoteNotifyCke = postResource.notifyCke; @@ -968,6 +981,8 @@ int BuildDirectAllToAll2RankLaunchPackage( postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; alltoallSpec.sourceCke = preResource.sourceCke; alltoallSpec.ckeMask = preResource.remoteNotifyMask == 0 ? 1U : preResource.remoteNotifyMask; + alltoallSpec.preSyncNotify = std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC") == nullptr; + alltoallSpec.preSyncWait = std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT") == nullptr; alltoallSpec.postSyncNotify = false; alltoallSpec.postSyncWait = false; alltoallSpec.emitFinish = false; @@ -994,6 +1009,10 @@ int BuildDirectAllToAll2RankLaunchPackage( << " preRemoteTokenXn=" << alltoallSpec.preSyncRemoteTokenXn << " copyCompletionCke=" << alltoallSpec.copyCompletionCke << " postNotifyCke=" << alltoallSpec.postSyncRemoteNotifyCke + << " preSyncNotify=" << (alltoallSpec.preSyncNotify ? 1 : 0) + << " preSyncWait=" << (alltoallSpec.preSyncWait ? 1 : 0) + << " preSyncOnCopyRoute=" << (preSyncOnCopyRoute ? 1 : 0) + << " preSyncPeerLocalXn=" << (preSyncPeerLocalXn ? 1 : 0) << " postSyncNotify=" << (alltoallSpec.postSyncNotify ? 1 : 0) << " postSyncWait=" << (alltoallSpec.postSyncWait ? 1 : 0) << " emitFinish=" << (alltoallSpec.emitFinish ? 1 : 0) @@ -1561,7 +1580,7 @@ int RunDirectInstallAttemptImpl( options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; attempt->resourceRequest.barrierMode = - alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : + alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); @@ -1602,7 +1621,7 @@ int RunDirectInstallAttemptImpl( } } attempt->plan.barrierMode = - alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke : + alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp index f8cf6a49..71945955 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp @@ -349,6 +349,65 @@ int TileXRCcuDriverAdapter::ReadMissionContext( return TILEXR_SUCCESS; } +int TileXRCcuDriverAdapter::ReadResourceRange( + uint8_t dieId, + uint32_t opcode, + uint32_t startId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (values == nullptr || count == 0) { + return Fail(report, "invalid CCU resource readback range"); + } + + uint32_t remaining = count; + uint32_t offset = startId; + uint32_t outputOffset = 0; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, opcode, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * sizeof(uint64_t); + + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, opcode, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + for (uint32_t i = 0; i < batch; ++i) { + std::memcpy(&values[outputOffset + i], &out.data.dataInfo.dataArray[i], sizeof(uint64_t)); + } + remaining -= batch; + offset += batch; + outputOffset += batch; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::ReadXnRange( + uint8_t dieId, + uint32_t startXnId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + return ReadResourceRange(dieId, TILEXR_CCU_U_OP_GET_XN, startXnId, values, count, report); +} + +int TileXRCcuDriverAdapter::ReadCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + return ReadResourceRange(dieId, TILEXR_CCU_U_OP_GET_CKE, startCkeId, values, count, report); +} + int TileXRCcuDriverAdapter::InstallInstructions( uint8_t dieId, uint16_t instructionStartId, diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.h b/src/comm/ccu/tilexr_ccu_driver_adapter.h index 53ed1be3..ce6ce16a 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.h +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.h @@ -17,6 +17,8 @@ namespace TileXR { constexpr uint32_t TILEXR_CCU_U_OP_GET_BASIC_INFO = 11; constexpr uint32_t TILEXR_CCU_U_OP_GET_DIE_WORKING = 15; constexpr uint32_t TILEXR_CCU_U_OP_GET_INSTRUCTION = 201; +constexpr uint32_t TILEXR_CCU_U_OP_GET_XN = 203; +constexpr uint32_t TILEXR_CCU_U_OP_GET_CKE = 204; constexpr uint32_t TILEXR_CCU_U_OP_GET_MISSION_CTX = 208; constexpr uint32_t TILEXR_CCU_U_OP_SET_MSID_TOKEN = 53; constexpr uint32_t TILEXR_CCU_U_OP_SET_TASKKILL = 54; @@ -87,6 +89,18 @@ class TileXRCcuDriverAdapter { void* missionContext, uint32_t missionContextBytes, TileXRCcuDriverAdapterReport* report) const; + int ReadXnRange( + uint8_t dieId, + uint32_t startXnId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; + int ReadCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; int InstallInstructions( uint8_t dieId, uint16_t instructionStartId, @@ -145,6 +159,13 @@ class TileXRCcuDriverAdapter { const TileXRCcuCustomChannelIn& in, TileXRCcuCustomChannelOut* out, TileXRCcuDriverAdapterReport* report) const; + int ReadResourceRange( + uint8_t dieId, + uint32_t opcode, + uint32_t startId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; uint32_t devicePhyId_ = 0; TileXRCcuCustomChannelFn customChannel_ = nullptr; diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index 04590eea..fe0825af 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -124,35 +124,22 @@ void NormalizeVerifiedEndpointRouteJettyWindow(TileXRCcuLowerLayerTransportSnaps return; } - uint16_t firstExplicitStart = 0; uint16_t minExplicitStart = 0; uint32_t explicitEnd = 0; uint32_t explicitStartCount = 0; - bool allExplicitStartsEqual = true; for (const auto& route : snapshot->routes) { if (route.startJettyId == 0) { continue; } - if (firstExplicitStart == 0) { - firstExplicitStart = route.startJettyId; + if (explicitStartCount == 0) { minExplicitStart = route.startJettyId; - } else if (route.startJettyId != firstExplicitStart) { - allExplicitStartsEqual = false; + } else { minExplicitStart = std::min(minExplicitStart, route.startJettyId); } explicitEnd = std::max(explicitEnd, static_cast(route.startJettyId) + 1U); ++explicitStartCount; } - if (explicitStartCount != 0 && allExplicitStartsEqual && snapshot->routes.size() > 1) { - const uint32_t routeCount = static_cast(snapshot->routes.size()); - for (uint32_t i = 0; i < routeCount; ++i) { - snapshot->routes[i].startJettyId = CheckedU16(static_cast(firstExplicitStart) + i); - } - minExplicitStart = firstExplicitStart; - explicitEnd = static_cast(firstExplicitStart) + routeCount; - } - if (minExplicitStart != 0) { snapshot->startJettyId = minExplicitStart; } diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 62db23bd..57984819 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -79,6 +79,8 @@ constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; constexpr const char* kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"; +constexpr const char* kAllToAllSingleRouteBidirectionalEnv = + "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL"; constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; @@ -176,6 +178,11 @@ bool AllToAllLongMissionEnabled() return EnvFlag(kAllToAllLongMissionEnv); } +bool AllToAllSingleRouteBidirectionalEnabled() +{ + return EnvFlag(kAllToAllSingleRouteBidirectionalEnv); +} + bool SyncXnPingSmokeEnabled() { return EnvFlag(kSyncXnPingEnv); @@ -1281,6 +1288,58 @@ void PrintMissionContext( std::cerr << std::dec << std::endl; } +void PrintCcuResourceState( + DirectCcuSmokeContext* context, + uint8_t dieId, + const TileXRDirectCcuPrepareOptions& options, + const char* label) +{ + if (context == nullptr || label == nullptr) { + return; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport report; + int ret = context->session.CreateDriverAdapter(&adapter, &report); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " resourceState adapterRet=" << ret + << " message=\"" << report.message << "\"" << std::endl; + return; + } + + uint64_t localXn[3] {}; + uint64_t remoteXn[3] {}; + uint64_t localWaitCke[3] {}; + uint64_t remoteNotifyCke[3] {}; + const uint32_t localXnStartId = options.xnStartId; + const uint32_t remoteXnStartId = options.remoteXnStartId; + const uint32_t localWaitCkeStartId = options.localWaitCkeStartId; + const uint32_t remoteNotifyCkeStartId = options.remoteNotifyCkeStartId; + const int localXnRet = adapter.ReadXnRange(dieId, localXnStartId, localXn, 3, &report); + const int remoteXnRet = adapter.ReadXnRange(dieId, remoteXnStartId, remoteXn, 3, &report); + const int localCkeRet = adapter.ReadCkeRange( + dieId, localWaitCkeStartId, localWaitCke, 3, &report); + const int remoteCkeRet = adapter.ReadCkeRange( + dieId, remoteNotifyCkeStartId, remoteNotifyCke, 3, &report); + + std::cerr << label << " resourceState" + << " localXnStartId=" << localXnStartId + << " localXnRet=" << localXnRet + << " localXn=0x" << std::hex << localXn[0] << ",0x" << localXn[1] << ",0x" << localXn[2] + << std::dec + << " remoteXnStartId=" << remoteXnStartId + << " remoteXnRet=" << remoteXnRet + << " remoteXn=0x" << std::hex << remoteXn[0] << ",0x" << remoteXn[1] << ",0x" << remoteXn[2] + << std::dec + << " localWaitCkeStartId=" << localWaitCkeStartId + << " localCkeRet=" << localCkeRet + << " localCke=0x" << std::hex << localWaitCke[0] << ",0x" << localWaitCke[1] << ",0x" << localWaitCke[2] + << std::dec + << " remoteNotifyCkeStartId=" << remoteNotifyCkeStartId + << " remoteCkeRet=" << remoteCkeRet + << " remoteCke=0x" << std::hex << remoteNotifyCke[0] << ",0x" << remoteNotifyCke[1] << ",0x" << remoteNotifyCke[2] + << std::dec << std::endl; +} + void PrintInstructionReadback(DirectCcuSmokeContext* context, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) { if (!EnvFlag(kReadbackInstructionsEnv)) { @@ -1582,13 +1641,15 @@ int RunAllToAllCopyPhase( } const int peer = 1 - rank; - const bool active = rank == phase; + const bool singleRouteBidirectional = AllToAllSingleRouteBidirectionalEnabled(); + const bool active = singleRouteBidirectional || rank == phase; const bool submitRequested = EnvFlag(kSubmitEnv); std::cout << "tilexr_ccu_alltoall phase" << " rank=" << rank << " phase=" << phase - << " direction=RemoteToLocal" + << " direction=" << (singleRouteBidirectional ? "LocalToRemote" : "RemoteToLocal") + << " singleRouteBidirectional=" << (singleRouteBidirectional ? 1 : 0) << std::endl; TileXR::TileXRCcuDirectInstallAttempt attempt; @@ -1603,7 +1664,9 @@ int RunAllToAllCopyPhase( reinterpret_cast(alltoall->destination.ptr), alltoall->bytes, static_cast(peer), - TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, + singleRouteBidirectional ? + TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote : + TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, prepared, &installReport); PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); @@ -1681,6 +1744,11 @@ int RunAllToAllCopyPhase( attempt.submitTasks.front(), "tilexr_ccu_alltoall"); } + PrintCcuResourceState( + context, + attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall"); if (missionAtEnd) { std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" << " rank=" << rank @@ -1859,6 +1927,11 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); } + PrintCcuResourceState( + context, + attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall"); if (missionAtEnd) { std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" << " rank=" << rank @@ -1933,7 +2006,8 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi PrintConfig(options, rankSize); int finalRet = alltoall.initRet == ACL_SUCCESS ? 0 : alltoall.initRet; - for (int phase = 0; phase < 2 && finalRet == 0; ++phase) { + const int phaseCount = AllToAllSingleRouteBidirectionalEnabled() ? 1 : 2; + for (int phase = 0; phase < phaseCount && finalRet == 0; ++phase) { finalRet = RunAllToAllCopyPhase(context, rank, rankSize, device, phase, &alltoall); } if (finalRet == 0) { diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 1ea5af0e..4f46f391 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -542,6 +542,18 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC=${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT=${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE=${TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE}") +fi if [ "${TILEXR_CCU_ALLTOALL_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_ALLTOALL_BYTES=${TILEXR_CCU_ALLTOALL_BYTES}") fi diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index f608d7e4..b3a9c2ae 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -83,6 +83,8 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): constexpr uint16_t kSetCkeHeader = 0x0802U; constexpr uint16_t kClearCkeHeader = 0x0804U; + constexpr uint16_t kLoadImdToXnHeader = 0x0003U; + constexpr uint16_t kSyncXnHeader = 0x100dU; uint16_t Slot(const TileXRCcuInstr& instr, uint32_t slot) { @@ -110,14 +112,18 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.localXn = 0x201; spec.remoteXn = 0x202; spec.lengthXn = 0x203; + spec.preSyncLocalAddrXn = 0x209; + spec.preSyncLocalTokenXn = 0x20a; spec.channelId = 0x12; + spec.preSyncChannelId = 0x13; + spec.preSyncTokenChannelId = 0x13; spec.copyCompletionCke = 0x301; spec.preSyncRemoteAddrXn = 0x211; spec.preSyncRemoteTokenXn = 0x212; spec.preSyncLocalWaitCke = 0x302; spec.preSyncRemoteNotifyCke = 0x303; - spec.preSyncTokenLocalWaitCke = 0x304; - spec.preSyncRemoteTokenNotifyCke = 0x305; + spec.preSyncTokenLocalWaitCke = 0x302; + spec.preSyncRemoteTokenNotifyCke = 0x303; spec.postSyncLocalWaitCke = 0x306; spec.postSyncRemoteNotifyCke = 0x307; spec.sourceCke = 0x308; @@ -131,10 +137,10 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): return 1; } const uint32_t expectedBlocks = 64; - const uint32_t expectedInstructions = 3 + expectedBlocks * 7 + 3 + 1; + const uint32_t expectedInstructions = 5 + expectedBlocks * 7 + 3 + 1; if (report.blockCount != expectedBlocks || report.copyInstructionCount != expectedBlocks * 7 || - report.preSyncInstructionCount != 3 || + report.preSyncInstructionCount != 5 || report.postSyncInstructionCount != 3 || report.finishInstructionCount != 1 || report.totalInstructionCount != expectedInstructions || @@ -153,17 +159,50 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): std::cerr << "unexpected block size or message\n"; return 3; } + const uint16_t outputMask = 1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID; + const uint16_t tokenMask = 1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID; + const uint16_t waitMask = outputMask | tokenMask; const uint16_t syncMask = spec.ckeMask; const uint32_t postSyncSetIndex = expectedInstructions - 4; const uint32_t postSyncWaitIndex = expectedInstructions - 2; - if (program.size() < 3 || - Slot(program[2], 0) != kSetCkeHeader || - Slot(program[2], 4) != spec.preSyncLocalWaitCke || - Slot(program[2], 5) != syncMask) { - std::cerr << "PreSync should use the allocated CKE resource mask" - << " header=0x" << std::hex << Slot(program[2], 0) - << " waitCke=0x" << Slot(program[2], 4) - << " waitMask=0x" << Slot(program[2], 5) + if (program.size() < 5 || + Slot(program[0], 0) != kLoadImdToXnHeader || + Slot(program[0], 1) != spec.preSyncLocalAddrXn || + Slot(program[1], 0) != kSyncXnHeader || + Slot(program[1], 1) != spec.preSyncRemoteAddrXn || + Slot(program[1], 2) != spec.preSyncLocalAddrXn || + Slot(program[1], 4) != spec.preSyncChannelId || + Slot(program[1], 5) != spec.preSyncRemoteNotifyCke || + Slot(program[1], 6) != outputMask || + Slot(program[2], 0) != kLoadImdToXnHeader || + Slot(program[2], 1) != spec.preSyncLocalTokenXn || + Slot(program[3], 0) != kSyncXnHeader || + Slot(program[3], 1) != spec.preSyncRemoteTokenXn || + Slot(program[3], 2) != spec.preSyncLocalTokenXn || + Slot(program[3], 4) != spec.preSyncTokenChannelId || + Slot(program[3], 5) != spec.preSyncRemoteTokenNotifyCke || + Slot(program[3], 6) != tokenMask || + Slot(program[4], 0) != kSetCkeHeader || + Slot(program[4], 4) != spec.preSyncLocalWaitCke || + Slot(program[4], 5) != waitMask) { + std::cerr << "PreSync should publish output and token then wait for both" + << " loadHeader=0x" << std::hex << Slot(program[0], 0) + << " loadXn=0x" << Slot(program[0], 1) + << " notifyHeader=0x" << Slot(program[1], 0) + << " remoteXn=0x" << Slot(program[1], 1) + << " localXn=0x" << Slot(program[1], 2) + << " channel=0x" << Slot(program[1], 4) + << " remoteCke=0x" << Slot(program[1], 5) + << " notifyMask=0x" << Slot(program[1], 6) + << " tokenNotifyHeader=0x" << Slot(program[3], 0) + << " tokenRemoteXn=0x" << Slot(program[3], 1) + << " tokenLocalXn=0x" << Slot(program[3], 2) + << " tokenChannel=0x" << Slot(program[3], 4) + << " tokenRemoteCke=0x" << Slot(program[3], 5) + << " tokenNotifyMask=0x" << Slot(program[3], 6) + << " waitHeader=0x" << Slot(program[4], 0) + << " waitCke=0x" << Slot(program[4], 4) + << " waitMask=0x" << Slot(program[4], 5) << std::dec << "\n"; return 4; } @@ -176,10 +215,36 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): << " postSetMask=0x" << std::hex << Slot(program[postSyncSetIndex], 3) << " postWaitCke=0x" << Slot(program[postSyncWaitIndex], 4) << " postWaitMask=0x" << Slot(program[postSyncWaitIndex], 5) - << " syncMask=0x" << syncMask + << " syncMask=0x" << spec.ckeMask << std::dec << "\n"; return 5; } + + spec.preSyncNotify = false; + spec.postSyncNotify = false; + spec.emitFinish = false; + ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS || + report.preSyncInstructionCount != 0 || + report.postSyncInstructionCount != 0 || + report.finishInstructionCount != 0 || + report.totalInstructionCount != expectedBlocks * 7 || + program.size() != expectedBlocks * 7) { + std::cerr << "copy-only diagnostic program has unexpected counts\n"; + return 6; + } + + spec.preSyncNotify = true; + spec.preSyncWait = false; + spec.sourceCke = 0; + ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS || + report.preSyncInstructionCount != 4 || + report.totalInstructionCount != expectedBlocks * 7 + 4 || + program.size() != expectedBlocks * 7 + 4) { + std::cerr << "notify-only PreSync program has unexpected counts\n"; + return 7; + } return 0; } ''' @@ -276,7 +341,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): << " rank1=" << report1.message << "\n"; return 1; } - const uint32_t expectedInstructions = 3 + 64 * 7 + 3 + 1; + const uint32_t expectedInstructions = 5 + 64 * 7 + 3 + 1; if (report0.totalInstructionCount != expectedInstructions || report1.totalInstructionCount != expectedInstructions || rank0.size() != expectedInstructions || @@ -294,7 +359,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): } const size_t rank0FirstCopy = FirstCopyIndex(rank0); const size_t rank1FirstCopy = FirstCopyIndex(rank1); - if (rank0FirstCopy != 8 || rank1FirstCopy != 8) { + if (rank0FirstCopy != 10 || rank1FirstCopy != 10) { std::cerr << "copy region should start at the same instruction after the single PreSync" << " rank0FirstCopy=" << rank0FirstCopy << " rank1FirstCopy=" << rank1FirstCopy << "\n"; @@ -378,7 +443,7 @@ def test_builder_rejects_invalid_slice_configuration(self): } spec = ValidSpec(); - spec.remoteSendToken = 0; + spec.remoteRecvToken = 0; if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || report.message.find("token") == std::string::npos) { @@ -419,8 +484,8 @@ def test_builder_is_wired_and_has_no_hccl_dependency_surface(self): self.assertNotIn("tokenLocalWaitCke", source) self.assertIn("PreSyncSignalMask", source) self.assertIn("PostSyncSignalMask", source) - self.assertIn("return spec.ckeMask;", source) - self.assertNotIn("1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID", source) + self.assertIn("1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID", source) + self.assertIn("1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID", source) self.assertNotIn("1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID", source) self.assertIn("post.clearWait = true;", source) self.assertIn("TILEXR_CCU_ALLTOALL_SIGNAL_MASK", header) diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index 20956afe..1da3bb8b 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1750,20 +1750,36 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 3U + 64U * 7U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 3U + 64U * 7U", planner) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 5U + 64U * 7U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 5U + 64U * 7U", planner) self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) - self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncCke", source) + self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn", source) self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) self.assertIn("const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]", source) self.assertIn("const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]", source) self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(\n 0U,", planner) - self.assertIn("preSyncRemoteAddrXn = preResource.remoteXn", source) - self.assertIn("preSyncRemoteTokenXn = postResource.remoteXn", source) - self.assertIn("preSyncTokenChannelId = postResource.channelId", source) - self.assertNotIn("preSyncTokenChannelId = preResource.channelId", source) - self.assertIn("preSyncRemoteNotifyCke = preResource.notifyCke", source) - self.assertIn("preSyncRemoteTokenNotifyCke = postResource.notifyCke", source) + self.assertIn( + "static_cast(peerResources.localWaitCkeStartId) +\n peerLocalWaitCkeOffset", + planner, + ) + self.assertNotIn( + "peerResources.remoteNotifyCkeStartId) +\n peerLocalWaitCkeOffset", planner + ) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.localXn : preResource.localXn", source + ) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.localXn : preSyncRemoteAddrXn", source + ) + self.assertIn( + "preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn", source + ) + self.assertIn("preSyncTokenChannelId = preResource.channelId", source) + self.assertNotIn("preSyncTokenChannelId = postResource.channelId", source) + self.assertIn( + "preSyncOnCopyRoute ? attempt->allocation.remoteNotifyCke.startId : preResource.notifyCke", source + ) + self.assertIn("preSyncRemoteTokenNotifyCke = preResource.notifyCke", source) self.assertIn("preSyncTokenLocalWaitCke =", source) self.assertIn("preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke", source) self.assertIn("alltoallSpec.copyCompletionCke =", source) @@ -1771,11 +1787,26 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): self.assertIn("postSyncRemoteNotifyCke = postResource.notifyCke", source) self.assertNotIn("postSyncRemoteNotifyCke = preResource.notifyCke", source) self.assertIn("postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke", source) - self.assertIn("preSyncChannelId = preResource.channelId", source) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId", source + ) self.assertIn("copyChannelId = copyResource.channelId", source) self.assertIn("postSyncChannelId = postResource.channelId", source) self.assertNotIn("postSyncChannelId = preResource.channelId", source) self.assertIn("postSyncNotify = false", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC", source) + self.assertIn("alltoallSpec.preSyncNotify", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT", source) + self.assertIn("alltoallSpec.preSyncWait", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE", source) + self.assertIn("preSyncOnCopyRoute", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_PEER_LOCAL_XN", source) + self.assertIn( + "preSyncPeerLocalXn ? preResource.localXn : preResource.remoteXn", source + ) + self.assertIn( + "preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn", source + ) self.assertIn("postSyncWait = false", source) self.assertIn("emitFinish = false", source) self.assertNotIn("postSyncNotify = true", source) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index e7a01a35..e7f79d1e 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -656,11 +656,16 @@ def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): self.assertIn('kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"', source) self.assertIn('kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"', source) + self.assertIn("kAllToAllSingleRouteBidirectionalEnv", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL", source) self.assertIn('kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"', source) self.assertIn('kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"', source) self.assertIn("struct AllToAllState", source) self.assertIn("AllToAllSmokeEnabled", source) self.assertIn("AllToAllLongMissionEnabled", source) + self.assertIn("AllToAllSingleRouteBidirectionalEnabled", source) + self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) + self.assertIn("singleRouteBidirectional", source) self.assertIn("InitAllToAllState", source) self.assertIn("RunAllToAllCopyPhase", source) self.assertIn("RunAllToAllLongMissionSmokeForRank", source) @@ -689,6 +694,17 @@ def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): self.assertIn("tilexr_ccu_sync_xn_ping submit", source) self.assertIn("tilexr_ccu_sync_xn_ping timing", source) self.assertIn("aclrtSynchronizeStreamWithTimeout", source) + + def test_alltoall_timeout_prints_xn_and_cke_readback(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrintCcuResourceState", source) + self.assertIn("adapter.ReadXnRange", source) + self.assertIn("adapter.ReadCkeRange", source) + self.assertIn("localXnStartId", source) + self.assertIn("remoteXnStartId", source) + self.assertIn("localWaitCkeStartId", source) + self.assertIn("remoteNotifyCkeStartId", source) self.assertIn("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", source) self.assertIn("tilexr_ccu_sync_xn_ping result passed=1", source) diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py index d6795f52..ec65c46f 100644 --- a/tests/ccu/test_tilexr_ccu_driver_adapter.py +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -488,6 +488,70 @@ def test_adapter_reads_each_instruction_from_its_own_data_array_slot(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_adapter_reads_xn_and_cke_ranges_from_driver_slots(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + #include + + using namespace TileXR; + + struct FakeState { + int calls = 0; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + if (in.data.dataInfo.udieIdx != 1 || in.offsetStartIdx != (in.op == 203 ? 2362 : 332) || + in.data.dataInfo.dataArraySize != 3 || in.data.dataInfo.dataLen != 24) { + return -1; + } + for (uint32_t i = 0; i < 3; ++i) { + const uint64_t value = (static_cast(in.op) << 32U) | i; + std::memcpy(&out->data.dataInfo.dataArray[i], &value, sizeof(value)); + } + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + return 1; + } + uint64_t xn[3] {}; + uint64_t cke[3] {}; + if (adapter.ReadXnRange(1, 2362, xn, 3, &report) != TILEXR_SUCCESS || + adapter.ReadCkeRange(1, 332, cke, 3, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 2; + } + if (state.calls != 2 || xn[0] != (203ULL << 32U) || xn[2] != ((203ULL << 32U) | 2U) || + cke[0] != (204ULL << 32U) || cke[2] != ((204ULL << 32U) | 2U)) { + std::cerr << "resource readback mismatch\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_adapter_set_instruction_trailer_wire_word_is_offset_then_opcode(self): code = textwrap.dedent( r''' diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 969d9be2..0efe9101 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -1563,7 +1563,7 @@ def test_overlay_verified_endpoint_routes_updates_only_matching_endpoint_fields( self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) - def test_overlay_verified_endpoint_route_expands_pfe_window_for_multi_route_snapshot(self): + def test_overlay_verified_endpoint_route_reuses_shared_jetty_for_multi_route_snapshot(self): code = textwrap.dedent( r''' #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" @@ -1661,9 +1661,9 @@ def test_overlay_verified_endpoint_route_expands_pfe_window_for_multi_route_snap return 5; } if (DecodeChannelStartJettyId(plan.channels[0].ctx) != 0x400 || - DecodeChannelStartJettyId(plan.channels[1].ctx) != 0x401 || - DecodeChannelStartJettyId(plan.channels[2].ctx) != 0x402) { - std::cerr << "multi-route channel jetty IDs were not expanded from verified start\n"; + DecodeChannelStartJettyId(plan.channels[1].ctx) != 0x400 || + DecodeChannelStartJettyId(plan.channels[2].ctx) != 0x400) { + std::cerr << "multi-route channels did not reuse the verified endpoint jetty\n"; return 6; } return 0; From b435f2a705e87dfa02f41adca4ce15ac74f80aeb Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 11:13:42 +0800 Subject: [PATCH 26/41] feat(ccu): support repeated direct all-to-all submissions --- .../2026-07-20-ccu-alltoall-loop-reuse.md | 112 +++++++ ...26-07-20-ccu-alltoall-loop-reuse-design.md | 42 +++ src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 54 +++- src/comm/ccu/tilexr_ccu_alltoall_program.h | 5 + .../ccu/tilexr_ccu_collective_planner.cpp | 2 +- .../ccu/tilexr_ccu_direct_orchestrator.cpp | 6 +- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 275 ++++++++++++++---- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 10 +- tests/ccu/test_tilexr_ccu_alltoall_program.py | 83 +++--- .../test_tilexr_ccu_direct_orchestrator.py | 8 +- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 36 ++- .../test_tilexr_ccu_direct_smoke_runner.py | 2 + 12 files changed, 530 insertions(+), 105 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md create mode 100644 docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md diff --git a/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md b/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md new file mode 100644 index 00000000..358a8d8e --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md @@ -0,0 +1,112 @@ +# CCU AlltoAll Loop Reuse Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Validate ten consecutive two-rank CCU AlltoAll submissions using one prepared mission and loop-specific device synchronization markers. + +**Architecture:** Extend PreSync with a marker loaded from SQE argument zero and synchronized through a dedicated XN with mask `0x1`; address and token retain masks `0x2` and `0x4`, and the receiver waits for `0x7`. The smoke probe prepares once, updates buffers and task arg zero per loop, submits repeatedly with phase-specific host gates, then validates both the peer marker and full destination data. + +**Tech Stack:** C++14, CCU microcode encoders, Python `unittest`, Bash hardware runner, ACL runtime. + +--- + +### Task 1: Encode The Loop Marker In PreSync + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.h` +- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.cpp` +- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` +- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` +- Test: `tests/ccu/test_tilexr_ccu_alltoall_program.py` +- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` + +- [ ] **Step 1: Write failing generator tests** + +Assert that marker-enabled PreSync emits `LoadSqeArgsToX(markerLocalXn, 0)`, then `SyncXn(markerRemoteXn, markerLocalXn, preChannel, notifyCke, 0x1)`, followed by the existing address/token notifications and a `SetCke` wait mask of `0x7`. + +- [ ] **Step 2: Run the focused tests and verify RED** + +Run: + +```bash +python3 -m unittest tests.ccu.test_tilexr_ccu_alltoall_program tests.ccu.test_tilexr_ccu_direct_orchestrator +``` + +Expected: failure because the program spec has no marker fields and the instruction count remains `5 + 64 * 7`. + +- [ ] **Step 3: Implement marker-enabled PreSync** + +Add `preSyncLocalMarkerXn`, `preSyncRemoteMarkerXn`, `preSyncMarkerArgIndex`, and `preSyncMarkerEnabled` to the program spec. Encode the marker load with `TileXRCcuEncodeLoadSqeArgsToX`, notify mask `0x1`, and wait mask `0x7`. Increase the AlltoAll instruction capacity to `7 + 64 * 7` and map the copy resource XNs to the marker pair. + +- [ ] **Step 4: Run focused tests and verify GREEN** + +Run the command from Step 2. Expected: all tests pass. + +### Task 2: Repeat One Prepared Mission With Per-Loop Data + +**Files:** +- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` +- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` +- Test: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` +- Test: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` + +- [ ] **Step 1: Write failing loop-control tests** + +Require `TILEXR_CCU_ALLTOALL_LOOP_COUNT`, a default of one, preparation outside the loop, task argument zero updated per loop, `phase=loopIndex` ready/done gates, source/destination refresh, peer marker readback, and loop-indexed result output. + +- [ ] **Step 2: Run focused smoke tests and verify RED** + +Run: + +```bash +python3 -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner +``` + +Expected: failure because no loop-count environment variable or repeated submission exists. + +- [ ] **Step 3: Implement repeated submission** + +Add a validated loop count, rank-and-loop pattern generation, destination reset, marker encoding, prepared task arg mutation, phase-specific gates, marker XN readback, and per-loop result reporting. Keep allocation, registration, plan preparation, and installation outside the loop. Forward the environment variable from the runner. + +- [ ] **Step 4: Run focused smoke tests and verify GREEN** + +Run the command from Step 2. Expected: all tests pass. + +### Task 3: Regression And Hardware Verification + +**Files:** +- Verify only; no planned production edits. + +- [ ] **Step 1: Run affected CCU tests** + +```bash +python3 -m unittest \ + tests.ccu.test_tilexr_ccu_alltoall_program \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_direct_smoke_runner \ + tests.ccu.test_tilexr_ccu_lower_layer_plan_builder +``` + +Expected: zero failures. + +- [ ] **Step 2: Build `tile-comm` and the smoke probe** + +```bash +source scripts/common_env.sh +cmake --build build_ccu_direct --target tile-comm ccu_tilexr_direct_smoke_probe -j2 +``` + +Expected: both targets build successfully. + +- [ ] **Step 3: Wait for NPU 6 and 7 to become idle** + +Poll `npu-smi info` through `tests/ccu/ccu_npu_smi_busy_guard.py --devices 6,7` every 30 seconds. Do not terminate unknown jobs. + +- [ ] **Step 4: Run loop-count ten hardware validation** + +Set `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10` with the established two-rank long-mission configuration. Expected: each rank prints ten successful loop results, every peer marker matches its loop, and every loop reports `mismatches=0`. + +- [ ] **Step 5: Inspect the final diff** + +Run `git diff --check` and confirm unrelated untracked files remain untouched. diff --git a/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md b/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md new file mode 100644 index 00000000..88742de3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md @@ -0,0 +1,42 @@ +# CCU AlltoAll Loop Reuse Design + +## Goal + +Validate ten consecutive two-rank bidirectional CCU AlltoAll submissions while reusing one communicator, one installed mission, and the same QP, jetty, CKE, XN, and registered-memory resources. + +## Round Protocol + +The smoke probe prepares the long AlltoAll mission once. For each loop index from zero through nine, both ranks: + +1. Fill the source buffer with a rank-and-loop-specific pattern. +2. Fill the destination buffer with a sentinel value. +3. Set SQE argument zero to a rank-and-loop-specific 64-bit marker. +4. Enter a loop-specific host ready gate using `phase=loopIndex`. +5. Submit the same prepared task and synchronize its stream. +6. Read the peer marker XN and require the exact marker for the current peer and loop. +7. Read the complete destination buffer and compare it with the current peer pattern. +8. Enter a loop-specific done gate with the local validation result. + +Any failure terminates the loop and reports the loop index, mission context, XN/CKE state, marker value, and mismatch details. + +## Device Marker + +PreSync carries three variables on the same channel: + +- peer receive address with CKE mask `0x2`; +- peer receive token with CKE mask `0x4`; +- current SQE loop marker with CKE mask `0x1`. + +The receiver waits for mask `0x7`. The marker is loaded with `LoadSqeArgsToX`, so the installed instruction sequence remains unchanged while each submission supplies a distinct value. The marker uses a fixed magic prefix plus rank and loop index, allowing host readback to reject a signal from another loop. + +The CKE bits remain presence flags and are cleared by the existing wait instruction. The loop-specific XN marker supplies the generation identity that the existing CKE protocol lacks. + +## Configuration + +`TILEXR_CCU_ALLTOALL_LOOP_COUNT` controls repeated submissions and defaults to one. Values outside `1..1024` fail parameter validation. The runner forwards the variable to both ranks. + +## Verification + +- Unit tests verify the marker instruction ordering, mask `0x7`, instruction capacity, default loop count, runner forwarding, and loop-specific phase gates. +- The remote hardware test uses NPU 6 and 7 with loop count ten. +- Success requires ten loop results per rank (twenty total), exact peer markers, and zero mismatches in every loop. diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 7a48a691..16a23e26 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -90,6 +90,11 @@ int ValidateSpec( (spec.postSyncNotify && spec.sourceCke == 0) || spec.ckeMask == 0) { return Fail(program, report, "missing direct CCU alltoall CKE/channel resource"); } + if (spec.preSyncMarkerEnabled && + (spec.preSyncLocalMarkerXn == 0 || spec.preSyncRemoteMarkerXn == 0 || + spec.preSyncMarkerArgIndex >= TILEXR_CCU_SQE_ARGS_LEN)) { + return Fail(program, report, "missing direct CCU alltoall loop marker resource"); + } return TILEXR_SUCCESS; } @@ -195,6 +200,36 @@ int AppendRemoteNotify( return TILEXR_SUCCESS; } +int AppendRemoteMarkerNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadSqeArgsToX( + spec.preSyncLocalMarkerXn, + spec.preSyncMarkerArgIndex, + &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PreSync loop marker load"); + } + program->push_back(instr); + + TileXRCcuSyncXnSpec notify; + notify.remoteXn = spec.preSyncRemoteMarkerXn; + notify.localXn = spec.preSyncLocalMarkerXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK; + notify.clearWait = true; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PreSync loop marker notify"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + int AppendPreSyncPhase( uint16_t remoteNotifyCke, uint16_t localWaitCke, @@ -205,7 +240,8 @@ int AppendPreSyncPhase( { const uint16_t outputMask = PreSyncSignalMask(spec); const uint16_t tokenMask = PreSyncTokenMask(spec); - const uint16_t waitMask = static_cast(outputMask | tokenMask); + const uint16_t markerMask = spec.preSyncMarkerEnabled ? TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK : 0U; + const uint16_t waitMask = static_cast(markerMask | outputMask | tokenMask); const uint16_t localOutputXn = spec.preSyncLocalAddrXn == 0 ? spec.localXn : spec.preSyncLocalAddrXn; const uint16_t localTokenXn = @@ -214,6 +250,15 @@ int AppendPreSyncPhase( spec.preSyncTokenChannelId == 0 ? outputChannelId : spec.preSyncTokenChannelId; const uint16_t tokenNotifyCke = spec.preSyncRemoteTokenNotifyCke == 0 ? remoteNotifyCke : spec.preSyncRemoteTokenNotifyCke; + if (spec.preSyncMarkerEnabled && + AppendRemoteMarkerNotify( + remoteNotifyCke, + outputChannelId, + spec, + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } if (AppendRemoteNotify( remoteNotifyCke, outputChannelId, @@ -341,7 +386,9 @@ void FillReport( return; } const uint32_t bytesPerBlock = spec.memorySliceBytes * spec.memSlicePerBlock; - report->preSyncInstructionCount = spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) : 0U; + const uint32_t markerInstructionCount = spec.preSyncMarkerEnabled ? 2U : 0U; + report->preSyncInstructionCount = + spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) + markerInstructionCount : 0U; report->blockCount = static_cast(spec.bytes / bytesPerBlock); report->bytesPerBlock = bytesPerBlock; report->copyInstructionCount = report->blockCount * 7U; @@ -371,8 +418,9 @@ int TileXRCcuBuildAllToAll2RankProgram( const uint32_t blockCount = static_cast(spec.bytes / bytesPerBlock); const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; + const uint32_t markerInstructionCount = spec.preSyncMarkerEnabled ? 2U : 0U; program->reserve( - (spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) : 0U) + blockCount * 7U + + (spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) + markerInstructionCount : 0U) + blockCount * 7U + (!spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U)) + (spec.emitFinish ? 1U : 0U)); diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index 5d06471c..484ff30a 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -20,6 +20,7 @@ constexpr uint32_t TILEXR_CCU_ALLTOALL_BLOCK_BYTES = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES * TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; constexpr uint16_t TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_TOKEN_XN_ID = 2U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_ID = 3U; constexpr uint16_t TILEXR_CCU_ALLTOALL_SIGNAL_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK = 1U; @@ -45,6 +46,9 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t lengthXn = 0; uint16_t preSyncLocalAddrXn = 0; uint16_t preSyncLocalTokenXn = 0; + uint16_t preSyncLocalMarkerXn = 0; + uint16_t preSyncRemoteMarkerXn = 0; + uint16_t preSyncMarkerArgIndex = 0; uint16_t channelId = 0; uint16_t preSyncChannelId = 0; uint16_t preSyncTokenChannelId = 0; @@ -63,6 +67,7 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t ckeMask = 1; bool preSyncNotify = true; bool preSyncWait = true; + bool preSyncMarkerEnabled = false; bool postSyncNotify = true; bool postSyncWait = true; bool emitFinish = true; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 4243b5b0..f4659250 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -26,7 +26,7 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 5U + 64U * 7U; + 7U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index ea9fb2f4..d13fc152 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -38,7 +38,7 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 5U + 64U * 7U; + 7U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; @@ -955,6 +955,10 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.preSyncLocalAddrXn = preSyncOnCopyRoute ? copyResource.localXn : preResource.localXn; alltoallSpec.preSyncLocalTokenXn = postResource.localXn; + alltoallSpec.preSyncLocalMarkerXn = copyResource.localXn; + alltoallSpec.preSyncRemoteMarkerXn = copyResource.remoteXn; + alltoallSpec.preSyncMarkerArgIndex = 0; + alltoallSpec.preSyncMarkerEnabled = true; alltoallSpec.channelId = copyResource.channelId; alltoallSpec.preSyncChannelId = preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId; diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 57984819..fd4e9b8b 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -83,6 +83,7 @@ constexpr const char* kAllToAllSingleRouteBidirectionalEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL"; constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; +constexpr const char* kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"; constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; @@ -575,6 +576,38 @@ int AllToAllMemSlicePerLoopFromEnv() return EnvInt(kAllToAllMemSlicePerLoopEnv, 8); } +int AllToAllLoopCountFromEnv() +{ + const char* value = std::getenv(kAllToAllLoopCountEnv); + if (value == nullptr || value[0] == '\0') { + return 1; + } + char* end = nullptr; + const long parsed = std::strtol(value, &end, 10); + if (end == value || *end != '\0' || parsed < 1 || parsed > 1024) { + return 0; + } + return static_cast(parsed); +} + +uint64_t BuildAllToAllLoopMarker(int rank, int loopIndex) +{ + return 0x4343554c00000000ULL | + (static_cast(rank & 0xff) << 16U) | + static_cast(loopIndex & 0xffff); +} + +std::vector BuildAllToAllLoopPattern(int rank, int loopIndex, size_t bytes) +{ + std::vector pattern(bytes); + for (size_t i = 0; i < bytes; ++i) { + pattern[i] = static_cast( + (static_cast(rank + 1) * 17U + + static_cast(loopIndex + 1) * 29U + i * 13U) & 0xffU); + } + return pattern; +} + int InitAllToAllState(int rank, int peer, AllToAllState* state) { if (state == nullptr) { @@ -620,6 +653,46 @@ int InitAllToAllState(int rank, int peer, AllToAllState* state) return ret; } +int ResetAllToAllStateForLoop(int rank, int peer, int loopIndex, AllToAllState* state) +{ + if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || + state->bytes == 0 || loopIndex < 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->expected = BuildAllToAllLoopPattern(peer, loopIndex, state->bytes); + state->observed.assign(state->bytes, 0); + state->readRet = ACL_SUCCESS; + state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + state->passed = false; + + const std::vector source = BuildAllToAllLoopPattern(rank, loopIndex, state->bytes); + const std::vector destination( + state->bytes, + static_cast(0xa5U ^ static_cast(loopIndex))); + int ret = aclrtMemcpy( + state->source.ptr, + state->bytes, + source.data(), + source.size(), + ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + return ret; + } + return aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + destination.size(), + ACL_MEMCPY_HOST_TO_DEVICE); +} + int CheckAllToAllState(AllToAllState* state) { if (state == nullptr || state->destination.ptr == nullptr || state->bytes == 0) { @@ -1340,6 +1413,40 @@ void PrintCcuResourceState( << std::dec << std::endl; } +int ReadAndValidatePeerLoopMarker( + DirectCcuSmokeContext* context, + uint8_t dieId, + uint32_t markerXnId, + int rank, + int loopIndex, + uint64_t expectedPeerLoopMarker) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport report; + int readRet = context->session.CreateDriverAdapter(&adapter, &report); + uint64_t peerLoopMarker = 0; + if (readRet == TileXR::TILEXR_SUCCESS) { + readRet = adapter.ReadXnRange(dieId, markerXnId, &peerLoopMarker, 1, &report); + } + const bool matched = readRet == TileXR::TILEXR_SUCCESS && + peerLoopMarker == expectedPeerLoopMarker; + std::cout << "tilexr_ccu_alltoall peerLoopMarker" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " xnId=" << markerXnId + << " readRet=" << readRet + << " observed=0x" << std::hex << peerLoopMarker + << " expected=0x" << expectedPeerLoopMarker + << std::dec + << " matched=" << (matched ? 1 : 0) + << " message=\"" << report.message << "\"" + << std::endl; + return matched ? TileXR::TILEXR_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + void PrintInstructionReadback(DirectCcuSmokeContext* context, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) { if (!EnvFlag(kReadbackInstructionsEnv)) { @@ -1703,6 +1810,7 @@ int RunAllToAllCopyPhase( << std::endl; } else if (submitRequested && !installReport.submitReady) { std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + finalRet = 6; } else if (submitRequested) { aclrtStream stream = nullptr; int streamRet = aclrtCreateStream(&stream); @@ -1778,11 +1886,12 @@ int RunAllToAllCopyPhase( return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; } -void PrintAllToAllResultAndMaybeFastExit(int rank, int finalRet, const AllToAllState& alltoall) +void PrintAllToAllResult(int rank, int loopIndex, int finalRet, const AllToAllState& alltoall) { if (finalRet == 0) { std::cout << "tilexr_ccu_alltoall result passed=1" << " rank=" << rank + << " loopIndex=" << loopIndex << " ret=" << finalRet << " readRet=" << alltoall.readRet << " mismatches=" << alltoall.mismatchCount @@ -1790,6 +1899,7 @@ void PrintAllToAllResultAndMaybeFastExit(int rank, int finalRet, const AllToAllS } else { std::cout << "tilexr_ccu_alltoall result passed=0" << " rank=" << rank + << " loopIndex=" << loopIndex << " ret=" << finalRet << " readRet=" << alltoall.readRet << " mismatches=" << alltoall.mismatchCount @@ -1803,6 +1913,10 @@ void PrintAllToAllResultAndMaybeFastExit(int rank, int finalRet, const AllToAllS << " lastMismatchedBlock=" << alltoall.lastMismatchedBlock << std::endl; } +} + +void MaybeFastExitAfterAllToAllRun(int finalRet) +{ if (ShouldFastExitAfterRun()) { std::cout << "tilexr_ccu_alltoall fastExitAfterRun=1" << " ret=" << finalRet @@ -1826,14 +1940,18 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, } const int peer = 1 - rank; + const int loopCount = AllToAllLoopCountFromEnv(); AllToAllState alltoall; alltoall.initRet = InitAllToAllState(rank, peer, &alltoall); + if (loopCount == 0 && alltoall.initRet == ACL_SUCCESS) { + alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); options.syncResourceCount = 3; - options.sqeArgCount = 0; + options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { - options.syncInstructionCount = 3 + 64 * 7; + options.syncInstructionCount = 5 + 64 * 7; } if (options.gsaStartId == 0) { options.gsaStartId = 1; @@ -1843,6 +1961,7 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, << " rank=" << rank << " peer=" << peer << " bytes=" << alltoall.bytes + << " loopCount=" << loopCount << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() << " blockCount=64" << " longMission=1" @@ -1872,18 +1991,16 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, int finalRet = 0; const bool submitRequested = EnvFlag(kSubmitEnv); - const bool collectiveSubmitReady = submitRequested ? - WaitForCollectiveSubmitReadiness( - rank, - rankSize, - prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : - false; if (prepareRet != TileXR::TILEXR_SUCCESS) { finalRet = 6; - } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { - std::cout << "tilexr_ccu_alltoall submit skipped reason=\"collective submitReady gate did not pass\"" - << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + } else if (attempt.submitTasks.empty() || + attempt.submitTasks.front().argSize != TILEXR_DIRECT_CCU_SQE_ARGS_LEN) { + std::cerr << "tilexr_ccu_alltoall invalidPreparedTask" + << " rank=" << rank + << " taskCount=" << attempt.submitTasks.size() + << " argSize=" << (attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().argSize) << std::endl; + finalRet = 6; } else if (submitRequested && !installReport.submitReady) { std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; } else if (submitRequested) { @@ -1894,58 +2011,90 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, finalRet = 7; } else { bool skipStreamDestroy = false; - TileXRDirectCcuSubmitReport submitReport; - const auto submitBegin = std::chrono::steady_clock::now(); - const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); - const auto submitEnd = std::chrono::steady_clock::now(); - PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); - const auto syncBegin = std::chrono::steady_clock::now(); const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); - TraceLifecycle("before alltoall long mission aclrtSynchronizeStreamWithTimeout"); - const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); - TraceLifecycle("after alltoall long mission aclrtSynchronizeStreamWithTimeout"); - const auto syncEnd = std::chrono::steady_clock::now(); - std::cout << "tilexr_ccu_alltoall timing" - << " rank=" << rank - << " longMission=1" - << " submitRet=" << submitRet - << " syncRet=" << syncRet - << " syncTimeoutMs=" << syncTimeoutMs - << " submitMs=" - << std::chrono::duration_cast(submitEnd - submitBegin).count() - << " syncMs=" - << std::chrono::duration_cast(syncEnd - syncBegin).count() - << std::endl; - if (syncRet != ACL_SUCCESS) { - std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet - << " timeoutMs=" << syncTimeoutMs << std::endl; - bool missionAtEnd = false; - if (!attempt.submitTasks.empty()) { - PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); - missionAtEnd = ReadMissionContextAtEnd( - context, - attempt.submitTasks.front(), - "tilexr_ccu_alltoall"); + for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { + const int resetRet = ResetAllToAllStateForLoop(rank, peer, loopIndex, &alltoall); + const uint64_t localLoopMarker = BuildAllToAllLoopMarker(rank, loopIndex); + attempt.submitTasks.front().args[0] = localLoopMarker; + const bool collectiveSubmitReady = WaitForCollectiveSubmitReadiness( + rank, + rankSize, + resetRet == ACL_SUCCESS && installReport.submitReady, + loopIndex); + if (resetRet != ACL_SUCCESS) { + finalRet = 14; + } else if (!collectiveSubmitReady) { + finalRet = 13; } - PrintCcuResourceState( - context, - attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, - options, - "tilexr_ccu_alltoall"); - if (missionAtEnd) { - std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" + + if (finalRet == 0) { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + TraceLifecycle("before alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_alltoall timing" << " rank=" << rank - << " reason=\"continuing to device buffer validation\"" + << " loopIndex=" << loopIndex + << " longMission=1" + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() << std::endl; - skipStreamDestroy = true; - } else { - finalRet = 8; + if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } else if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + finalRet = 8; + skipStreamDestroy = true; + } + } + + if (finalRet == 0) { + const int markerRet = ReadAndValidatePeerLoopMarker( + context, + attempt.submitTasks.front().dieId, + options.remoteXnStartId, + rank, + loopIndex, + BuildAllToAllLoopMarker(peer, loopIndex)); + if (markerRet != TileXR::TILEXR_SUCCESS) { + finalRet = 15; + } + } + if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { + finalRet = 14; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex) && finalRet == 0) { + finalRet = 13; + } + PrintAllToAllResult(rank, loopIndex, finalRet, alltoall); + if (finalRet != 0) { + std::cerr << "tilexr_ccu_alltoall loopFailure" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet << std::endl; + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + PrintCcuResourceState( + context, + attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall"); + break; } - } else if (submitRet != TileXR::TILEXR_SUCCESS) { - finalRet = 9; - } - if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { - finalRet = 13; } if (skipStreamDestroy) { std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" @@ -1958,13 +2107,14 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, } } - if (finalRet == 0) { + if (!submitRequested && finalRet == 0) { const int checkRet = CheckAllToAllState(&alltoall); if (checkRet != ACL_SUCCESS) { finalRet = 14; } + PrintAllToAllResult(rank, -1, finalRet, alltoall); } - PrintAllToAllResultAndMaybeFastExit(rank, finalRet, alltoall); + MaybeFastExitAfterAllToAllRun(finalRet); const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; } @@ -2017,7 +2167,8 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi } } - PrintAllToAllResultAndMaybeFastExit(rank, finalRet, alltoall); + PrintAllToAllResult(rank, -1, finalRet, alltoall); + MaybeFastExitAfterAllToAllRun(finalRet); return finalRet; } diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 4f46f391..ef704493 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -213,14 +213,16 @@ apply_alltoall_defaults() export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" + export TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" - export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453}" else + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" fi @@ -269,6 +271,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION \ TILEXR_CCU_ALLTOALL_BYTES \ TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ + TILEXR_CCU_ALLTOALL_LOOP_COUNT \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do diagnostic_value="${!diagnostic_var:-}" @@ -560,6 +563,9 @@ fi if [ "${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-}" != "" ]; then common_env+=("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP}") fi +if [ "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN}") fi diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index b3a9c2ae..45a77265 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -83,6 +83,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): constexpr uint16_t kSetCkeHeader = 0x0802U; constexpr uint16_t kClearCkeHeader = 0x0804U; + constexpr uint16_t kLoadSqeArgsToXnHeader = 0x0001U; constexpr uint16_t kLoadImdToXnHeader = 0x0003U; constexpr uint16_t kSyncXnHeader = 0x100dU; @@ -114,6 +115,10 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.lengthXn = 0x203; spec.preSyncLocalAddrXn = 0x209; spec.preSyncLocalTokenXn = 0x20a; + spec.preSyncLocalMarkerXn = 0x208; + spec.preSyncRemoteMarkerXn = 0x210; + spec.preSyncMarkerArgIndex = 0; + spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; spec.preSyncChannelId = 0x13; spec.preSyncTokenChannelId = 0x13; @@ -137,10 +142,10 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): return 1; } const uint32_t expectedBlocks = 64; - const uint32_t expectedInstructions = 5 + expectedBlocks * 7 + 3 + 1; + const uint32_t expectedInstructions = 7 + expectedBlocks * 7 + 3 + 1; if (report.blockCount != expectedBlocks || report.copyInstructionCount != expectedBlocks * 7 || - report.preSyncInstructionCount != 5 || + report.preSyncInstructionCount != 7 || report.postSyncInstructionCount != 3 || report.finishInstructionCount != 1 || report.totalInstructionCount != expectedInstructions || @@ -159,33 +164,41 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): std::cerr << "unexpected block size or message\n"; return 3; } + const uint16_t markerMask = 1U; const uint16_t outputMask = 1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID; const uint16_t tokenMask = 1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID; - const uint16_t waitMask = outputMask | tokenMask; + const uint16_t waitMask = markerMask | outputMask | tokenMask; const uint16_t syncMask = spec.ckeMask; const uint32_t postSyncSetIndex = expectedInstructions - 4; const uint32_t postSyncWaitIndex = expectedInstructions - 2; - if (program.size() < 5 || - Slot(program[0], 0) != kLoadImdToXnHeader || - Slot(program[0], 1) != spec.preSyncLocalAddrXn || + if (program.size() < 7 || + Slot(program[0], 0) != kLoadSqeArgsToXnHeader || + Slot(program[0], 1) != spec.preSyncLocalMarkerXn || + Slot(program[0], 2) != spec.preSyncMarkerArgIndex || Slot(program[1], 0) != kSyncXnHeader || - Slot(program[1], 1) != spec.preSyncRemoteAddrXn || - Slot(program[1], 2) != spec.preSyncLocalAddrXn || + Slot(program[1], 1) != spec.preSyncRemoteMarkerXn || + Slot(program[1], 2) != spec.preSyncLocalMarkerXn || Slot(program[1], 4) != spec.preSyncChannelId || Slot(program[1], 5) != spec.preSyncRemoteNotifyCke || - Slot(program[1], 6) != outputMask || + Slot(program[1], 6) != markerMask || Slot(program[2], 0) != kLoadImdToXnHeader || - Slot(program[2], 1) != spec.preSyncLocalTokenXn || + Slot(program[2], 1) != spec.preSyncLocalAddrXn || Slot(program[3], 0) != kSyncXnHeader || - Slot(program[3], 1) != spec.preSyncRemoteTokenXn || - Slot(program[3], 2) != spec.preSyncLocalTokenXn || - Slot(program[3], 4) != spec.preSyncTokenChannelId || - Slot(program[3], 5) != spec.preSyncRemoteTokenNotifyCke || - Slot(program[3], 6) != tokenMask || - Slot(program[4], 0) != kSetCkeHeader || - Slot(program[4], 4) != spec.preSyncLocalWaitCke || - Slot(program[4], 5) != waitMask) { - std::cerr << "PreSync should publish output and token then wait for both" + Slot(program[3], 1) != spec.preSyncRemoteAddrXn || + Slot(program[3], 2) != spec.preSyncLocalAddrXn || + Slot(program[3], 6) != outputMask || + Slot(program[4], 0) != kLoadImdToXnHeader || + Slot(program[4], 1) != spec.preSyncLocalTokenXn || + Slot(program[5], 0) != kSyncXnHeader || + Slot(program[5], 1) != spec.preSyncRemoteTokenXn || + Slot(program[5], 2) != spec.preSyncLocalTokenXn || + Slot(program[5], 4) != spec.preSyncTokenChannelId || + Slot(program[5], 5) != spec.preSyncRemoteTokenNotifyCke || + Slot(program[5], 6) != tokenMask || + Slot(program[6], 0) != kSetCkeHeader || + Slot(program[6], 4) != spec.preSyncLocalWaitCke || + Slot(program[6], 5) != waitMask) { + std::cerr << "PreSync should publish marker, output, and token then wait for all" << " loadHeader=0x" << std::hex << Slot(program[0], 0) << " loadXn=0x" << Slot(program[0], 1) << " notifyHeader=0x" << Slot(program[1], 0) @@ -194,15 +207,15 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): << " channel=0x" << Slot(program[1], 4) << " remoteCke=0x" << Slot(program[1], 5) << " notifyMask=0x" << Slot(program[1], 6) - << " tokenNotifyHeader=0x" << Slot(program[3], 0) - << " tokenRemoteXn=0x" << Slot(program[3], 1) - << " tokenLocalXn=0x" << Slot(program[3], 2) - << " tokenChannel=0x" << Slot(program[3], 4) - << " tokenRemoteCke=0x" << Slot(program[3], 5) - << " tokenNotifyMask=0x" << Slot(program[3], 6) - << " waitHeader=0x" << Slot(program[4], 0) - << " waitCke=0x" << Slot(program[4], 4) - << " waitMask=0x" << Slot(program[4], 5) + << " tokenNotifyHeader=0x" << Slot(program[5], 0) + << " tokenRemoteXn=0x" << Slot(program[5], 1) + << " tokenLocalXn=0x" << Slot(program[5], 2) + << " tokenChannel=0x" << Slot(program[5], 4) + << " tokenRemoteCke=0x" << Slot(program[5], 5) + << " tokenNotifyMask=0x" << Slot(program[5], 6) + << " waitHeader=0x" << Slot(program[6], 0) + << " waitCke=0x" << Slot(program[6], 4) + << " waitMask=0x" << Slot(program[6], 5) << std::dec << "\n"; return 4; } @@ -239,9 +252,9 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.sourceCke = 0; ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); if (ret != TILEXR_SUCCESS || - report.preSyncInstructionCount != 4 || - report.totalInstructionCount != expectedBlocks * 7 + 4 || - program.size() != expectedBlocks * 7 + 4) { + report.preSyncInstructionCount != 6 || + report.totalInstructionCount != expectedBlocks * 7 + 6 || + program.size() != expectedBlocks * 7 + 6) { std::cerr << "notify-only PreSync program has unexpected counts\n"; return 7; } @@ -292,6 +305,10 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): spec.localXn = 0x201; spec.remoteXn = 0x202; spec.lengthXn = 0x203; + spec.preSyncLocalMarkerXn = 0x201; + spec.preSyncRemoteMarkerXn = 0x213; + spec.preSyncMarkerArgIndex = 0; + spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; spec.copyCompletionCke = 0x301; spec.preSyncRemoteAddrXn = 0x211; @@ -341,7 +358,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): << " rank1=" << report1.message << "\n"; return 1; } - const uint32_t expectedInstructions = 5 + 64 * 7 + 3 + 1; + const uint32_t expectedInstructions = 7 + 64 * 7 + 3 + 1; if (report0.totalInstructionCount != expectedInstructions || report1.totalInstructionCount != expectedInstructions || rank0.size() != expectedInstructions || @@ -359,7 +376,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): } const size_t rank0FirstCopy = FirstCopyIndex(rank0); const size_t rank1FirstCopy = FirstCopyIndex(rank1); - if (rank0FirstCopy != 10 || rank1FirstCopy != 10) { + if (rank0FirstCopy != 12 || rank1FirstCopy != 12) { std::cerr << "copy region should start at the same instruction after the single PreSync" << " rank0FirstCopy=" << rank0FirstCopy << " rank1FirstCopy=" << rank1FirstCopy << "\n"; diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index 1da3bb8b..006553b3 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1750,8 +1750,8 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 5U + 64U * 7U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 5U + 64U * 7U", planner) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", planner) self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn", source) self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) @@ -1775,6 +1775,10 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): "preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn", source ) self.assertIn("preSyncTokenChannelId = preResource.channelId", source) + self.assertIn("preSyncLocalMarkerXn = copyResource.localXn", source) + self.assertIn("preSyncRemoteMarkerXn = copyResource.remoteXn", source) + self.assertIn("preSyncMarkerArgIndex = 0", source) + self.assertIn("preSyncMarkerEnabled = true", source) self.assertNotIn("preSyncTokenChannelId = postResource.channelId", source) self.assertIn( "preSyncOnCopyRoute ? attempt->allocation.remoteNotifyCke.startId : preResource.notifyCke", source diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index e7f79d1e..c76a80a9 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -677,6 +677,38 @@ def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): self.assertIn("tilexr_ccu_alltoall result passed=1", source) self.assertIn("BuildP2pCcuCopyPattern(peer", source) + def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + body = source[ + source.index("int RunAllToAllLongMissionSmokeForRank"): + source.index("int RunAllToAllSmokeForRank") + ] + + self.assertIn('kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"', source) + self.assertIn("AllToAllLoopCountFromEnv", source) + loop_count_body = source[ + source.index("int AllToAllLoopCountFromEnv"): + source.index("uint64_t BuildAllToAllLoopMarker") + ] + self.assertIn("std::strtol", loop_count_body) + self.assertIn("parsed < 1 || parsed > 1024", loop_count_body) + self.assertIn("BuildAllToAllLoopMarker", source) + self.assertIn("BuildAllToAllLoopPattern", source) + self.assertIn("ResetAllToAllStateForLoop", source) + self.assertIn("ReadAndValidatePeerLoopMarker", source) + self.assertIn("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)", body) + self.assertIn("attempt.submitTasks.front().args[0] =", body) + self.assertIn("WaitForCollectiveSubmitReadiness(", body) + self.assertIn("loopIndex);", body) + self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex)", body) + self.assertIn("adapter.ReadXnRange", source) + self.assertIn("peerLoopMarker", source) + self.assertIn("loopIndex=", body) + self.assertLess( + body.index("PrepareDirectCcuAllToAll2RankInstallAttempt"), + body.index("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)"), + ) + def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): source = PROBE_SOURCE.read_text(encoding="utf-8") @@ -715,8 +747,10 @@ def test_smoke_runner_forwards_alltoall_env(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION", runner) self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_LOOP_COUNT", runner) self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) - self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-452", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-458", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-454", runner) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 66b1b45c..cad3e2ba 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -180,6 +180,8 @@ def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}"', source) + self.assertIn('common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}")', source) self.assertIn('TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}"', source) self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}"', source) self.assertLess(source.index("apply_sync_xn_ping_defaults"), source.index("apply_alltoall_defaults")) From bc50d9bce23719e3136020c204cb73c4ea23b9f2 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 15:18:40 +0800 Subject: [PATCH 27/41] docs(ccu): design four-rank all-to-all mesh --- .../2026-07-20-ccu-alltoall-4p-mesh-design.md | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md diff --git a/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md b/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md new file mode 100644 index 00000000..2eb906de --- /dev/null +++ b/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md @@ -0,0 +1,191 @@ +# CCU AlltoAll 4P Mesh Design + +## Goal + +Extend the validated direct CCU AlltoAll path from two ranks to a true four-rank +Mesh1D collective. Each rank uses one communicator, one installed mission, and +one stable set of QP, jetty, CKE, XN, channel, and registered-memory resources. +Validation first runs one submission and then ten consecutive submissions with +the same prepared resources. + +The hardware target is `141.61.49.192` on NPU devices `4,5,6,7`. Each rank owns +an 8 MiB send buffer and an 8 MiB receive buffer. Each peer chunk is 2 MiB. + +## Collective Semantics + +Buffers use rank-major chunk layout: + +```text +send[targetRank][2 MiB] +recv[sourceRank][2 MiB] +``` + +For local rank `r`, the collective must produce: + +```text +recv_r[s] == send_s[r] +``` + +for every source rank `s` in `0..3`. The test pattern encodes source rank, +target rank, loop index, and byte offset. This detects incorrect source, +destination, generation, chunk placement, and partial-copy behavior. + +## Architecture + +Add a general four-rank Mesh spec and program instead of changing the meaning +of `TileXRCcuDirectAllToAll2RankSpec`. The existing 2P interfaces remain as a +regression baseline. Shared block-copy, marker, and diagnostic helpers may be +factored only where this does not change 2P behavior. + +Each rank has three peer descriptors sorted by peer rank. A descriptor contains +the imported remote destination address and token, peer rank, loop-marker XNs, +and the copy, PreSync, and token route resources. + +The fixed sync-resource layout is: + +```text +peer[0]: copy=0, pre=1, token=2 +peer[1]: copy=3, pre=4, token=5 +peer[2]: copy=6, pre=7, token=8 +``` + +The planner performs one endpoint AllGather, validates all four endpoints, +imports the other three destination buffers, and installs route-specific +memory overrides for all nine sync routes. The lower-layer override state must +therefore become a collection indexed by sync-route index instead of a single +optional override. + +The program uses one mission. It publishes marker, output address, and token to +all three peers before waiting on any peer, then waits for every peer's PreSync +mask `0x7`. This all-post-before-wait ordering prevents rank-dependent peer +iteration from creating a wait cycle. + +After PreSync, the mission copies the three remote chunks to +`peer.recv[localRank]` and copies the self chunk from `send[localRank]` to +`recv[localRank]`. It waits for all local and remote completion signals before +returning. + +## CCU Self Copy + +The self chunk is copied by CCU, not by ACL. HCCL/hcomm reference code maps +`GroupCopy` through `CcuRepLocCpy` to the hardware +`TransLocMemToLocMemInstr` instruction. TileXR will add an independent +`TileXRCcuEncodeTransLocMemToLocMem` encoder using the verified instruction +field layout without including, linking, or calling private HCCL/hcomm code. + +The AlltoAll program builder will load self source and destination addresses, +their tokens, and length, issue local-to-local transfers, and wait for local +completion. Unit tests must decode every relevant instruction field and verify +the self offsets for local ranks zero through three. + +## Instruction And Resource Capacity + +Each 2 MiB remote chunk retains the validated 64 blocks by 7 instructions copy +shape. Three remote chunks plus PreSync are expected to require about 1365 +instructions, with additional instructions for CCU self copy and final waits. + +Prepare must calculate the exact program size before installation and compare +the requested repository range with device basic-info capacity. Insufficient +instruction, channel, XN, CKE, GSA, or route resources must fail with requested +and available counts. The implementation must not truncate a program or reuse +one peer's resource IDs for another peer. + +## Repeated Submission Protocol + +Allocation, endpoint exchange, remote import, route construction, +registration, mission installation, prepared-task creation, and stream creation +all occur once outside the loop. + +For each loop index: + +1. Fill all four send chunks with source/target/loop-specific patterns. +2. Reset the complete 8 MiB receive buffer. +3. Update prepared-task argument zero with a rank-and-loop marker. +4. Enter the four-rank `ready.phaseN` gate. +5. Submit the same prepared task and synchronize the same stream. +6. Read and validate the current marker from all three peers. +7. Read and compare the complete 8 MiB receive buffer. +8. Enter the four-rank `done.phaseN` gate with the local validation result. + +The marker contains a fixed magic prefix, sender rank, and loop index. CKE bits +remain presence flags and are cleared by their waits; marker XNs carry the +generation identity needed to reject stale synchronization. + +If any rank reports an error, all ranks stop before the next loop. Diagnostics +must include local rank, loop index, peer rank, route and channel IDs, marker +XN/CKE values, mission current instruction, local-copy completion, and the first +mismatch's source rank, chunk offset, global offset, expected byte, and observed +byte. + +## Runner + +The smoke runner must accept rank size four and a four-device list. It will +construct rank-specific environment arrays for ranks zero through three, +launch four processes, track four PIDs and statuses, and validate four logs. +Hard-coded rank0/rank1 loops and result thresholds must become rank-size-driven. + +Rank-specific endpoint and resource-window fields, including EID index, must be +forwarded for all four ranks. Four-rank AlltoAll defaults use EID index 3 for +each rank unless explicitly overridden. + +The runner must require the exact number of successful results and marker +checks for the requested loop count. Seeing one successful line is not +sufficient. + +## Testing + +Automated coverage includes: + +- microcode encoding and decoding for `TransLocMemToLocMem`; +- rank 0..3 program generation, peer ordering, all-post-before-wait ordering, + self offsets, remote offsets, completion waits, and instruction counts; +- planner endpoint exchange, three remote imports, nine route mappings, and + resource-exhaustion diagnostics; +- smoke pattern/reset, three peer-marker checks, full 8 MiB mismatch reporting, + and prepare-before-loop ordering; +- dynamic four-process runner launch, environment forwarding, status handling, + exact result counts, and dry-run output; +- all existing 2P program, planner, smoke, runner, and loop-reuse regression + tests. + +## Hardware Validation + +Before every hardware run, query `npu-smi` and apply the repository busy guard +to devices `4,5,6,7`. If any selected device is busy, poll every 30 seconds and +do not terminate or bypass existing jobs. + +Run these stages in order: + +1. Build `tile-comm` and the four-rank smoke probe. +2. Run 4P with loop count one in a fresh work directory. +3. Run 4P with loop count ten in another fresh work directory. +4. Re-run the existing 2P loop-count-ten test on its validated device pair. + +The 4P loop-one run requires four successful rank results, twelve matching peer +markers, and zero mismatches over every 8 MiB receive buffer. + +The 4P loop-ten run requires forty successful rank results and 120 matching +peer markers. Each rank must retain the same mission ID, key, instruction start, +instruction count, task count, and installed resource IDs across all ten loops. +Every loop must report zero mismatches. + +## Deployment And Cleanup + +Use Mutagen as the preferred source synchronization mechanism. First determine +whether the installed Mutagen version supports an explicit SSH identity without +the disabled Windows `ssh-agent`. If it does not, record that incompatibility +and use the previously approved `scp -i` fallback for only the changed files. + +A temporary root SSH key may be installed for validation. After successful or +aborted validation, remove only its uniquely tagged remote authorized-key entry +and delete its local private key, public key, askpass helper, and any temporary +SSH wrapper or configuration. Do not modify unrelated untracked files or stop +unrelated remote workloads. + +## Non-Goals + +- Supporting arbitrary rank sizes beyond four in this validation cycle. +- Adding AlltoAllV variable counts or displacements. +- Matching multi-jetty performance tuning from HCCL. +- Replacing TileXR's runtime with HCCL/hcomm private APIs. +- Claiming network-copy concurrency or performance until profiling proves it. From c86e3182d4def5a02091b15b91d204b9a6564515 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 16:15:13 +0800 Subject: [PATCH 28/41] docs(ccu): retarget four-rank validation server --- .../2026-07-20-ccu-alltoall-4p-mesh-design.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md b/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md index 2eb906de..d27fdcca 100644 --- a/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md +++ b/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md @@ -8,7 +8,7 @@ one stable set of QP, jetty, CKE, XN, channel, and registered-memory resources. Validation first runs one submission and then ten consecutive submissions with the same prepared resources. -The hardware target is `141.61.49.192` on NPU devices `4,5,6,7`. Each rank owns +The hardware target is `141.61.50.31` on NPU devices `4,5,6,7`. Each rank owns an 8 MiB send buffer and an 8 MiB receive buffer. Each peer chunk is 2 MiB. ## Collective Semantics @@ -171,16 +171,15 @@ Every loop must report zero mismatches. ## Deployment And Cleanup -Use Mutagen as the preferred source synchronization mechanism. First determine -whether the installed Mutagen version supports an explicit SSH identity without -the disabled Windows `ssh-agent`. If it does not, record that incompatibility -and use the previously approved `scp -i` fallback for only the changed files. +Use Mutagen as the preferred source synchronization mechanism and reuse the +existing passwordless SSH configuration for `141.61.50.31`. Do not install a +new key or modify the remote `authorized_keys` file. Confirm the remote account +and repository path before creating a sync session or transferring files. If +the installed Mutagen version cannot use the existing SSH configuration, record +that incompatibility and use `scp` for only the changed files. -A temporary root SSH key may be installed for validation. After successful or -aborted validation, remove only its uniquely tagged remote authorized-key entry -and delete its local private key, public key, askpass helper, and any temporary -SSH wrapper or configuration. Do not modify unrelated untracked files or stop -unrelated remote workloads. +Do not modify unrelated untracked files, SSH configuration, or remote +authentication state, and do not stop unrelated remote workloads. ## Non-Goals From c03575918eeffb77370f3ae962180bdd82322278 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 16:23:47 +0800 Subject: [PATCH 29/41] docs(ccu): plan four-rank all-to-all mesh --- .../plans/2026-07-20-ccu-alltoall-4p-mesh.md | 626 ++++++++++++++++++ 1 file changed, 626 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md diff --git a/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md b/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md new file mode 100644 index 00000000..d1e0decc --- /dev/null +++ b/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md @@ -0,0 +1,626 @@ +# CCU AlltoAll 4P Mesh Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add and validate one true four-rank direct-CCU Mesh1D AlltoAll mission that moves four 2 MiB chunks per rank, including the self chunk, and supports ten submissions with unchanged communicator, mission, and transport resources. + +**Architecture:** Keep the validated two-rank API unchanged and add a four-rank Mesh program, orchestrator entry point, and planner entry point. Each rank owns three peer descriptors sorted by peer rank and nine sync routes (`copy`, `pre`, `token` per peer); the mission publishes all three peer handshakes before any wait, then performs three remote CCU copies plus one CCU local copy and waits for all completions. Prepare/import/install happens once, while each loop rewrites data and the SQE marker and uses phase-specific four-rank ready/done gates. + +**Tech Stack:** C++14, TileXR direct CCU runtime and microcode encoders, Python `unittest` source/compile probes, Bash hardware runner, CMake, ACL runtime, Mutagen, SSH. + +--- + +## File Map + +- `src/comm/ccu/tilexr_ccu_microcode.{h,cpp}`: encode the hardware local-memory-to-local-memory transfer instruction. +- `src/comm/ccu/tilexr_ccu_alltoall_program.{h,cpp}`: describe and build one rank's four-rank Mesh mission. +- `src/comm/ccu/tilexr_ccu_direct_orchestrator.{h,cpp}`: allocate nine resources, build the Mesh program, check capacity, and package one prepared mission. +- `src/comm/ccu/tilexr_ccu_collective_planner.{h,cpp}`: AllGather four endpoints, import three destinations, map nine routes, and invoke the Mesh orchestrator. +- `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp`: prepare once, submit repeatedly, verify three peer markers and all 8 MiB of receive data. +- `tests/ccu/run_tilexr_ccu_direct_smoke.sh`: launch and evaluate a rank-size-driven process set. +- `tests/ccu/test_tilexr_ccu_*.py`: unit, source-contract, compile-probe, and runner tests for every layer. + +### Task 1: Encode CCU Local-To-Local Transfer + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_microcode.h` +- Modify: `src/comm/ccu/tilexr_ccu_microcode.cpp` +- Test: `tests/ccu/test_tilexr_ccu_microcode.py` + +- [ ] **Step 1: Write the failing encoder tests** + +Add a compile-and-run probe that calls the new API with distinct field values and decodes all four words: + +```cpp +TileXR::TileXRCcuMemTransferSpec spec; +spec.localGsa = 0x101; +spec.localXn = 0x102; +spec.remoteGsa = 0x201; // destination GSA for the local-copy opcode +spec.remoteXn = 0x202; // destination XN for the local-copy opcode +spec.lengthXn = 0x103; +spec.channelId = 0x104; +spec.setCkeId = 0x105; +spec.setCkeMask = 0x7; +spec.waitCkeId = 0x106; +spec.waitCkeMask = 0x8; +TileXR::TileXRCcuInstr instr; +assert(TileXR::TileXRCcuEncodeTransLocMemToLocMem(spec, &instr) == TileXR::TILEXR_SUCCESS); +assert(slot(instr.words[0], 0) == 0x100a); +assert(slot(instr.words[0], 1) == spec.remoteGsa); +assert(slot(instr.words[0], 2) == spec.remoteXn); +assert(slot(instr.words[0], 3) == spec.localGsa); +assert(slot(instr.words[1], 0) == spec.localXn); +assert(slot(instr.words[1], 1) == spec.lengthXn); +assert(slot(instr.words[1], 2) == spec.channelId); +assert(slot(instr.words[3], 0) == spec.setCkeId); +assert(slot(instr.words[3], 1) == spec.setCkeMask); +assert(slot(instr.words[3], 2) == spec.waitCkeId); +assert(slot(instr.words[3], 3) == spec.waitCkeMask); +``` + +Also assert null output, zero GSA/XN/length/channel, half-specified CKE pairs, and invalid reduce fields return `TILEXR_ERROR_PARA_CHECK_FAIL`. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_microcode -v` + +Expected: FAIL because `TileXRCcuEncodeTransLocMemToLocMem` is undeclared/undefined. + +- [ ] **Step 3: Add the public declaration and minimal encoder** + +Add to the header: + +```cpp +int TileXRCcuEncodeTransLocMemToLocMem( + const TileXRCcuMemTransferSpec& spec, + TileXRCcuInstr* instr); +``` + +Add opcode `0x100a` beside the two existing transfer opcodes and encode using the same validation and flag packing: + +```cpp +int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + instr->words[0] = PackSlots(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER, + spec.remoteGsa, spec.remoteXn, spec.localGsa); + instr->words[1] = PackSlots(spec.localXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); + instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} +``` + +- [ ] **Step 4: Run the focused and full microcode tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_microcode -v` + +Expected: PASS, including the existing remote/local transfer encoders. + +- [ ] **Step 5: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_microcode.h src/comm/ccu/tilexr_ccu_microcode.cpp tests/ccu/test_tilexr_ccu_microcode.py +git commit -m "feat(ccu): encode local memory transfers" +``` + +### Task 2: Build The Four-Rank Mesh Mission + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.h` +- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.cpp` +- Test: `tests/ccu/test_tilexr_ccu_alltoall_program.py` + +- [ ] **Step 1: Add failing rank-parameterized Mesh tests** + +Define a fixture for each `localRank` in `0..3` with peers in intentionally unsorted input order. Decode the generated instructions and assert: + +```python +self.assertEqual([p for p in range(4) if p != local_rank], report.peerRanks) +self.assertEqual(3, report.peerCount) +self.assertEqual(9, report.syncResourceCount) +self.assertEqual(64, report.remoteBlockCountPerPeer) +self.assertEqual(64, report.selfBlockCount) +self.assertLess(last_peer_post_index, first_peer_wait_index) +self.assertEqual(local_rank * 2 * 1024 * 1024, decoded_self_source_offset) +self.assertEqual(local_rank * 2 * 1024 * 1024, decoded_self_destination_offset) +``` + +For every peer `p`, assert remote source offset is `localRank * chunkBytes`, remote destination offset is `localRank * chunkBytes`, the peer route IDs are `3*ordinal+{0,1,2}`, and the PreSync wait mask is `0x7`. Add failures for rank size other than four, duplicate/missing/self peer, zero token/address, non-2-MiB-aligned size, duplicate resource IDs, and insufficient output pointer/report. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_alltoall_program -v` + +Expected: FAIL because the Mesh types and builder do not exist. + +- [ ] **Step 3: Add explicit Mesh types** + +Add these interfaces without changing `TileXRCcuAllToAll2RankProgramSpec`: + +```cpp +struct TileXRCcuAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; + uint16_t copyResourceIndex = 0; + uint16_t preSyncResourceIndex = 0; + uint16_t tokenResourceIndex = 0; +}; + +struct TileXRCcuAllToAllMeshProgramSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 2ULL * 1024ULL * 1024ULL; + uint16_t markerArgIndex = 0; + uint16_t localGsa = 0; + uint16_t selfDestinationGsa = 0; + uint16_t localXn = 0; + uint16_t selfDestinationXn = 0; + uint16_t lengthXn = 0; + uint16_t selfChannelId = 0; + uint16_t selfCompletionCke = 0; + std::vector peers; +}; + +int TileXRCcuBuildAllToAllMeshProgram( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report); +``` + +Extend the report with peer ranks, peer/resource counts, self/remote block counts, and local/remote completion counts. + +- [ ] **Step 4: Implement deterministic validation and emission** + +Implement this exact high-level ordering using the existing `Append*` helpers and the Task 1 local-copy encoder: + +```cpp +ValidateFourRankShapeAndUniqueResources(spec); +auto peers = spec.peers; +std::sort(peers.begin(), peers.end(), ByPeerRank); +LoadMarkerFromSqeArg(spec.markerArgIndex); +for (const auto& peer : peers) { + PostMarker(peer); + PostRemoteDestinationAddress(peer); + PostRemoteDestinationToken(peer); +} +for (const auto& peer : peers) { + WaitForPeerPreSync(peer, 0x7); +} +for (const auto& peer : peers) { + EmitRemoteBlocks(peer, spec.localRank * spec.chunkBytes, spec.chunkBytes); +} +EmitLocalBlocks(spec.localRank * spec.chunkBytes, spec.chunkBytes); +for (const auto& peer : peers) { + WaitForRemoteCopyCompletion(peer); +} +WaitForSelfCopyCompletion(spec.selfCompletionCke); +EmitFinish(); +``` + +Do not reuse one peer's channel/XN/CKE IDs, do not emit a wait inside the publish loop, and derive `report.totalInstructionCount` from `program->size()`. + +- [ ] **Step 5: Run Mesh and 2P regression tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_alltoall_program -v` + +Expected: PASS for ranks 0..3 and all existing 2P cases. + +- [ ] **Step 6: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_alltoall_program.h src/comm/ccu/tilexr_ccu_alltoall_program.cpp tests/ccu/test_tilexr_ccu_alltoall_program.py +git commit -m "feat(ccu): build four-rank all-to-all mesh mission" +``` + +### Task 3: Allocate And Package Mesh Resources + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.h` +- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` +- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` + +- [ ] **Step 1: Write failing resource/package tests** + +Add offline tests that provide a four-peer endpoint spec and assert one mission, one task, nine sync resources, peer-ordinal route mapping `0/1/2`, `3/4/5`, `6/7/8`, and exact program-sized repository installation. Mutate each basic-info capacity (`mission`, `instruction`, `channel`, `xn`, `cke`, `gsa`) below the requested count and require the report message to contain both `requested=` and `available=`. + +- [ ] **Step 2: Run the test to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator -v` + +Expected: FAIL because `TileXRCcuRunDirectAllToAllMeshInstallAttempt` is missing. + +- [ ] **Step 3: Add the orchestrator API** + +```cpp +struct TileXRCcuDirectAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; +}; + +struct TileXRCcuDirectAllToAllMeshSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 2ULL * 1024ULL * 1024ULL; + std::vector peers; +}; + +int TileXRCcuRunDirectAllToAllMeshInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); +``` + +- [ ] **Step 4: Build once to size, validate capacity, then install once** + +Create a resource request with `syncResourceCount=9`, allocate distinct copy/pre/token resources per peer, reserve self-copy GSA/XN/channel/CKE, build the Mesh program into a temporary vector, and use its exact size before repository installation: + +```cpp +const uint32_t requestedInstructions = static_cast(program.size()); +const uint32_t availableInstructions = RangeAvailableCount( + attempt->specInfo.instructionNum, options.instructionStartId); +if (requestedInstructions > availableInstructions) { + return CapacityError("instruction", requestedInstructions, + availableInstructions, report); +} +attempt->package.missions.resize(1); +attempt->submitTasks.resize(1); +attempt->submitTasks[0].argSize = std::max(1, options.sqeArgCount); +``` + +Preserve the existing 2P path byte-for-byte except for shared pure helpers extracted to avoid duplication. + +- [ ] **Step 5: Run orchestrator and AlltoAll tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator tests.ccu.test_tilexr_ccu_alltoall_program -v` + +Expected: PASS; invalid capacities identify the exhausted resource and exact counts. + +- [ ] **Step 6: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_direct_orchestrator.h src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp tests/ccu/test_tilexr_ccu_direct_orchestrator.py +git commit -m "feat(ccu): package four-rank all-to-all resources" +``` + +### Task 4: Gather Four Endpoints And Import Three Destinations + +**Files:** +- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.h` +- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` +- Test: `tests/ccu/test_tilexr_ccu_collective_planner.py` +- Test: `tests/ccu/test_tilexr_ccu_direct_backend.py` + +- [ ] **Step 1: Write failing planner tests** + +Use the existing fake session/backend to return four gathered `DirectCcuMemoryCopyEndpoint` values. Assert exactly one AllGather, exactly three imports in peer-rank order, nine lower-layer routes, three distinct imported `targetSegVa` values repeated only across that peer's three routes, and no import for the local endpoint. Add invalid endpoint tests for wrong rank size, duplicate endpoint rank, `valid=0`, wrong byte count, missing token, and failed import. + +- [ ] **Step 2: Run planner tests to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_collective_planner tests.ccu.test_tilexr_ccu_direct_backend -v` + +Expected: FAIL because the Mesh planner API and multi-override storage are absent. + +- [ ] **Step 3: Add the Mesh planner entry point** + +```cpp +int PrepareDirectCcuAllToAllMeshInstallAttempt( + TileXRCcuRuntimeSession& session, + const TileXRCcuDirectInstallOptions& options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t chunkBytes, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); +``` + +Require `session.RankSize() == 4` and register `4 * chunkBytes` for both local buffers while retaining `chunkBytes` as the per-peer transfer size. + +- [ ] **Step 4: Replace the single testing override with route-indexed overrides** + +```cpp +struct DirectCcuRemoteRouteMemoryOverride { + uint32_t syncRouteIndex = 0; + TileXRCcuRemoteCcuBufferInfo buffer; +}; +std::vector directCcuRemoteRouteMemoryOverrides_; +``` + +`SetDirectCcuRemoteRouteMemoryOverride` must keep its old all-routes behavior for 2P tests. `SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute` updates/inserts one indexed entry, `Apply...` overlays every matching route, and `Clear...` clears the vector. + +- [ ] **Step 5: Implement one gather and three imports** + +```cpp +std::vector endpoints(4); +TILEXR_RETURN_IF_ERROR(session.AllGather(&localEndpoint, sizeof(localEndpoint), endpoints.data())); +for (uint32_t peer = 0, ordinal = 0; peer < 4; ++peer) { + if (peer == static_cast(session.Rank())) continue; + ValidateEndpoint(endpoints[peer], peer, 4 * chunkBytes); + auto imported = ImportDestination(session, endpoints[peer]); + mesh.peers.push_back({peer, endpoints[peer].destinationRemoteImport.addr, + TileXRCcuPackMemoryToken(endpoints[peer].destinationRemoteImport.tokenId, + endpoints[peer].destinationRemoteImport.tokenValue, true)}); + for (uint32_t route = 0; route < 3; ++route) { + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(3 * ordinal + route, peer, + imported.targetSegVa, endpoints[peer].destinationRemoteImport.tokenId, + endpoints[peer].destinationRemoteImport.rawTokenId, + endpoints[peer].destinationRemoteImport.tokenValue); + } + ++ordinal; +} +``` + +Set `syncResourceCount=9`, derive `syncInstructionCount` from the built Mesh program rather than a stale constant, invoke the Mesh orchestrator, and clear overrides on every return path with a small RAII guard. + +- [ ] **Step 6: Run planner, backend, and 2P regression tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_collective_planner tests.ccu.test_tilexr_ccu_direct_backend tests.ccu.test_tilexr_ccu_direct_orchestrator -v` + +Expected: PASS; the existing 2P single-route behavior remains unchanged. + +- [ ] **Step 7: Commit** + +```bash +git add src/comm/ccu/tilexr_ccu_collective_planner.h src/comm/ccu/tilexr_ccu_collective_planner.cpp tests/ccu/test_tilexr_ccu_collective_planner.py tests/ccu/test_tilexr_ccu_direct_backend.py +git commit -m "feat(ccu): plan four-rank all-to-all endpoints" +``` + +### Task 5: Reuse One Prepared Mission For Ten Mesh Submissions + +**Files:** +- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` +- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` + +- [ ] **Step 1: Write failing source-contract and helper tests** + +Add tests requiring Mesh mode to allocate `rankSize * chunkBytes`, call `PrepareDirectCcuAllToAllMeshInstallAttempt` before the loop, create the stream before the loop, and call neither prepare nor stream creation inside it. Require a phase path containing `ready.phase` and `done.phase`, three marker validations per loop, full-buffer validation, and failure output fields `rank`, `loopIndex`, `peerRank`, `route`, `channel`, `xn`, `cke`, `currentInstruction`, `selfCopyCompletion`, `sourceRank`, `chunkOffset`, and `globalOffset`. + +- [ ] **Step 2: Run smoke-probe tests to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` + +Expected: FAIL because the probe remains two-rank-specific. + +- [ ] **Step 3: Add generation-specific data and marker helpers** + +```cpp +uint8_t ExpectedAllToAllByte(uint32_t source, uint32_t target, uint32_t loop, uint64_t offset) +{ + return static_cast((source * 67U + target * 29U + loop * 17U + + static_cast(offset * 13U)) & 0xffU); +} + +uint64_t MakeLoopMarker(uint32_t rank, uint32_t loop) +{ + return 0x5458524100000000ULL | (static_cast(rank & 0xffU) << 8U) | + static_cast(loop & 0xffU); +} +``` + +Fill `send[target][offset]` from these fields, reset all 8 MiB of receive memory each loop, and validate `recv[source][offset]` with source/local-rank/loop. + +- [ ] **Step 4: Prepare once and execute phase-isolated loops** + +```cpp +PrepareMeshOnce(&attempt, &report); +CreateStreamOnce(&stream); +const auto stable = CaptureMissionAndResourceIdentity(attempt); +for (uint32_t loopIndex = 0; loopIndex < loopCount; ++loopIndex) { + FillFourSendChunks(rank, loopIndex); + ResetCompleteReceiveBuffer(); + attempt.submitTasks[0].args[0] = MakeLoopMarker(rank, loopIndex); + FourRankGate(workDir / ("ready.phase" + std::to_string(loopIndex)), rank, 0); + SubmitAndSynchronizeSameTask(attempt.submitTasks[0], stream); + ValidateThreePeerMarkers(loopIndex); + const auto localResult = ValidateCompleteReceiveBuffer(rank, loopIndex); + FourRankGate(workDir / ("done.phase" + std::to_string(loopIndex)), rank, localResult); + StopAllRanksIfAnyDoneResultFailed(loopIndex); + AssertMissionAndResourceIdentity(stable, attempt, loopIndex); +} +``` + +The ready/done payload must include the loop marker, not a reused boolean. Any failure prints diagnostics and exits before beginning the next phase. + +- [ ] **Step 5: Run probe tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` + +Expected: PASS for Mesh contracts and existing 2P loop-reuse contracts. + +- [ ] **Step 6: Commit** + +```bash +git add tests/ccu/ccu_tilexr_direct_smoke_probe.cpp tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +git commit -m "test(ccu): exercise repeated four-rank all-to-all" +``` + +### Task 6: Generalize The Smoke Runner To Four Processes + +**Files:** +- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` +- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` + +- [ ] **Step 1: Add failing four-rank runner tests** + +Create fake probe logs for ranks `0..3`. Verify device list `4,5,6,7`, per-rank endpoint/EID/resource-window forwarding, four PIDs/statuses/logs, and exact result rules: + +```python +self.assertIn("expectedResults=40", result.stdout) +self.assertIn("expectedMarkerMatches=120", result.stdout) +self.assertIn("rank=3 device=7", result.stdout) +self.assertNotIn("resultCount>=1", runner_source) +``` + +Add negative cases for one missing rank log, one nonzero status, 39/40 results, 119/120 markers, invalid device count, and duplicate devices. + +- [ ] **Step 2: Run runner tests to verify RED** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_runner -v` + +Expected: FAIL because launch/status/log handling is hard-coded to two ranks. + +- [ ] **Step 3: Replace rank0/rank1 variables with indexed arrays** + +```bash +rank_size="${TILEXR_CCU_RANK_SIZE:-2}" +IFS=',' read -r -a devices <<< "${TILEXR_CCU_DEVICE_LIST:-0,1}" +[ "${#devices[@]}" -eq "${rank_size}" ] || fail "device count does not match rank size" +declare -a pids statuses logs +for ((rank=0; rank"${work_dir}/ccu_rank${rank}.log" 2>&1 & + pids[$rank]=$! + logs[$rank]="${work_dir}/ccu_rank${rank}.log" +done +``` + +Wait every PID even after one fails, record every status, and never kill unrelated processes. `export_rank_environment` must forward rank-specific EID, endpoint, token, XN, CKE, and resource-window fields for ranks `0..3`; default Mesh EID index is `3`. + +- [ ] **Step 4: Enforce exact aggregate counts** + +```bash +expected_results=$((rank_size * loop_count)) +expected_markers=$((rank_size * (rank_size - 1) * loop_count)) +[ "${result_count}" -eq "${expected_results}" ] || fail "result count mismatch" +[ "${marker_count}" -eq "${expected_markers}" ] || fail "marker count mismatch" +``` + +Keep rank size two as the default so current callers remain compatible. + +- [ ] **Step 5: Run runner and probe tests** + +Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_runner tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` + +Expected: PASS for 2P and 4P dry-run/fake-process cases. + +- [ ] **Step 6: Commit** + +```bash +git add tests/ccu/run_tilexr_ccu_direct_smoke.sh tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +git commit -m "test(ccu): launch four-rank direct smoke" +``` + +### Task 7: Local Regression And Remote Hardware Validation + +**Files:** +- Modify only if a test exposes a defect in files from Tasks 1-6. +- Verify: all affected CCU suites and `build_ccu_direct`. + +- [ ] **Step 1: Run all affected local tests** + +```bash +python -m unittest \ + tests.ccu.test_tilexr_ccu_microcode \ + tests.ccu.test_tilexr_ccu_alltoall_program \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_collective_planner \ + tests.ccu.test_tilexr_ccu_direct_backend \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_direct_smoke_runner -v +``` + +Expected: PASS with zero failures/errors. + +- [ ] **Step 2: Build the library and smoke probe on Linux** + +```bash +source scripts/common_env.sh +cmake --build build_ccu_direct --target tile-comm -j2 +cmake --build build_ccu_direct --target ccu_tilexr_direct_smoke_probe -j2 +``` + +Expected: both targets complete successfully and link against the real runtime libraries, not the `devlib` HAL stub. + +- [ ] **Step 3: Confirm passwordless access and the existing remote path** + +Run from Windows: + +```powershell +ssh -o BatchMode=yes root@141.61.50.31 'pwd; test -d /root/TileXR && echo TILEXR_REMOTE_OK' +``` + +Expected: passwordless login succeeds and prints the confirmed repository path. Do not modify `authorized_keys`. + +- [ ] **Step 4: Reuse or create the confirmed Mutagen sync** + +Inspect first: + +```powershell +& 'C:\Users\l00654177\AppData\Local\Programs\Mutagen\mutagen.exe' sync list +``` + +If no session exactly matches local `C:\Users\l00654177\Desktop\TileXR` and confirmed `root@141.61.50.31:/root/TileXR`, create one with ignores for `.git`, build/cache directories, credentials, and the unrelated untracked home-directory artifacts: + +```powershell +& 'C:\Users\l00654177\AppData\Local\Programs\Mutagen\mutagen.exe' sync create ` + --name tilexr-141-61-50-31 ` + --ignore-vcs ` + --ignore 'build*' --ignore '.anaconda' --ignore '.conda' --ignore '.mutagen*' --ignore '.ssh' ` + 'C:\Users\l00654177\Desktop\TileXR' 'root@141.61.50.31:/root/TileXR' +``` + +Then run `mutagen sync monitor tilexr-141-61-50-31` until status is `Watching for changes`. If Mutagen cannot use the existing SSH configuration, record the error and transfer only Task 1-6 changed files with `scp`. + +- [ ] **Step 5: Poll devices 4,5,6,7 until idle** + +On the server, run the repository busy guard or `npu-smi info`; if any selected device is occupied, query again every 30 seconds. Do not stop, signal, or reconfigure another workload. + +- [ ] **Step 6: Run 4P loop=1 in a fresh directory** + +```bash +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH=1 \ +TILEXR_CCU_RANK_SIZE=4 \ +TILEXR_CCU_DEVICE_LIST=4,5,6,7 \ +TILEXR_CCU_ALLTOALL_BYTES=2097152 \ +TILEXR_CCU_ALLTOALL_LOOP_COUNT=1 \ +TILEXR_CCU_SMOKE_WORK_DIR=/tmp/tilexr-ccu-4p-loop1-$(date +%s) \ +bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` + +Expected: exit 0, exactly 4 successful loop results, exactly 12 peer marker matches, and zero mismatches over four 8 MiB receive buffers. + +- [ ] **Step 7: Run 4P loop=10 in another fresh directory** + +Use the Step 6 command with `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10` and a `loop10` work directory. + +Expected: exit 0, exactly 40 successful loop results, exactly 120 peer marker matches, zero mismatches, phase files `0..9`, and stable mission ID/key/instruction range/task count/QP/jetty/CKE/XN/channel IDs across all ten loops. + +- [ ] **Step 8: Re-run the 2P loop=10 regression** + +After polling the selected validated pair idle, run the existing 2P command with `TILEXR_CCU_RANK_SIZE=2`, its two-device list, and `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10`. + +Expected: exit 0, exactly 20 successful loop results, exactly 20 peer marker matches, and unchanged 2P mission/resource identity. + +- [ ] **Step 9: Inspect evidence and repository hygiene** + +```bash +git diff --check +git status --short +``` + +Expected: no whitespace errors; only intended Task 1-6 files are modified. Preserve unrelated untracked files. + +- [ ] **Step 10: Commit the verified implementation** + +Stage only the explicit Task 1-6 paths and commit: + +```bash +git commit -m "feat(ccu): validate four-rank all-to-all mesh" +``` + +Record in the final verification report the three hardware commands, work directories, exact result/marker counts, mission/resource identity evidence, and any temporary Mutagen session name. From 3e97f9472eeb4eac2700cd73ea4433485d034293 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 17:29:46 +0800 Subject: [PATCH 30/41] feat(ccu): encode local memory transfers --- src/comm/ccu/tilexr_ccu_microcode.cpp | 18 ++++++++++++++++++ src/comm/ccu/tilexr_ccu_microcode.h | 2 ++ tests/ccu/test_tilexr_ccu_microcode.py | 24 +++++++++++++++++++++--- 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_microcode.cpp b/src/comm/ccu/tilexr_ccu_microcode.cpp index 44abfdbe..597465a3 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.cpp +++ b/src/comm/ccu/tilexr_ccu_microcode.cpp @@ -15,6 +15,7 @@ constexpr uint64_t TILEXR_CCU_SET_CKE_HEADER = 0x0802U; constexpr uint64_t TILEXR_CCU_CLEAR_CKE_HEADER = 0x0804U; constexpr uint64_t TILEXR_CCU_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; constexpr uint64_t TILEXR_CCU_SYNC_CKE_HEADER = 0x100bU; constexpr uint64_t TILEXR_CCU_SYNC_XN_HEADER = 0x100dU; constexpr uint64_t TILEXR_CCU_SYNC_XN_TRACE_FLAG = 0x0001000000000000ULL; @@ -255,6 +256,23 @@ int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, Til return TILEXR_SUCCESS; } +int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER), + spec.remoteGsa, + spec.remoteXn, + spec.localGsa); + instr->words[1] = PackSlots(spec.localXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); + instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + int TileXRCcuBuildSqeLoadProgram(uint16_t firstXnId, uint32_t argCount, std::vector* program) { if (program == nullptr || firstXnId == 0 || argCount == 0 || argCount > TILEXR_CCU_SQE_ARGS_LEN) { diff --git a/src/comm/ccu/tilexr_ccu_microcode.h b/src/comm/ccu/tilexr_ccu_microcode.h index fe8c0328..65cfdbd1 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.h +++ b/src/comm/ccu/tilexr_ccu_microcode.h @@ -86,6 +86,8 @@ int TileXRCcuEncodeTransRmtMemToLocMem(const TileXRCcuMemTransferSpec& spec, Til int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); +int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); + int TileXRCcuBuildSqeLoadProgram( uint16_t firstXnId, uint32_t argCount, diff --git a/tests/ccu/test_tilexr_ccu_microcode.py b/tests/ccu/test_tilexr_ccu_microcode.py index bf06d5e5..b61e29be 100644 --- a/tests/ccu/test_tilexr_ccu_microcode.py +++ b/tests/ccu/test_tilexr_ccu_microcode.py @@ -465,6 +465,19 @@ def test_memory_transfer_microcode_encoders_match_hcomm_v1_layout(self): return 4; } + TileXRCcuInstr local; + if (TileXRCcuEncodeTransLocMemToLocMem(spec, &local) != TILEXR_SUCCESS) { + std::cerr << "trans loc->loc encode failed\n"; + return 5; + } + if (local.words[0] != 0x010102020201100aULL || + local.words[1] != 0x5a00001203010102ULL || + local.words[2] != 0x0007000000000000ULL || + local.words[3] != 0x0003040200020401ULL) { + std::cerr << "unexpected trans loc->loc words\n"; + return 6; + } + return 0; } ''' @@ -489,7 +502,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): TileXRCcuInstr instr; TileXRCcuMemTransferSpec empty; if (TileXRCcuEncodeTransRmtMemToLocMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || - TileXRCcuEncodeTransLocMemToRmtMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + TileXRCcuEncodeTransLocMemToRmtMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "empty transfer accepted\n"; return 1; } @@ -505,7 +519,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): spec.setCkeMask = 8; if (TileXRCcuEncodeTransRmtMemToLocMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL || - TileXRCcuEncodeTransLocMemToRmtMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + TileXRCcuEncodeTransLocMemToRmtMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "null output accepted\n"; return 2; } @@ -518,7 +533,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): spec.reduceDataType = 0; spec.reduceOpCode = 0x10; - if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "out-of-range reduce op code accepted\n"; return 4; } @@ -552,12 +568,14 @@ def test_microcode_builder_is_wired_and_has_no_private_hcomm_surface(self): self.assertIn("struct TileXRCcuMemTransferSpec", header) self.assertIn("TileXRCcuEncodeTransRmtMemToLocMem", header) self.assertIn("TileXRCcuEncodeTransLocMemToRmtMem", header) + self.assertIn("TileXRCcuEncodeTransLocMemToLocMem", header) self.assertIn("0x0001U", source) self.assertIn("0x0002U", source) self.assertIn("0x0003U", source) self.assertIn("0x0802U", source) self.assertIn("0x0804U", source) self.assertIn("0x1008U", source) + self.assertIn("0x100aU", source) self.assertIn("0x1009U", source) self.assertIn("0x100bU", source) self.assertIn("0x100dU", source) From 88104c12c4b097c29c6f3c096bb135841ea3fed9 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 17:41:24 +0800 Subject: [PATCH 31/41] feat(ccu): build four-rank all-to-all mesh mission --- src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 298 ++++++++++++++++++ src/comm/ccu/tilexr_ccu_alltoall_program.h | 32 ++ tests/ccu/test_tilexr_ccu_alltoall_program.py | 164 ++++++++++ 3 files changed, 494 insertions(+) diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 16a23e26..bbb3739e 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -5,6 +5,10 @@ #include "ccu/tilexr_ccu_alltoall_program.h" +#include +#include +#include + namespace TileXR { namespace { @@ -398,6 +402,198 @@ void FillReport( report->message = "ok"; } +bool InsertUnique(std::set* ids, std::initializer_list values) +{ + for (const uint16_t value : values) { + if (value == 0 || !ids->insert(value).second) { + return false; + } + } + return true; +} + +int ValidateMeshSpec( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (program == nullptr) { + return Fail(program, report, "missing output direct CCU alltoall mesh program"); + } + if (spec.rankSize != 4U || spec.localRank >= spec.rankSize || spec.peers.size() != 3U) { + return Fail(program, report, "direct CCU alltoall mesh requires four ranks and three peers"); + } + if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || + spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.chunkBytes == 0 || + spec.chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return Fail(program, report, "invalid direct CCU alltoall mesh local buffer"); + } + if (spec.selfSourceGsa == 0 || spec.selfDestinationGsa == 0 || spec.selfSourceXn == 0 || + spec.selfDestinationXn == 0 || spec.selfLengthXn == 0 || spec.selfChannelId == 0 || + spec.selfCompletionCke == 0) { + return Fail(program, report, "missing direct CCU alltoall mesh self-copy resource"); + } + bool peerRanks[4] = {}; + std::set gsaIds; + std::set localXnIds; + std::set remoteXnIds; + std::set channelIds; + std::set localCkeIds; + std::set remoteCkeIds; + for (const auto& peer : spec.peers) { + if (peer.peerRank >= spec.rankSize || peer.peerRank == spec.localRank || peerRanks[peer.peerRank]) { + return Fail(program, report, "invalid direct CCU alltoall mesh peer rank"); + } + peerRanks[peer.peerRank] = true; + if (peer.route.localRank != spec.localRank || peer.route.localSendAddr != spec.localSendAddr || + peer.route.localSendToken != spec.localSendToken || peer.route.localRecvAddr != spec.localRecvAddr || + peer.route.localRecvToken != spec.localRecvToken || peer.route.bytes != spec.chunkBytes || + !peer.route.preSyncMarkerEnabled || !peer.route.preSyncNotify || !peer.route.preSyncWait || + !peer.route.postSyncNotify || !peer.route.postSyncWait) { + return Fail(program, report, "invalid direct CCU alltoall mesh peer route"); + } + if (!InsertUnique(&gsaIds, {peer.route.localGsa, peer.route.remoteGsa}) || + !InsertUnique(&localXnIds, + {peer.route.localXn, peer.route.lengthXn, peer.route.preSyncLocalAddrXn, + peer.route.preSyncLocalTokenXn, peer.route.preSyncLocalMarkerXn}) || + !InsertUnique(&remoteXnIds, + {peer.route.remoteXn, peer.route.preSyncRemoteAddrXn, + peer.route.preSyncRemoteTokenXn, peer.route.preSyncRemoteMarkerXn}) || + !InsertUnique(&channelIds, + {peer.route.preSyncChannelId, peer.route.preSyncTokenChannelId, peer.route.copyChannelId}) || + !InsertUnique(&localCkeIds, + {peer.route.copyCompletionCke, peer.route.preSyncLocalWaitCke, + peer.route.postSyncLocalWaitCke, peer.route.sourceCke}) || + !InsertUnique(&remoteCkeIds, + {peer.route.preSyncRemoteNotifyCke, peer.route.preSyncRemoteTokenNotifyCke, + peer.route.postSyncRemoteNotifyCke})) { + return Fail(program, report, "duplicate direct CCU alltoall mesh peer resource"); + } + TileXRCcuAllToAll2RankProgramSpec validationRoute = peer.route; + validationRoute.localRank = 0; + std::vector ignored; + TileXRCcuAllToAllProgramReport ignoredReport; + if (ValidateSpec(validationRoute, &ignored, &ignoredReport) != TILEXR_SUCCESS) { + return Fail(program, report, ignoredReport.message); + } + } + return TILEXR_SUCCESS; +} + +int AppendMeshPeerPosts( + const TileXRCcuAllToAll2RankProgramSpec& route, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (AppendRemoteMarkerNotify( + route.preSyncRemoteNotifyCke, + route.preSyncChannelId, + route, + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AppendRemoteNotify( + route.preSyncRemoteNotifyCke, + route.preSyncChannelId, + route.preSyncLocalAddrXn, + route.preSyncRemoteAddrXn, + route.localRecvAddr, + 0, + PreSyncSignalMask(route), + "mesh PreSync output", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return AppendRemoteNotify( + route.preSyncRemoteTokenNotifyCke, + route.preSyncTokenChannelId, + route.preSyncLocalTokenXn, + route.preSyncRemoteTokenXn, + route.localRecvToken, + 1, + PreSyncTokenMask(route), + "mesh PreSync token", + program, + report); +} + +int AppendLocalCopyBlock( + const TileXRCcuAllToAllMeshProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToGsa(spec.selfSourceGsa, spec.localSendAddr + offset, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self source address"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfSourceXn, spec.localSendToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self source token"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToGsa(spec.selfDestinationGsa, spec.localRecvAddr + offset, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self destination address"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfDestinationXn, spec.localRecvToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self destination token"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfLengthXn, bytesPerBlock, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self length"); + } + program->push_back(instr); + + TileXRCcuMemTransferSpec transfer; + transfer.localGsa = spec.selfSourceGsa; + transfer.localXn = spec.selfSourceXn; + transfer.remoteGsa = spec.selfDestinationGsa; + transfer.remoteXn = spec.selfDestinationXn; + transfer.lengthXn = spec.selfLengthXn; + transfer.channelId = spec.selfChannelId; + transfer.setCkeId = spec.selfCompletionCke; + transfer.setCkeMask = 1U; + if (TileXRCcuEncodeTransLocMemToLocMem(transfer, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer"); + } + program->push_back(instr); + + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.selfCompletionCke; + wait.waitMask = 1U; + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to wait direct CCU alltoall mesh self transfer"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendMeshPostNotify( + const TileXRCcuAllToAll2RankProgramSpec& route, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (AppendSetSourceCke(route, route.ckeMask, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRCcuSyncCkeSpec post; + post.remoteCke = route.postSyncRemoteNotifyCke; + post.localCke = route.sourceCke; + post.localCkeMask = route.ckeMask; + post.channelId = route.postSyncChannelId; + post.clearWait = true; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to notify direct CCU alltoall mesh completion"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + } // namespace int TileXRCcuBuildAllToAll2RankProgram( @@ -468,4 +664,106 @@ int TileXRCcuBuildAllToAll2RankProgram( return TILEXR_SUCCESS; } +int TileXRCcuBuildAllToAllMeshProgram( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + ResetReport(report); + if (program != nullptr) { + program->clear(); + } + int ret = ValidateMeshSpec(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + auto peers = spec.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuAllToAllMeshPeerSpec& lhs, + const TileXRCcuAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + const uint64_t bytesPerBlock = TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint32_t blocksPerChunk = static_cast(spec.chunkBytes / bytesPerBlock); + program->reserve(3U * 7U + 4U * blocksPerChunk * 7U + 3U * 3U + 1U); + + for (const auto& peer : peers) { + ret = AppendMeshPeerPosts(peer.route, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + for (const auto& peer : peers) { + ret = AppendNotifyWait(peer.route.preSyncLocalWaitCke, 0x7U, "mesh PreSync", false, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + for (const auto& peer : peers) { + TileXRCcuAllToAll2RankProgramSpec route = peer.route; + route.localSendAddr = spec.localSendAddr + static_cast(peer.peerRank) * spec.chunkBytes; + route.remoteRecvAddr += static_cast(spec.localRank) * spec.chunkBytes; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + ret = AppendCopyBlock(route, static_cast(block) * bytesPerBlock, bytesPerBlock, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + } + const uint64_t selfBaseOffset = static_cast(spec.localRank) * spec.chunkBytes; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + ret = AppendLocalCopyBlock( + spec, + selfBaseOffset + static_cast(block) * bytesPerBlock, + bytesPerBlock, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + for (const auto& peer : peers) { + ret = AppendMeshPostNotify(peer.route, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + for (const auto& peer : peers) { + ret = AppendNotifyWait( + peer.route.postSyncLocalWaitCke, + peer.route.ckeMask, + "mesh PostSync", + true, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + TileXRCcuInstr finish; + if (TileXRCcuEncodeLoadImdToXn(spec.selfSourceXn, 0, 0, &finish) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh finish"); + } + program->push_back(finish); + + if (report != nullptr) { + report->preSyncInstructionCount = 21U; + report->blockCount = blocksPerChunk; + report->bytesPerBlock = static_cast(bytesPerBlock); + report->copyInstructionCount = 4U * blocksPerChunk * 7U; + report->postSyncInstructionCount = 9U; + report->finishInstructionCount = 1U; + report->totalInstructionCount = static_cast(program->size()); + report->peerCount = 3U; + report->syncResourceCount = 9U; + report->remoteBlockCount = 3U * blocksPerChunk; + report->selfBlockCount = blocksPerChunk; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index 484ff30a..d73db788 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -73,6 +73,29 @@ struct TileXRCcuAllToAll2RankProgramSpec { bool emitFinish = true; }; +struct TileXRCcuAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + TileXRCcuAllToAll2RankProgramSpec route; +}; + +struct TileXRCcuAllToAllMeshProgramSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 0; + uint16_t selfSourceGsa = 0; + uint16_t selfDestinationGsa = 0; + uint16_t selfSourceXn = 0; + uint16_t selfDestinationXn = 0; + uint16_t selfLengthXn = 0; + uint16_t selfChannelId = 0; + uint16_t selfCompletionCke = 0; + std::vector peers; +}; + struct TileXRCcuAllToAllProgramReport { uint32_t preSyncInstructionCount = 0; uint32_t blockCount = 0; @@ -81,6 +104,10 @@ struct TileXRCcuAllToAllProgramReport { uint32_t postSyncInstructionCount = 0; uint32_t finishInstructionCount = 0; uint32_t totalInstructionCount = 0; + uint32_t peerCount = 0; + uint32_t syncResourceCount = 0; + uint32_t remoteBlockCount = 0; + uint32_t selfBlockCount = 0; std::string message; }; @@ -89,6 +116,11 @@ int TileXRCcuBuildAllToAll2RankProgram( std::vector* program, TileXRCcuAllToAllProgramReport* report); +int TileXRCcuBuildAllToAllMeshProgram( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report); + } // namespace TileXR #endif // TILEXR_CCU_ALLTOALL_PROGRAM_H diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index 45a77265..f1704841 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -526,6 +526,170 @@ def test_local_rank_does_not_split_the_long_mission_into_copy_phases(self): self.assertIn("alltoallSpec.localRank = alltoall.localRank", orchestrator) self.assertIn("alltoall.localRank = static_cast(rank)", planner) + def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + #include + + using namespace TileXR; + + uint16_t Slot(const TileXRCcuInstr& instr, uint32_t slot) + { + return static_cast((instr.words[slot / 4U] >> ((slot % 4U) * 16U)) & 0xffffU); + } + + uint64_t Immediate(const TileXRCcuInstr& instr) + { + return (instr.words[0] >> 32U) | (instr.words[1] << 32U); + } + + TileXRCcuAllToAllMeshPeerSpec Peer(uint32_t localRank, uint32_t peerRank, uint16_t ordinal) + { + TileXRCcuAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + auto& route = peer.route; + route.localRank = localRank; + route.localSendAddr = 0x10000000ULL; + route.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + route.localRecvAddr = 0x20000000ULL; + route.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + route.remoteRecvAddr = 0x30000000ULL + static_cast(peerRank) * 0x1000000ULL; + route.remoteRecvToken = TileXRCcuPackMemoryToken(10 + peerRank, 20 + peerRank, true); + route.bytes = 2ULL * 1024ULL * 1024ULL; + route.localGsa = static_cast(0x100 + ordinal * 2); + route.remoteGsa = static_cast(0x101 + ordinal * 2); + route.localXn = static_cast(0x200 + ordinal * 8); + route.remoteXn = static_cast(0x201 + ordinal * 8); + route.lengthXn = static_cast(0x202 + ordinal * 8); + route.preSyncLocalAddrXn = static_cast(0x203 + ordinal * 8); + route.preSyncLocalTokenXn = static_cast(0x204 + ordinal * 8); + route.preSyncLocalMarkerXn = static_cast(0x205 + ordinal * 8); + route.preSyncRemoteMarkerXn = static_cast(0x300 + ordinal * 3); + route.preSyncRemoteAddrXn = static_cast(0x301 + ordinal * 3); + route.preSyncRemoteTokenXn = static_cast(0x302 + ordinal * 3); + route.preSyncMarkerArgIndex = 0; + route.preSyncMarkerEnabled = true; + route.preSyncChannelId = static_cast(0x10 + ordinal * 3); + route.preSyncTokenChannelId = static_cast(0x11 + ordinal * 3); + route.copyChannelId = static_cast(0x12 + ordinal * 3); + route.postSyncChannelId = route.preSyncChannelId; + route.copyCompletionCke = static_cast(0x400 + ordinal * 4); + route.preSyncLocalWaitCke = static_cast(0x401 + ordinal * 4); + route.preSyncRemoteNotifyCke = static_cast(0x500 + ordinal * 3); + route.preSyncRemoteTokenNotifyCke = static_cast(0x501 + ordinal * 3); + route.postSyncLocalWaitCke = static_cast(0x402 + ordinal * 4); + route.postSyncRemoteNotifyCke = static_cast(0x502 + ordinal * 3); + route.sourceCke = static_cast(0x403 + ordinal * 4); + route.ckeMask = 1; + return peer; + } + + int main() + { + TileXRCcuAllToAllMeshProgramSpec spec; + spec.rankSize = 4; + spec.localRank = 2; + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.localRecvAddr = 0x20000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + spec.chunkBytes = 2ULL * 1024ULL * 1024ULL; + spec.selfSourceGsa = 0x180; + spec.selfDestinationGsa = 0x181; + spec.selfSourceXn = 0x280; + spec.selfDestinationXn = 0x281; + spec.selfLengthXn = 0x282; + spec.selfChannelId = 0x30; + spec.selfCompletionCke = 0x480; + spec.peers = {Peer(2, 3, 2), Peer(2, 0, 0), Peer(2, 1, 1)}; + + std::vector program; + TileXRCcuAllToAllProgramReport report; + const int ret = TileXRCcuBuildAllToAllMeshProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 1; + } + if (report.peerCount != 3 || report.syncResourceCount != 9 || + report.remoteBlockCount != 192 || report.selfBlockCount != 64 || + report.preSyncInstructionCount != 21 || report.copyInstructionCount != 1792 || + report.postSyncInstructionCount != 9 || report.finishInstructionCount != 1 || + report.totalInstructionCount != 1823 || program.size() != 1823) { + std::cerr << "unexpected mesh counts total=" << program.size() << "\n"; + return 2; + } + // Three peers publish marker/address/token (6 instructions each) before any wait. + for (uint32_t i = 0; i < 18; ++i) { + if (Slot(program[i], 0) == 0x0802U || Slot(program[i], 0) == 0x0804U) { + std::cerr << "wait appeared before all peer posts\n"; + return 3; + } + } + for (uint32_t i = 18; i < 21; ++i) { + if (Slot(program[i], 0) != 0x0802U || Slot(program[i], 5) != 0x7U) { + std::cerr << "missing presync wait mask\n"; + return 4; + } + } + // Sorted peer 0 copy: send[target=0] -> recv_peer0[source=2]. + if (Immediate(program[21]) != spec.localSendAddr || + Immediate(program[23]) != 0x30000000ULL + 2ULL * spec.chunkBytes || + Slot(program[26], 0) != 0x1009U) { + std::cerr << "unexpected first remote copy offsets\n"; + return 5; + } + const uint32_t selfStart = 21 + 3 * 64 * 7; + const uint64_t selfOffset = 2ULL * spec.chunkBytes; + if (Immediate(program[selfStart]) != spec.localSendAddr + selfOffset || + Immediate(program[selfStart + 2]) != spec.localRecvAddr + selfOffset || + Slot(program[selfStart + 5], 0) != 0x100aU) { + std::cerr << "unexpected CCU self copy offsets\n"; + return 6; + } + auto duplicate = spec; + duplicate.peers[1].route.copyChannelId = duplicate.peers[0].route.copyChannelId; + if (TileXRCcuBuildAllToAllMeshProgram(duplicate, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("duplicate") == std::string::npos) { + std::cerr << "duplicate peer resource accepted: " << report.message << "\n"; + return 7; + } + for (uint32_t localRank = 0; localRank < 4; ++localRank) { + auto rankSpec = spec; + rankSpec.localRank = localRank; + rankSpec.peers.clear(); + uint16_t ordinal = 0; + for (uint32_t peerRank = 0; peerRank < 4; ++peerRank) { + if (peerRank != localRank) { + rankSpec.peers.push_back(Peer(localRank, peerRank, ordinal++)); + } + } + if (TileXRCcuBuildAllToAllMeshProgram(rankSpec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "rank " << localRank << " rejected: " << report.message << "\n"; + return 8; + } + const uint64_t rankOffset = static_cast(localRank) * rankSpec.chunkBytes; + if (program.size() != 1823 || + Immediate(program[selfStart]) != rankSpec.localSendAddr + rankOffset || + Immediate(program[selfStart + 2]) != rankSpec.localRecvAddr + rankOffset) { + std::cerr << "rank " << localRank << " self offset mismatch\n"; + return 9; + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + if __name__ == "__main__": unittest.main() From b65444ad02ab3e8d5f7516d47dd487df6e69276b Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 18:01:24 +0800 Subject: [PATCH 32/41] feat(ccu): package four-rank all-to-all resources --- src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 29 +- .../ccu/tilexr_ccu_direct_orchestrator.cpp | 319 +++++++++++++++++- src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 23 ++ .../test_tilexr_ccu_direct_orchestrator.py | 104 +++++- 4 files changed, 458 insertions(+), 17 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index bbb3739e..8f8fc4b6 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -412,6 +412,23 @@ bool InsertUnique(std::set* ids, std::initializer_list value return true; } +bool InsertPeerUnique(std::set* ids, std::initializer_list values) +{ + std::set peerIds; + for (const uint16_t value : values) { + if (value == 0) { + return false; + } + peerIds.insert(value); + } + for (const uint16_t value : peerIds) { + if (!ids->insert(value).second) { + return false; + } + } + return true; +} + int ValidateMeshSpec( const TileXRCcuAllToAllMeshProgramSpec& spec, std::vector* program, @@ -434,7 +451,6 @@ int ValidateMeshSpec( return Fail(program, report, "missing direct CCU alltoall mesh self-copy resource"); } bool peerRanks[4] = {}; - std::set gsaIds; std::set localXnIds; std::set remoteXnIds; std::set channelIds; @@ -452,19 +468,18 @@ int ValidateMeshSpec( !peer.route.postSyncNotify || !peer.route.postSyncWait) { return Fail(program, report, "invalid direct CCU alltoall mesh peer route"); } - if (!InsertUnique(&gsaIds, {peer.route.localGsa, peer.route.remoteGsa}) || - !InsertUnique(&localXnIds, + if (!InsertPeerUnique(&localXnIds, {peer.route.localXn, peer.route.lengthXn, peer.route.preSyncLocalAddrXn, peer.route.preSyncLocalTokenXn, peer.route.preSyncLocalMarkerXn}) || - !InsertUnique(&remoteXnIds, + !InsertPeerUnique(&remoteXnIds, {peer.route.remoteXn, peer.route.preSyncRemoteAddrXn, peer.route.preSyncRemoteTokenXn, peer.route.preSyncRemoteMarkerXn}) || !InsertUnique(&channelIds, {peer.route.preSyncChannelId, peer.route.preSyncTokenChannelId, peer.route.copyChannelId}) || - !InsertUnique(&localCkeIds, + !InsertPeerUnique(&localCkeIds, {peer.route.copyCompletionCke, peer.route.preSyncLocalWaitCke, - peer.route.postSyncLocalWaitCke, peer.route.sourceCke}) || - !InsertUnique(&remoteCkeIds, + peer.route.postSyncLocalWaitCke}) || + !InsertPeerUnique(&remoteCkeIds, {peer.route.preSyncRemoteNotifyCke, peer.route.preSyncRemoteTokenNotifyCke, peer.route.postSyncRemoteNotifyCke})) { return Fail(program, report, "duplicate direct CCU alltoall mesh peer resource"); diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index d13fc152..d94b28fe 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -27,6 +27,7 @@ constexpr uint16_t TILEXR_CCU_TRACE_SET_CKE_HEADER = 0x0802U; constexpr uint16_t TILEXR_CCU_TRACE_CLEAR_CKE_HEADER = 0x0804U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALID_SHIFT = 52ULL; @@ -39,11 +40,63 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = 7U + 64U * 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT = 9U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; +uint32_t DirectAllToAllMeshInstructionCount(uint64_t chunkBytes) +{ + if (chunkBytes == 0 || chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return 0; + } + const uint64_t blocks = chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint64_t instructions = 21ULL + 4ULL * blocks * 7ULL + 9ULL + 1ULL; + return instructions > std::numeric_limits::max() ? 0U : static_cast(instructions); +} + +bool DirectAllToAllMeshCapacityFits( + const TileXRCcuResourceSpec& resources, + uint32_t instructionCount, + std::string* message) +{ + const auto require = [message](const char* resource, uint32_t requested, uint32_t available) { + if (requested <= available) { + return true; + } + if (message != nullptr) { + std::ostringstream stream; + stream << "insufficient alltoall mesh " << resource + << " resources requested=" << requested + << " available=" << available; + *message = stream.str(); + } + return false; + }; + const uint32_t missionInstructionStart = resources.missionInstructionStartId == 0 ? + resources.instructionStartId : resources.missionInstructionStartId; + const uint32_t repositoryPrefix = missionInstructionStart - resources.instructionStartId; + const uint32_t localWaitCkeCount = resources.localWaitCkeCount == 0 ? + resources.ckeCount : resources.localWaitCkeCount; + const uint32_t remoteNotifyCkeCount = resources.remoteNotifyCkeCount == 0 ? + resources.ckeCount : resources.remoteNotifyCkeCount; + return require("mission", 1U, resources.missionCount) && + require("instruction", repositoryPrefix + instructionCount, resources.instructionCount) && + require("GSA", TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT, resources.gsaCount) && + require("local XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.xnCount) && + require("remote XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + resources.remoteXnCount == 0 ? + (resources.xnCount > TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT ? + resources.xnCount - TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : 0U) : + resources.remoteXnCount) && + require("local CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT + 1U, + localWaitCkeCount) && + require("remote CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + remoteNotifyCkeCount) && + require("channel", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.channelCount); +} + void ResetReport(TileXRCcuDirectInstallReport* report) { if (report != nullptr) { @@ -395,6 +448,27 @@ void TraceDecodedInstr(const char* label, size_t index, const TileXRCcuInstr& in << " waitCkeMask=" << TraceSlot(instr.words[3], 3); break; } + case TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER: { + const uint16_t control = TraceSlot(instr.words[1], 3); + const uint16_t flags = TraceSlot(instr.words[2], 3); + std::cerr << "decoded=TransLocMemToLocMem" + << " sourceGsa=" << TraceSlot(instr.words[0], 3) + << " sourceXn=" << TraceSlot(instr.words[1], 0) + << " destinationGsa=" << TraceSlot(instr.words[0], 1) + << " destinationXn=" << TraceSlot(instr.words[0], 2) + << " lengthXn=" << TraceSlot(instr.words[1], 1) + << " channelId=" << TraceSlot(instr.words[1], 2) + << " clearType=" << (flags & 0x1U) + << " lengthEn=" << ((flags >> 1U) & 0x1U) + << " reduceEn=" << ((flags >> 2U) & 0x1U) + << " reduceDataType=" << ((control >> 8U) & 0xfU) + << " reduceOpCode=" << ((control >> 12U) & 0xfU) + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + } case TILEXR_CCU_TRACE_SYNC_XN_HEADER: std::cerr << "decoded=SyncXn" << " remoteXn=" << TraceSlot(instr.words[0], 1) @@ -804,6 +878,48 @@ int ConfigureDirectAllToAll2RankResources( return TILEXR_SUCCESS; } +int ConfigureDirectAllToAllMeshResources( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + const uint32_t instructionCount = DirectAllToAllMeshInstructionCount(alltoall.chunkBytes); + if (attempt == nullptr || instructionCount == 0 || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.taskWindows.size() != 1U) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU plan requires nine sync resources and one task"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (options.gsaStartId == 0 || attempt->resourceSpec.gsaCount < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU requires two kernel-local GSA resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (attempt->allocation.localXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.remoteXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.localWaitCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.remoteNotifyCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.channels.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.sourceCke.num != 1U) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU allocation is missing XN/CKE/channel resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->allocation.localGsa = MakeRange( + attempt->specInfo.dieId, + options.gsaStartId, + static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT)); + attempt->plan.kernelLocalGsa = attempt->allocation.localGsa; + attempt->plan.taskWindows[0].instCnt = static_cast(instructionCount); + return TILEXR_SUCCESS; +} + int BuildDirectMemoryCopyLaunchPackage( const TileXRCcuDirectMemoryCopySpec& memoryCopy, TileXRCcuDirectInstallAttempt* attempt, @@ -1071,6 +1187,135 @@ int BuildDirectAllToAll2RankLaunchPackage( return TILEXR_SUCCESS; } +int BuildDirectAllToAllMeshLaunchPackage( + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall mesh producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + auto peers = alltoall.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuDirectAllToAllMeshPeerSpec& lhs, + const TileXRCcuDirectAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + TileXRCcuAllToAllMeshProgramSpec mesh; + mesh.rankSize = alltoall.rankSize; + mesh.localRank = alltoall.localRank; + mesh.localSendAddr = alltoall.localSendAddr; + mesh.localSendToken = alltoall.localSendToken; + mesh.localRecvAddr = alltoall.localRecvAddr; + mesh.localRecvToken = alltoall.localRecvToken; + mesh.chunkBytes = alltoall.chunkBytes; + mesh.selfSourceGsa = attempt->plan.kernelLocalGsa.startId; + mesh.selfDestinationGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + + for (uint32_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const uint32_t base = ordinal * 3U; + const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[base]; + const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[base + 1U]; + const TileXRCcuSyncResource& tokenResource = attempt->plan.syncResources[base + 2U]; + TileXRCcuAllToAllMeshPeerSpec peer; + peer.peerRank = peers[ordinal].peerRank; + auto& route = peer.route; + route.localRank = alltoall.localRank; + route.localSendAddr = alltoall.localSendAddr; + route.localSendToken = alltoall.localSendToken; + route.localRecvAddr = alltoall.localRecvAddr; + route.localRecvToken = alltoall.localRecvToken; + route.remoteRecvAddr = peers[ordinal].remoteRecvAddr; + route.remoteRecvToken = peers[ordinal].remoteRecvToken; + route.bytes = alltoall.chunkBytes; + route.localGsa = attempt->plan.kernelLocalGsa.startId; + route.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + route.localXn = copyResource.localXn; + route.remoteXn = copyResource.remoteXn; + route.lengthXn = tokenResource.localXn; + route.preSyncLocalAddrXn = preResource.localXn; + route.preSyncLocalTokenXn = tokenResource.localXn; + route.preSyncLocalMarkerXn = copyResource.localXn; + route.preSyncRemoteMarkerXn = copyResource.remoteXn; + route.preSyncRemoteAddrXn = preResource.remoteXn; + route.preSyncRemoteTokenXn = tokenResource.remoteXn; + route.preSyncMarkerArgIndex = 0; + route.preSyncMarkerEnabled = true; + route.preSyncChannelId = preResource.channelId; + route.preSyncTokenChannelId = tokenResource.channelId; + route.copyChannelId = copyResource.channelId; + route.postSyncChannelId = copyResource.channelId; + route.copyCompletionCke = copyResource.localWaitCke; + route.preSyncLocalWaitCke = preResource.localWaitCke; + route.preSyncRemoteNotifyCke = preResource.notifyCke; + route.preSyncTokenLocalWaitCke = tokenResource.localWaitCke; + route.preSyncRemoteTokenNotifyCke = preResource.notifyCke; + route.postSyncLocalWaitCke = copyResource.localWaitCke; + route.postSyncRemoteNotifyCke = copyResource.notifyCke; + route.sourceCke = attempt->allocation.sourceCke.startId; + route.ckeMask = 1U; + route.preSyncNotify = true; + route.preSyncWait = true; + route.postSyncNotify = true; + route.postSyncWait = true; + route.emitFinish = false; + mesh.peers.push_back(peer); + } + const TileXRCcuSyncResource& selfCopy = attempt->plan.syncResources[0]; + const TileXRCcuSyncResource& selfPre = attempt->plan.syncResources[1]; + const TileXRCcuSyncResource& selfToken = attempt->plan.syncResources[2]; + mesh.selfSourceXn = selfCopy.localXn; + mesh.selfDestinationXn = selfToken.localXn; + mesh.selfLengthXn = selfPre.localXn; + mesh.selfChannelId = selfCopy.channelId; + mesh.selfCompletionCke = selfCopy.localWaitCke; + + TileXRCcuProgram program; + TileXRCcuAllToAllProgramReport alltoallReport; + if (TileXRCcuBuildAllToAllMeshProgram(mesh, &program.sync, &alltoallReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = alltoallReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (program.sync.empty() || program.sync.size() > std::numeric_limits::max()) { + if (report != nullptr) { + report->message = "invalid direct CCU alltoall mesh instruction count"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + uint32_t SignalWaitInstructionCount(TileXRCcuSignalWaitProgramRole role) { if (role == TileXRCcuSignalWaitProgramRole::Wait) { @@ -1520,6 +1765,7 @@ int RunDirectInstallAttemptImpl( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectMemoryCopySpec* memoryCopy, const TileXRCcuDirectAllToAll2RankSpec* alltoall, + const TileXRCcuDirectAllToAllMeshSpec* alltoallMesh, const TileXRCcuDirectSignalWaitSpec* signalWait, const TileXRCcuDirectSyncXnPingSpec* syncXnPing, TileXRCcuDirectInstallAttempt* attempt, @@ -1565,11 +1811,21 @@ int RunDirectInstallAttemptImpl( attempt->resourceSpec.missionInstructionStartId = options.missionInstructionStartId; ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); + if (alltoallMesh != nullptr) { + std::string capacityMessage; + if (!DirectAllToAllMeshCapacityFits( + attempt->resourceSpec, + DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes), + &capacityMessage)) { + return Fail(attempt, report, capacityMessage); + } + } - const bool customProgram = - memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr || syncXnPing != nullptr; + const bool customProgram = memoryCopy != nullptr || alltoall != nullptr || alltoallMesh != nullptr || + signalWait != nullptr || syncXnPing != nullptr; attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = + alltoallMesh != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : customProgram ? 1U : options.syncResourceCount; attempt->resourceRequest.syncInstructionCount = @@ -1577,6 +1833,9 @@ int RunDirectInstallAttemptImpl( std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : alltoall != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : + alltoallMesh != nullptr ? + std::max(options.syncInstructionCount, + DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes)) : signalWait != nullptr ? std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : syncXnPing != nullptr ? @@ -1584,6 +1843,7 @@ int RunDirectInstallAttemptImpl( options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; attempt->resourceRequest.barrierMode = + alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); @@ -1623,8 +1883,19 @@ int RunDirectInstallAttemptImpl( "failed to configure direct CCU alltoall resources" : report->message); } + } else if (alltoallMesh != nullptr) { + ret = ConfigureDirectAllToAllMeshResources(options, *alltoallMesh, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to configure direct CCU alltoall mesh resources" : + report->message); + } } attempt->plan.barrierMode = + alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); @@ -1658,6 +1929,8 @@ int RunDirectInstallAttemptImpl( TileXRCcuLaunchPackageReport packageReport; ret = memoryCopy != nullptr ? BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report) : + alltoallMesh != nullptr ? + BuildDirectAllToAllMeshLaunchPackage(*alltoallMesh, attempt, report) : alltoall != nullptr ? BuildDirectAllToAll2RankLaunchPackage(*alltoall, attempt, report) : signalWait != nullptr ? @@ -1673,6 +1946,8 @@ int RunDirectInstallAttemptImpl( (report == nullptr || report->message.empty() ? (memoryCopy != nullptr ? "failed to build direct CCU memory copy launch package" : + alltoallMesh != nullptr ? + "failed to build direct CCU alltoall mesh launch package" : alltoall != nullptr ? "failed to build direct CCU alltoall launch package" : syncXnPing != nullptr ? @@ -1740,7 +2015,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1759,7 +2034,7 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectAllToAll2RankInstallAttempt( @@ -1783,7 +2058,37 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, nullptr, attempt, report); +} + +int TileXRCcuRunDirectAllToAllMeshInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + bool peerRanks[4] = {}; + bool valid = alltoall.rankSize == 4U && alltoall.localRank < alltoall.rankSize && + alltoall.localSendAddr != 0 && alltoall.localSendToken != 0 && + alltoall.localRecvAddr != 0 && alltoall.localRecvToken != 0 && + DirectAllToAllMeshInstructionCount(alltoall.chunkBytes) != 0 && alltoall.peers.size() == 3U; + for (const auto& peer : alltoall.peers) { + if (peer.peerRank >= alltoall.rankSize || peer.peerRank == alltoall.localRank || + peerRanks[peer.peerRank] || peer.remoteRecvAddr == 0 || peer.remoteRecvToken == 0) { + valid = false; + break; + } + peerRanks[peer.peerRank] = true; + } + if (!valid) { + ResetReport(report); + ClearAttempt(attempt); + if (report != nullptr) { + report->message = "invalid direct CCU alltoall mesh address/token/rank inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &alltoall, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectSignalWaitInstallAttempt( @@ -1792,7 +2097,7 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &signalWait, nullptr, attempt, report); } int TileXRCcuRunDirectSyncXnPingInstallAttempt( @@ -1801,7 +2106,7 @@ int TileXRCcuRunDirectSyncXnPingInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index dd5a2ee5..a98b945f 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -85,6 +85,23 @@ struct TileXRCcuDirectAllToAll2RankSpec { uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; }; +struct TileXRCcuDirectAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; +}; + +struct TileXRCcuDirectAllToAllMeshSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 0; + std::vector peers; +}; + struct TileXRCcuDirectSignalWaitSpec { TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; bool overrideBarrierMode = false; @@ -161,6 +178,12 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectAllToAllMeshInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuRunDirectSignalWaitInstallAttempt( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectSignalWaitSpec& signalWait, diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index 006553b3..f19ded29 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1741,13 +1741,19 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("decoded=LoadImdToGSA", source) self.assertIn("decoded=TransRmtMemToLocMem", source) self.assertIn("decoded=TransLocMemToRmtMem", source) + self.assertIn("decoded=TransLocMemToLocMem", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER", source) + self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER", source) def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") planner = PLANNER_SOURCE.read_text(encoding="utf-8") + two_rank_body = source[ + source.index("int BuildDirectAllToAll2RankLaunchPackage"): + source.index("int BuildDirectAllToAllMeshLaunchPackage") + ] self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", source) @@ -1813,9 +1819,9 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): ) self.assertIn("postSyncWait = false", source) self.assertIn("emitFinish = false", source) - self.assertNotIn("postSyncNotify = true", source) - self.assertNotIn("postSyncWait = true", source) - self.assertNotIn("emitFinish = true", source) + self.assertNotIn("postSyncNotify = true", two_rank_body) + self.assertNotIn("postSyncWait = true", two_rank_body) + self.assertNotIn("emitFinish = true", two_rank_body) self.assertIn("LocalToRemote", source) self.assertIn("uint32_t memSlicePerBlock", header) @@ -1824,6 +1830,98 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): with self.subTest(needle=needle): self.assertNotIn(needle, combined) + def test_direct_four_rank_mesh_builds_one_nine_route_launch_package(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 4095U; + basic.caps.cap1 = (63U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 63U; + basic.caps.cap3 = (127U << 16) | 31U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.deviceId = 4; + options.rank = 2; + options.provider = "unit-test-direct-alltoall-mesh"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.missionInstructionStartId = 489; + options.xnStartId = 1961; + options.gsaStartId = 510; + options.ckeStartId = 332; + options.channelStartId = 2; + options.offlineOnly = true; + + TileXRCcuDirectAllToAllMeshSpec mesh; + mesh.rankSize = 4; + mesh.localRank = 2; + mesh.localSendAddr = 0x10000000ULL; + mesh.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + mesh.localRecvAddr = 0x20000000ULL; + mesh.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + mesh.chunkBytes = 2ULL * 1024ULL * 1024ULL; + for (uint32_t peerRank : {3U, 0U, 1U}) { + TileXRCcuDirectAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + peer.remoteRecvAddr = 0x30000000ULL + peerRank * 0x1000000ULL; + peer.remoteRecvToken = TileXRCcuPackMemoryToken(10 + peerRank, 20 + peerRank, true); + mesh.peers.push_back(peer); + } + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh, &attempt, &report); + (void)ret; + if (!report.pipelineBuilt || attempt.plan.syncResources.size() != 9 || + attempt.plan.taskWindows.size() != 1 || attempt.package.tasks.size() != 1 || + attempt.package.program.sync.size() != 1823 || + attempt.plan.taskWindows[0].instCnt != 1823 || + attempt.plan.kernelLocalGsa.num != 2 || attempt.allocation.sourceCke.num != 1 || + attempt.plan.barrierMode != TileXRCcuBarrierMode::SyncCke) { + std::cerr << "unexpected mesh package: " << report.message + << " resources=" << attempt.plan.syncResources.size() + << " instructions=" << attempt.package.program.sync.size() << "\n"; + return 1; + } + basic.caps.cap0 = (7U << 24) | (11U << 16) | 1799U; + TileXRCcuDirectInstallAttempt smallAttempt; + TileXRCcuDirectInstallReport smallReport; + if (TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh, &smallAttempt, &smallReport) == TILEXR_SUCCESS || + smallReport.message.find("instruction") == std::string::npos || + smallReport.message.find("requested=") == std::string::npos || + smallReport.message.find("available=") == std::string::npos) { + std::cerr << "missing mesh capacity diagnostics: " << smallReport.message << "\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_direct_install_options_default_to_lower_layer_first(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") From 043da132038711c16ab427a3e37602cfbc3c84be Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 18:15:52 +0800 Subject: [PATCH 33/41] feat(ccu): plan four-rank all-to-all endpoints --- .../ccu/tilexr_ccu_collective_planner.cpp | 284 +++++++++++++++--- src/comm/ccu/tilexr_ccu_collective_planner.h | 18 +- tests/ccu/test_tilexr_ccu_backend_boundary.py | 42 ++- 3 files changed, 291 insertions(+), 53 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index f4659250..7d8caa88 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef TILEXR_CCU_TESTING #include "runtime/dev.h" @@ -553,13 +554,14 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } const size_t peerRouteCount = static_cast(rankSize - 1); const size_t syncRouteCount = allocation.remoteXn.num; + const size_t routesPerPeer = syncRouteCount / peerRouteCount; if (allocation.localXn.num == 0 || allocation.localWaitCke.num == 0 || allocation.remoteNotifyCke.num == 0 || allocation.remoteXn.num < static_cast(rankSize - 1) || allocation.localWaitCke.num < allocation.remoteXn.num || allocation.remoteNotifyCke.num < allocation.remoteXn.num || - allocation.channels.num == 0 || + allocation.channels.num == 0 || routesPerPeer == 0 || syncRouteCount % peerRouteCount != 0 || remoteCcuBuffers->size() != peerRouteCount) { if (report != nullptr) { report->message = "invalid direct CCU peer XN/CKE exchange shape"; @@ -618,15 +620,16 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( size_t routeIndex = 0; for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { - const size_t peerBufferIndex = syncIndex % peerRouteCount; + const size_t peerBufferIndex = syncIndex / routesPerPeer; + const size_t routeWithinPeer = syncIndex % routesPerPeer; const int peer = peerRanks[peerBufferIndex]; const PeerResourceExchange &peerResources = all[peer]; const size_t peerLocalIndex = static_cast(rank < peer ? rank : rank - 1); - const uint32_t peerLocalXnOffset = - SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t selectedRemoteXnOffset = - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t peerLocalWaitCkeOffset = routeIndex; + const uint32_t peerLocalResourceOffset = static_cast( + peerLocalIndex * routesPerPeer + routeWithinPeer); + const uint32_t peerLocalXnOffset = peerLocalResourceOffset; + const uint32_t selectedRemoteXnOffset = peerLocalResourceOffset; + const uint32_t peerLocalWaitCkeOffset = peerLocalResourceOffset; if (peerResources.localXnCount == 0 || peerResources.remoteXnCount == 0 || peerResources.localWaitCkeCount == 0 || @@ -634,7 +637,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( peerResources.channelCount == 0 || peerLocalXnOffset >= peerResources.localXnCount || selectedRemoteXnOffset >= peerResources.remoteXnCount || - peerLocalIndex >= peerResources.channelCount || + peerLocalResourceOffset >= peerResources.channelCount || peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount || peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount) { if (report != nullptr) { @@ -642,11 +645,8 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } return TILEXR_ERROR_NOT_FOUND; } - uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( - peerResources.remoteXnStartId, - peerLocalIndex, - syncIndex, - peerRouteCount); + uint16_t channelBoundRemoteXnId = static_cast( + static_cast(peerResources.remoteXnStartId) + peerLocalResourceOffset); const uint16_t peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); uint16_t remoteNotifyCke = @@ -672,7 +672,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && routeIndex < allocation.channels.num && peerResources.channelStartId != 0 && - peerLocalIndex < peerResources.channelCount; + peerLocalResourceOffset < peerResources.channelCount; const bool transportResourceExchangeVerified = notifyCkeOwnerVerified && allocation.localWaitCke.num != 0 && @@ -693,14 +693,17 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) { - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - 0, - peerRank, - remoteCcuVa, - memoryTokenId, - rawMemoryTokenId, - memoryTokenValue); - directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + directCcuRemoteRouteMemoryOverrides_.clear(); + DirectCcuRemoteRouteMemoryOverride override; + override.allRoutes = true; + override.buffer.peerRank = peerRank; + override.buffer.remoteCcuVa = remoteCcuVa; + override.buffer.memoryTokenId = memoryTokenId; + override.buffer.rawMemoryTokenId = rawMemoryTokenId; + override.buffer.memoryTokenValue = memoryTokenValue; + if (remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0) { + directCcuRemoteRouteMemoryOverrides_.push_back(override); + } } void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( @@ -711,44 +714,49 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRou uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) { - directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; - directCcuRemoteRouteMemoryOverride_.peerRank = peerRank; - directCcuRemoteRouteMemoryOverride_.remoteCcuVa = remoteCcuVa; - directCcuRemoteRouteMemoryOverride_.memoryTokenId = memoryTokenId; - directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId = rawMemoryTokenId; - directCcuRemoteRouteMemoryOverride_.memoryTokenValue = memoryTokenValue; - directCcuRemoteRouteMemoryOverrideValid_ = - remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; - directCcuRemoteRouteMemoryOverrideAllRoutes_ = false; - directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = syncRouteIndex; + DirectCcuRemoteRouteMemoryOverride override; + override.syncRouteIndex = syncRouteIndex; + override.buffer.peerRank = peerRank; + override.buffer.remoteCcuVa = remoteCcuVa; + override.buffer.memoryTokenId = memoryTokenId; + override.buffer.rawMemoryTokenId = rawMemoryTokenId; + override.buffer.memoryTokenValue = memoryTokenValue; + if (remoteCcuVa == 0 || memoryTokenId == 0 || memoryTokenValue == 0) { + return; + } + for (auto &existing : directCcuRemoteRouteMemoryOverrides_) { + if (!existing.allRoutes && existing.syncRouteIndex == syncRouteIndex) { + existing = override; + return; + } + } + directCcuRemoteRouteMemoryOverrides_.push_back(override); } void TileXRCcuCollectivePlanner::ClearDirectCcuRemoteRouteMemoryOverride() { - directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; - directCcuRemoteRouteMemoryOverrideValid_ = false; - directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; - directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; + directCcuRemoteRouteMemoryOverrides_.clear(); } void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( std::vector *remoteCcuBuffers) const { - if (!directCcuRemoteRouteMemoryOverrideValid_ || remoteCcuBuffers == nullptr) { + if (directCcuRemoteRouteMemoryOverrides_.empty() || remoteCcuBuffers == nullptr) { return; } uint32_t routeIndex = 0; for (auto &remoteCcuBuffer : *remoteCcuBuffers) { - if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank || - (!directCcuRemoteRouteMemoryOverrideAllRoutes_ && - routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_)) { - ++routeIndex; - continue; + for (const auto &override : directCcuRemoteRouteMemoryOverrides_) { + if (remoteCcuBuffer.peerRank != override.buffer.peerRank || + (!override.allRoutes && override.syncRouteIndex != routeIndex)) { + continue; + } + remoteCcuBuffer.remoteCcuVa = override.buffer.remoteCcuVa; + remoteCcuBuffer.memoryTokenId = override.buffer.memoryTokenId; + remoteCcuBuffer.rawMemoryTokenId = override.buffer.rawMemoryTokenId; + remoteCcuBuffer.memoryTokenValue = override.buffer.memoryTokenValue; + break; } - remoteCcuBuffer.remoteCcuVa = directCcuRemoteRouteMemoryOverride_.remoteCcuVa; - remoteCcuBuffer.memoryTokenId = directCcuRemoteRouteMemoryOverride_.memoryTokenId; - remoteCcuBuffer.rawMemoryTokenId = directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId; - remoteCcuBuffer.memoryTokenValue = directCcuRemoteRouteMemoryOverride_.memoryTokenValue; ++routeIndex; } } @@ -1380,6 +1388,190 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( return ret; } +int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t chunkBytes, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU alltoall mesh install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 4 || rank < 0 || rank >= rankSize || localSourceAddr == 0 || localDestinationAddr == 0 || + chunkBytes == 0 || chunkBytes > std::numeric_limits::max() / 4ULL) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall mesh endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const uint64_t bufferBytes = chunkBytes * 4ULL; + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int refreshRet = session.RefreshDirectCcuBasicInfo(installDieId); + if (refreshRet != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return refreshRet; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for alltoall mesh install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before alltoall mesh buffers"; + } + return ret; + } + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bufferBytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU alltoall mesh local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather(&localEndpoint, sizeof(localEndpoint), allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU alltoall mesh endpoints"; + } + return ret; + } + + TileXRCcuDirectAllToAllMeshSpec alltoall; + alltoall.rankSize = static_cast(rankSize); + alltoall.localRank = static_cast(rank); + alltoall.localSendAddr = localEndpoint.sourceAddr; + alltoall.localSendToken = localEndpoint.sourceToken; + alltoall.localRecvAddr = localEndpoint.destinationAddr; + alltoall.localRecvToken = localEndpoint.destinationToken; + alltoall.chunkBytes = chunkBytes; + struct ImportedPeer { + uint32_t peerRank = 0; + TileXRCcuImportedRemoteMemoryBufferInfo imported; + TileXRCcuRemoteMemoryBufferImportRequest request; + }; + std::vector importedPeers; + importedPeers.reserve(3U); + for (uint32_t peerRank = 0; peerRank < static_cast(rankSize); ++peerRank) { + if (peerRank == static_cast(rank)) { + continue; + } + const DirectCcuMemoryCopyEndpoint &endpoint = allEndpoints[peerRank]; + if (endpoint.valid == 0 || endpoint.rank != peerRank || endpoint.bytes != bufferBytes || + !endpoint.destinationRemoteImport.valid) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall mesh peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + ImportedPeer importedPeer; + importedPeer.peerRank = peerRank; + importedPeer.request = endpoint.destinationRemoteImport; + ret = session.ImportRemoteMemoryBuffer(importedPeer.request, &importedPeer.imported); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU alltoall mesh remote destination"; + } + return ret; + } + TileXRCcuDirectAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + peer.remoteRecvAddr = importedPeer.request.addr; + peer.remoteRecvToken = TileXRCcuPackMemoryToken( + importedPeer.request.tokenId, importedPeer.request.tokenValue, true); + alltoall.peers.push_back(peer); + importedPeers.push_back(importedPeer); + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 9U; + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-alltoall-mesh"; + } + + ClearDirectCcuRemoteRouteMemoryOverride(); + for (uint32_t peerOrdinal = 0; peerOrdinal < importedPeers.size(); ++peerOrdinal) { + const ImportedPeer &peer = importedPeers[peerOrdinal]; + for (uint32_t routeWithinPeer = 0; routeWithinPeer < 3U; ++routeWithinPeer) { + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + peerOrdinal * 3U + routeWithinPeer, + peer.peerRank, + peer.imported.targetSegVa, + peer.request.tokenId, + peer.request.rawTokenId, + peer.request.tokenValue); + } + } + ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt(next, alltoall, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} + int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index f09489ca..182cd533 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -72,6 +72,14 @@ class TileXRCcuCollectivePlanner { uint32_t peerRank, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuAllToAllMeshInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t chunkBytes, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); int PrepareDirectCcuSyncXnPingInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, @@ -140,10 +148,12 @@ class TileXRCcuCollectivePlanner { TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; bool directCcuLocalVerifiedEndpointRouteValid_ = false; #ifdef TILEXR_CCU_TESTING - TileXRCcuRemoteCcuBufferInfo directCcuRemoteRouteMemoryOverride_ = {}; - bool directCcuRemoteRouteMemoryOverrideValid_ = false; - bool directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; - uint32_t directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; + struct DirectCcuRemoteRouteMemoryOverride { + uint32_t syncRouteIndex = 0; + bool allRoutes = false; + TileXRCcuRemoteCcuBufferInfo buffer; + }; + std::vector directCcuRemoteRouteMemoryOverrides_; #endif }; diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 47aefa71..c300fdec 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -235,11 +235,47 @@ def test_alltoall_overrides_only_copy_route_memory_not_sync_routes(self): ] self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(", prepare_alltoall) - self.assertIn("1U", prepare_alltoall) + self.assertIn("0U", prepare_alltoall) self.assertIn("uint32_t routeIndex = 0", override_apply) - self.assertIn("routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_", override_apply) + self.assertIn("override.syncRouteIndex != routeIndex", override_apply) self.assertIn("++routeIndex", override_apply) - self.assertIn("directCcuRemoteRouteMemoryOverrideAllRoutes_", override_apply) + self.assertIn("override.allRoutes", override_apply) + + def test_four_rank_mesh_gathers_imports_and_maps_three_routes_per_peer(self): + header = PLANNER_HEADER.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", header) + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", planner) + mesh_body = planner[ + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt"): + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt") + ] + self.assertIn("rankSize != 4", mesh_body) + self.assertEqual(1, mesh_body.count("session.AllGather(")) + self.assertIn("endpoint.rank != peerRank", mesh_body) + self.assertIn("session.ImportRemoteMemoryBuffer", mesh_body) + self.assertIn("for (uint32_t routeWithinPeer = 0; routeWithinPeer < 3U", mesh_body) + self.assertIn("peerOrdinal * 3U + routeWithinPeer", mesh_body) + self.assertIn("TileXRCcuRunDirectAllToAllMeshInstallAttempt", mesh_body) + + exchange = planner[ + planner.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner.index("void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride") + ] + self.assertIn("routesPerPeer = syncRouteCount / peerRouteCount", exchange) + self.assertIn("peerBufferIndex = syncIndex / routesPerPeer", exchange) + self.assertIn("peerLocalResourceOffset =", exchange) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", exchange) + + self.assertIn("std::vector", header) + self.assertIn("directCcuRemoteRouteMemoryOverrides_", header) + override_apply = planner[ + planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride"): + planner.index("#endif", planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride")) + ] + self.assertIn("for (const auto &override : directCcuRemoteRouteMemoryOverrides_)", override_apply) + self.assertIn("override.syncRouteIndex != routeIndex", override_apply) if __name__ == "__main__": From 1acceecbd495a7a9353b0570a82f88219a2ef952 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 18:26:44 +0800 Subject: [PATCH 34/41] test(ccu): exercise repeated four-rank all-to-all --- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 389 +++++++++++++++++- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 44 ++ 2 files changed, 413 insertions(+), 20 deletions(-) diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index fd4e9b8b..041e24cd 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -79,6 +79,7 @@ constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; constexpr const char* kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"; +constexpr const char* kAllToAllMeshEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH"; constexpr const char* kAllToAllSingleRouteBidirectionalEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL"; constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; @@ -141,6 +142,8 @@ struct AllToAllState { std::vector expected; std::vector observed; size_t bytes = 0; + size_t chunkBytes = 0; + int rankSize = 0; int initRet = ACL_SUCCESS; int readRet = ACL_SUCCESS; uint32_t mismatchCount = 0; @@ -179,6 +182,11 @@ bool AllToAllLongMissionEnabled() return EnvFlag(kAllToAllLongMissionEnv); } +bool AllToAllMeshSmokeEnabled() +{ + return EnvFlag(kAllToAllMeshEnv); +} + bool AllToAllSingleRouteBidirectionalEnabled() { return EnvFlag(kAllToAllSingleRouteBidirectionalEnv); @@ -608,6 +616,97 @@ std::vector BuildAllToAllLoopPattern(int rank, int loopIndex, size_t by return pattern; } +uint8_t BuildAllToAllMeshByte( + uint32_t sourceRank, + uint32_t targetRank, + uint32_t loopIndex, + size_t chunkOffset) +{ + return static_cast( + ((sourceRank + 1U) * 67U + (targetRank + 1U) * 29U + + (loopIndex + 1U) * 17U + chunkOffset * 13U) & 0xffU); +} + +int InitAllToAllMeshState(int rank, int rankSize, AllToAllState* state) +{ + if (state == nullptr || rank < 0 || rank >= rankSize || rankSize != 4) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->chunkBytes = AllToAllBytesFromEnv(); + state->rankSize = rankSize; + state->bytes = static_cast(rankSize) * state->chunkBytes; + if (state->chunkBytes != 2U * 1024U * 1024U || + state->bytes / state->chunkBytes != static_cast(rankSize) || + AllToAllMemSlicePerLoopFromEnv() != 8) { + state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + return state->initRet; + } + state->expected.assign(state->bytes, 0); + state->observed.assign(state->bytes, 0); + std::vector initial(state->bytes, 0xa5U); + int ret = state->source.Allocate(state->bytes); + if (ret == ACL_SUCCESS) { + ret = state->destination.Allocate(state->bytes); + } + if (ret == ACL_SUCCESS) { + ret = aclrtMemcpy( + state->source.ptr, state->bytes, initial.data(), initial.size(), ACL_MEMCPY_HOST_TO_DEVICE); + } + if (ret == ACL_SUCCESS) { + ret = aclrtMemcpy( + state->destination.ptr, state->bytes, initial.data(), initial.size(), ACL_MEMCPY_HOST_TO_DEVICE); + } + state->initRet = ret; + return ret; +} + +int ResetAllToAllMeshStateForLoop(int rank, int loopIndex, AllToAllState* state) +{ + if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || + state->rankSize != 4 || state->bytes != static_cast(state->rankSize) * state->chunkBytes || + loopIndex < 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector source(state->bytes); + state->expected.assign(state->bytes, 0); + for (int targetRank = 0; targetRank < state->rankSize; ++targetRank) { + for (size_t chunkOffset = 0; chunkOffset < state->chunkBytes; ++chunkOffset) { + source[static_cast(targetRank) * state->chunkBytes + chunkOffset] = + BuildAllToAllMeshByte(rank, targetRank, loopIndex, chunkOffset); + } + } + for (int sourceRank = 0; sourceRank < state->rankSize; ++sourceRank) { + for (size_t chunkOffset = 0; chunkOffset < state->chunkBytes; ++chunkOffset) { + state->expected[static_cast(sourceRank) * state->chunkBytes + chunkOffset] = + BuildAllToAllMeshByte(sourceRank, rank, loopIndex, chunkOffset); + } + } + state->observed.assign(state->bytes, 0); + state->readRet = ACL_SUCCESS; + state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + state->passed = false; + std::vector destination( + state->bytes, static_cast(0xa5U ^ static_cast(loopIndex))); + int ret = aclrtMemcpy( + state->source.ptr, state->bytes, source.data(), source.size(), ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + return ret; + } + return aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + destination.size(), + ACL_MEMCPY_HOST_TO_DEVICE); +} + int InitAllToAllState(int rank, int peer, AllToAllState* state) { if (state == nullptr) { @@ -1365,9 +1464,10 @@ void PrintCcuResourceState( DirectCcuSmokeContext* context, uint8_t dieId, const TileXRDirectCcuPrepareOptions& options, - const char* label) + const char* label, + uint32_t resourceCount = 3U) { - if (context == nullptr || label == nullptr) { + if (context == nullptr || label == nullptr || resourceCount == 0) { return; } TileXR::TileXRCcuDriverAdapter adapter; @@ -1379,38 +1479,49 @@ void PrintCcuResourceState( return; } - uint64_t localXn[3] {}; - uint64_t remoteXn[3] {}; - uint64_t localWaitCke[3] {}; - uint64_t remoteNotifyCke[3] {}; + std::vector localXn(resourceCount, 0); + std::vector remoteXn(resourceCount, 0); + std::vector localWaitCke(resourceCount, 0); + std::vector remoteNotifyCke(resourceCount, 0); const uint32_t localXnStartId = options.xnStartId; const uint32_t remoteXnStartId = options.remoteXnStartId; const uint32_t localWaitCkeStartId = options.localWaitCkeStartId; const uint32_t remoteNotifyCkeStartId = options.remoteNotifyCkeStartId; - const int localXnRet = adapter.ReadXnRange(dieId, localXnStartId, localXn, 3, &report); - const int remoteXnRet = adapter.ReadXnRange(dieId, remoteXnStartId, remoteXn, 3, &report); + const int localXnRet = adapter.ReadXnRange( + dieId, localXnStartId, localXn.data(), resourceCount, &report); + const int remoteXnRet = adapter.ReadXnRange( + dieId, remoteXnStartId, remoteXn.data(), resourceCount, &report); const int localCkeRet = adapter.ReadCkeRange( - dieId, localWaitCkeStartId, localWaitCke, 3, &report); + dieId, localWaitCkeStartId, localWaitCke.data(), resourceCount, &report); const int remoteCkeRet = adapter.ReadCkeRange( - dieId, remoteNotifyCkeStartId, remoteNotifyCke, 3, &report); + dieId, remoteNotifyCkeStartId, remoteNotifyCke.data(), resourceCount, &report); + + const auto values = [](const std::vector& data) { + std::ostringstream out; + for (size_t i = 0; i < data.size(); ++i) { + if (i != 0) { + out << ","; + } + out << "0x" << std::hex << data[i]; + } + return out.str(); + }; std::cerr << label << " resourceState" + << " resourceCount=" << resourceCount << " localXnStartId=" << localXnStartId << " localXnRet=" << localXnRet - << " localXn=0x" << std::hex << localXn[0] << ",0x" << localXn[1] << ",0x" << localXn[2] - << std::dec + << " localXn=" << values(localXn) << " remoteXnStartId=" << remoteXnStartId << " remoteXnRet=" << remoteXnRet - << " remoteXn=0x" << std::hex << remoteXn[0] << ",0x" << remoteXn[1] << ",0x" << remoteXn[2] - << std::dec + << " remoteXn=" << values(remoteXn) << " localWaitCkeStartId=" << localWaitCkeStartId << " localCkeRet=" << localCkeRet - << " localCke=0x" << std::hex << localWaitCke[0] << ",0x" << localWaitCke[1] << ",0x" << localWaitCke[2] - << std::dec + << " localCke=" << values(localWaitCke) << " remoteNotifyCkeStartId=" << remoteNotifyCkeStartId << " remoteCkeRet=" << remoteCkeRet - << " remoteCke=0x" << std::hex << remoteNotifyCke[0] << ",0x" << remoteNotifyCke[1] << ",0x" << remoteNotifyCke[2] - << std::dec << std::endl; + << " remoteCke=" << values(remoteNotifyCke) + << std::endl; } int ReadAndValidatePeerLoopMarker( @@ -1418,7 +1529,11 @@ int ReadAndValidatePeerLoopMarker( uint8_t dieId, uint32_t markerXnId, int rank, + int peerRank, int loopIndex, + uint32_t routeIndex, + uint32_t channelId, + uint32_t ckeId, uint64_t expectedPeerLoopMarker) { if (context == nullptr) { @@ -1435,7 +1550,11 @@ int ReadAndValidatePeerLoopMarker( peerLoopMarker == expectedPeerLoopMarker; std::cout << "tilexr_ccu_alltoall peerLoopMarker" << " rank=" << rank + << " peerRank=" << peerRank << " loopIndex=" << loopIndex + << " route=" << routeIndex + << " channel=" << channelId + << " cke=" << ckeId << " xnId=" << markerXnId << " readRet=" << readRet << " observed=0x" << std::hex << peerLoopMarker @@ -1688,7 +1807,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult, int ph << " allRanksDone=1" << " allRanksSucceeded=" << (allSucceeded ? 1 : 0) << std::endl; - return true; + return allSucceeded; } const auto elapsedMs = std::chrono::duration_cast( std::chrono::steady_clock::now() - start).count(); @@ -1897,12 +2016,18 @@ void PrintAllToAllResult(int rank, int loopIndex, int finalRet, const AllToAllSt << " mismatches=" << alltoall.mismatchCount << std::endl; } else { + const size_t chunkBytes = alltoall.chunkBytes == 0 ? alltoall.bytes : alltoall.chunkBytes; + const size_t sourceRank = chunkBytes == 0 ? 0 : alltoall.firstMismatchOffset / chunkBytes; + const size_t chunkOffset = chunkBytes == 0 ? 0 : alltoall.firstMismatchOffset % chunkBytes; std::cout << "tilexr_ccu_alltoall result passed=0" << " rank=" << rank << " loopIndex=" << loopIndex << " ret=" << finalRet << " readRet=" << alltoall.readRet << " mismatches=" << alltoall.mismatchCount + << " sourceRank=" << sourceRank + << " chunkOffset=" << chunkOffset + << " globalOffset=" << alltoall.firstMismatchOffset << " firstMismatchOffset=" << alltoall.firstMismatchOffset << " lastMismatchOffset=" << alltoall.lastMismatchOffset << " firstMismatchObserved=0x" << std::hex << alltoall.firstMismatchObserved @@ -1928,11 +2053,231 @@ void MaybeFastExitAfterAllToAllRun(int finalRet) } } +bool MeshPreparedIdentityMatches( + const TileXR::TileXRCcuDirectInstallAttempt& attempt, + const TileXR::TileXRCcuTask& stableTask, + const std::vector& stableResources) +{ + if (attempt.submitTasks.size() != 1U || attempt.plan.syncResources.size() != stableResources.size()) { + return false; + } + const auto& task = attempt.submitTasks.front(); + if (task.dieId != stableTask.dieId || task.missionId != stableTask.missionId || + task.key != stableTask.key || task.instStartId != stableTask.instStartId || + task.instCnt != stableTask.instCnt || task.argSize != stableTask.argSize) { + return false; + } + for (size_t i = 0; i < stableResources.size(); ++i) { + const auto& current = attempt.plan.syncResources[i]; + const auto& stable = stableResources[i]; + if (current.localXn != stable.localXn || current.remoteXn != stable.remoteXn || + current.notifyCke != stable.notifyCke || current.localWaitCke != stable.localWaitCke || + current.sourceCke != stable.sourceCke || current.channelId != stable.channelId) { + return false; + } + } + return true; +} + +int RunAllToAllMeshLongMissionSmokeForRank( + DirectCcuSmokeContext* context, + int rank, + int rankSize, + int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 4) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall mesh requires four ranks\"" << std::endl; + return 0; + } + + const int loopCount = AllToAllLoopCountFromEnv(); + AllToAllState alltoall; + alltoall.initRet = InitAllToAllMeshState(rank, rankSize, &alltoall); + if (loopCount == 0 && alltoall.initRet == ACL_SUCCESS) { + alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 9U; + options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; + if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { + options.syncInstructionCount = 1823U; + } + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " rankSize=" << rankSize + << " chunkBytes=" << alltoall.chunkBytes + << " bytes=" << alltoall.bytes + << " loopCount=" << loopCount + << " resourceCount=9" + << " mesh=1" + << " longMission=1" + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall.initRet != ACL_SUCCESS ? + alltoall.initRet : + context->planner.PrepareDirectCcuAllToAllMeshInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall.source.ptr), + reinterpret_cast(alltoall.destination.ptr), + alltoall.chunkBytes, + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (attempt.submitTasks.size() != 1U || attempt.submitTasks.front().argSize != + TILEXR_DIRECT_CCU_SQE_ARGS_LEN || attempt.plan.syncResources.size() != 9U) { + std::cerr << "tilexr_ccu_alltoall invalidMeshPreparedTask" + << " rank=" << rank + << " taskCount=" << attempt.submitTasks.size() + << " resourceCount=" << attempt.plan.syncResources.size() + << std::endl; + finalRet = 6; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" + << std::endl; + } else if (submitRequested) { + const TileXR::TileXRCcuTask stableTask = attempt.submitTasks.front(); + const auto stableResources = attempt.plan.syncResources; + aclrtStream stream = nullptr; + const int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + bool skipStreamDestroy = false; + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { + const int resetRet = ResetAllToAllMeshStateForLoop(rank, loopIndex, &alltoall); + const uint64_t localLoopMarker = BuildAllToAllLoopMarker(rank, loopIndex); + attempt.submitTasks.front().args[0] = localLoopMarker; + const bool ready = WaitForCollectiveSubmitReadiness( + rank, + rankSize, + resetRet == ACL_SUCCESS && installReport.submitReady, + loopIndex); + if (resetRet != ACL_SUCCESS) { + finalRet = 14; + } else if (!ready) { + finalRet = 13; + } + + if (finalRet == 0) { + TileXRDirectCcuSubmitReport submitReport; + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " mesh=1" + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << std::endl; + if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } else if (syncRet != ACL_SUCCESS) { + finalRet = 8; + skipStreamDestroy = true; + } + } + + uint32_t peerOrdinal = 0; + for (int peerRank = 0; peerRank < rankSize; ++peerRank) { + if (peerRank == rank) { + continue; + } + const uint32_t routeIndex = peerOrdinal * 3U; + const auto& markerResource = attempt.plan.syncResources[routeIndex]; + if (finalRet == 0 && ReadAndValidatePeerLoopMarker( + context, + attempt.submitTasks.front().dieId, + markerResource.localXn, + rank, + peerRank, + loopIndex, + routeIndex, + markerResource.channelId, + markerResource.localWaitCke, + BuildAllToAllLoopMarker(peerRank, loopIndex)) != TileXR::TILEXR_SUCCESS) { + finalRet = 15; + } + ++peerOrdinal; + } + if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { + finalRet = 14; + } + if (finalRet == 0 && !MeshPreparedIdentityMatches(attempt, stableTask, stableResources)) { + finalRet = 16; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex) && finalRet == 0) { + finalRet = 13; + } + PrintAllToAllResult(rank, loopIndex, finalRet, alltoall); + if (finalRet != 0) { + std::cerr << "tilexr_ccu_alltoall loopFailure" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet + << " resourceCount=9" + << " selfCopyCompletionCke=" << attempt.plan.syncResources[0].localWaitCke + << std::endl; + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + PrintCcuResourceState( + context, + attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall", + 9U); + break; + } + std::cout << "tilexr_ccu_alltoall stableResources=1" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " missionId=" << static_cast(stableTask.missionId) + << " instStartId=" << stableTask.instStartId + << " instCnt=" << stableTask.instCnt + << std::endl; + } + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1 rank=" << rank << std::endl; + } else { + aclrtDestroyStream(stream); + } + } + } + + MaybeFastExitAfterAllToAllRun(finalRet); + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } + if (AllToAllMeshSmokeEnabled()) { + return RunAllToAllMeshLongMissionSmokeForRank(context, rank, rankSize, device); + } if (rankSize != 2) { std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; @@ -2067,9 +2412,13 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, const int markerRet = ReadAndValidatePeerLoopMarker( context, attempt.submitTasks.front().dieId, - options.remoteXnStartId, + attempt.plan.syncResources[0].localXn, rank, + peer, loopIndex, + 0U, + attempt.plan.syncResources[0].channelId, + attempt.plan.syncResources[0].localWaitCke, BuildAllToAllLoopMarker(peer, loopIndex)); if (markerRet != TileXR::TILEXR_SUCCESS) { finalRet = 15; diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index c76a80a9..12aeaae1 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -709,6 +709,50 @@ def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): body.index("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)"), ) + def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn( + 'kAllToAllMeshEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH"', source + ) + self.assertIn("AllToAllMeshSmokeEnabled", source) + self.assertIn("InitAllToAllMeshState", source) + self.assertIn("ResetAllToAllMeshStateForLoop", source) + self.assertIn("BuildAllToAllMeshByte", source) + self.assertIn("RunAllToAllMeshLongMissionSmokeForRank", source) + body = source[ + source.index("int RunAllToAllMeshLongMissionSmokeForRank"): + source.index("int RunAllToAllLongMissionSmokeForRank") + ] + loop = "for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)" + self.assertIn("rankSize != 4", body) + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", body) + self.assertIn("aclrtCreateStream", body) + self.assertIn(loop, body) + self.assertLess(body.index("PrepareDirectCcuAllToAllMeshInstallAttempt"), body.index(loop)) + self.assertLess(body.index("aclrtCreateStream"), body.index(loop)) + self.assertIn("attempt.submitTasks.front().args[0] = localLoopMarker", body) + self.assertIn("WaitForCollectiveSubmitReadiness", body) + self.assertIn("WaitForCollectiveSubmitDone", body) + self.assertIn("for (int peerRank = 0; peerRank < rankSize; ++peerRank)", body) + self.assertIn("peerOrdinal * 3U", body) + self.assertIn("ReadAndValidatePeerLoopMarker", body) + self.assertIn("CheckAllToAllState(&alltoall)", body) + self.assertIn("PrintCcuResourceState", body) + self.assertIn("resourceCount=9", body) + + pattern = source[ + source.index("uint8_t BuildAllToAllMeshByte"): + source.index("int InitAllToAllMeshState") + ] + for field in ["sourceRank", "targetRank", "loopIndex", "chunkOffset"]: + with self.subTest(field=field): + self.assertIn(field, pattern) + self.assertIn("static_cast(rankSize) * state->chunkBytes", source) + self.assertIn("sourceRank=", source) + self.assertIn("chunkOffset=", source) + self.assertIn("globalOffset=", source) + def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): source = PROBE_SOURCE.read_text(encoding="utf-8") From 5b41bbbb7b1a36487afd11258a7d9e1b22a665a9 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 20 Jul 2026 18:42:02 +0800 Subject: [PATCH 35/41] test(ccu): run all-to-all smoke across four ranks --- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 299 +++++++++--------- .../test_tilexr_ccu_direct_smoke_runner.py | 125 ++++++-- 2 files changed, 252 insertions(+), 172 deletions(-) diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index ef704493..8f7a9a98 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -2,7 +2,7 @@ # # Copyright (c) 2026 TileXR Project # -# Two-rank runner for the private TileXR direct CCU smoke probe. +# Multi-rank runner for the private TileXR direct CCU smoke probe. # Default execution is safe and does not touch ACL/NPU runtime. set -euo pipefail @@ -92,6 +92,11 @@ alltoall_mode_enabled() [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0}" = "1" ] } +alltoall_mesh_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH:-0}" = "1" ] +} + alltoall_long_mission_enabled() { [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ] @@ -217,7 +222,15 @@ apply_alltoall_defaults() export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" - if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + if alltoall_mesh_mode_enabled; then + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-9}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-1823}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-16}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-16}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-16}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX:-3}" + elif [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453}" @@ -253,6 +266,30 @@ apply_signal_wait_defaults apply_sync_xn_ping_defaults apply_alltoall_defaults +rank_size="$(parse_int "${TILEXR_CCU_RANK_SIZE:-${TILEXR_CCU_PROBE_RANK_SIZE:-2}}" 2)" +if [ "${rank_size}" -lt 1 ]; then + echo "ERROR: rank size must be positive: ${rank_size}" >&2 + exit 2 +fi +devices="${TILEXR_CCU_SMOKE_DEVICES:-${TILEXR_TEST_DEVICES:-0,1}}" +IFS=',' read -r -a device_list <<< "${devices}" +if [ "${#device_list[@]}" -ne "${rank_size}" ]; then + echo "ERROR: device count ${#device_list[@]} does not match rank size ${rank_size}: ${devices}" >&2 + exit 2 +fi +declare -A seen_devices=() +for device in "${device_list[@]}"; do + if [ -z "${device}" ]; then + echo "ERROR: empty device in list: ${devices}" >&2 + exit 2 + fi + if [ "${seen_devices[${device}]+set}" = "set" ]; then + echo "ERROR: duplicate device ${device} in list: ${devices}" >&2 + exit 2 + fi + seen_devices["${device}"]=1 +done + if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" for diagnostic_var in \ @@ -268,6 +305,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_BARRIER \ TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING \ TILEXR_CCU_DIRECT_SMOKE_ALLTOALL \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH \ TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION \ TILEXR_CCU_ALLTOALL_BYTES \ TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ @@ -279,6 +317,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "dryRun ${diagnostic_var}=${diagnostic_value}" fi done + echo "dryRun TILEXR_CCU_PROBE_RANK_SIZE=${rank_size} devices=${devices}" sqe_arg_count="$(parse_int "${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" 13)" sync_resource_count="$(parse_int "${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" 1)" default_sync_instruction_count_value="$(default_sync_instruction_count "${sync_resource_count}")" @@ -324,30 +363,24 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then for endpoint_field in "${endpoint_fields[@]}"; do endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" common_endpoint_value="${!endpoint_var:-}" - rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" - rank0_endpoint_value="${!rank0_endpoint_var:-${common_endpoint_value}}" - rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" - rank1_endpoint_value="${!rank1_endpoint_var:-${common_endpoint_value}}" - if [ "${rank0_endpoint_value}" != "" ]; then - echo "dryRun rank0 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}" - fi - if [ "${rank1_endpoint_value}" != "" ]; then - echo "dryRun rank1 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}" - fi + for ((rank=0; rank/dev/null 2>&1; then npu_smi_rc=0 timeout "${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}s" npu-smi info > "${work_dir}/npu-smi.log" 2>&1 || npu_smi_rc=$? @@ -435,11 +466,13 @@ comm_domain="${TILEXR_CCU_PROBE_COMM_DOMAIN:-0}" timeout_s="${TILEXR_CCU_SMOKE_TIMEOUT:-180}" ready_dir="${work_dir}/submit_ready_${comm_port}" done_dir="${work_dir}/submit_done_${comm_port}" -rank0_log="${work_dir}/ccu_rank0.log" -rank1_log="${work_dir}/ccu_rank1.log" rm -rf "${ready_dir}" "${done_dir}" mkdir -p "${ready_dir}" "${done_dir}" -rm -f "${rank0_log}" "${rank1_log}" +rank_logs=() +for ((rank=0; rank&2 + exit 2 + fi common_env+=("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1") fi if [ "${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-0}" = "1" ]; then @@ -539,6 +576,9 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING}") fi @@ -612,90 +652,45 @@ for token_field in "${resource_window_token_fields[@]}"; do if [ "${token_value}" != "" ]; then common_env+=("${token_var}=${token_value}") fi - rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" - rank0_token_value="${!rank0_token_var:-}" - if [ "${rank0_token_value}" != "" ]; then - common_env+=("${rank0_token_var}=${rank0_token_value}") - fi - rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" - rank1_token_value="${!rank1_token_var:-}" - if [ "${rank1_token_value}" != "" ]; then - common_env+=("${rank1_token_var}=${rank1_token_value}") - fi done -rank0_env=() -rank1_env=() -if [ "${TILEXR_CCU_PROBE_RANK0_XN_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK0_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_XN_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK1_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT}") -elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT}") -elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT}") -fi -for endpoint_field in "${endpoint_fields[@]}"; do - rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" - rank0_endpoint_value="${!rank0_endpoint_var:-}" - if [ "${rank0_endpoint_value}" != "" ]; then - rank0_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}") - fi - rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" - rank1_endpoint_value="${!rank1_endpoint_var:-}" - if [ "${rank1_endpoint_value}" != "" ]; then - rank1_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}") - fi -done -for token_field in "${resource_window_token_fields[@]}"; do - rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" - rank0_token_value="${!rank0_token_var:-}" - if [ "${rank0_token_value}" != "" ]; then - rank0_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}") - fi - rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" - rank1_token_value="${!rank1_token_var:-}" - if [ "${rank1_token_value}" != "" ]; then - rank1_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}") - fi -done +build_rank_env() +{ + local rank="$1" + rank_env=() + local mapping generic rank_var rank_value + for mapping in \ + XN_START \ + REMOTE_XN_START \ + REMOTE_XN_COUNT \ + LOCAL_WAIT_CKE_START \ + LOCAL_WAIT_CKE_COUNT \ + REMOTE_NOTIFY_CKE_START \ + REMOTE_NOTIFY_CKE_COUNT; do + generic="TILEXR_CCU_PROBE_${mapping}" + rank_var="TILEXR_CCU_PROBE_RANK${rank}_${mapping}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done + for endpoint_field in "${endpoint_fields[@]}"; do + generic="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" + rank_var="${generic}_RANK${rank}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done + for token_field in "${resource_window_token_fields[@]}"; do + generic="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}" + rank_var="${generic}_RANK${rank}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done +} echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} syncXnPing=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} alltoallLongMission=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" @@ -808,31 +803,43 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then exit 0 fi -timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0 "${probe_bin}" > "${rank0_log}" 2>&1 & -rank0_pid=$! -sleep "${TILEXR_CCU_SMOKE_RANK1_DELAY:-1}" -timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1 "${probe_bin}" > "${rank1_log}" 2>&1 & -rank1_pid=$! - -rank0_status=0 -rank1_status=0 -wait "${rank0_pid}" || rank0_status=$? -wait "${rank1_pid}" || rank1_status=$? +rank_pids=() +rank_statuses=() +for ((rank=0; rank "${rank_logs[${rank}]}" 2>&1 & + rank_pids+=("$!") + if [ "${rank}" -eq 0 ] && [ "${rank_size}" -gt 1 ]; then + sleep "${TILEXR_CCU_SMOKE_RANK1_DELAY:-1}" + fi +done -cat "${rank0_log}" -cat "${rank1_log}" +any_rank_failed=0 +for ((rank=0; rank&2 - echo "rank0 log: ${rank0_log}" >&2 - echo "rank1 log: ${rank1_log}" >&2 +if [ "${any_rank_failed}" -ne 0 ]; then + echo "ERROR: direct CCU smoke rank process failed statuses=${rank_statuses[*]}" >&2 exit 4 fi if alltoall_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_alltoall prepare ret=0" "${log}"; then echo "ERROR: direct CCU alltoall prepare did not return success in ${log}" >&2 exit 5 @@ -843,7 +850,7 @@ if alltoall_mode_enabled; then fi done elif signal_wait_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_signal_wait prepare ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait prepare did not return success in ${log}" >&2 exit 5 @@ -854,7 +861,7 @@ elif signal_wait_mode_enabled; then fi done else - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_direct_smoke prepare ret=0" "${log}"; then echo "ERROR: direct CCU prepare did not return success in ${log}" >&2 exit 5 @@ -874,13 +881,13 @@ rank_skipped_p2p_ccu_copy_submit() } if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "submitReady=1" "${log}"; then echo "ERROR: direct CCU submit requested but prepare did not reach submitReady=1 in ${log}" >&2 exit 6 fi done - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if alltoall_mode_enabled; then if ! grep -q "tilexr_ccu_alltoall submit ret=0" "${log}"; then echo "ERROR: direct CCU alltoall submit did not return success in ${log}" >&2 @@ -921,14 +928,22 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then fi if alltoall_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do - if ! grep -q "tilexr_ccu_alltoall result passed=1" "${log}"; then - echo "ERROR: direct CCU alltoall result did not pass in ${log}" >&2 - exit 9 - fi - done + loop_count="$(parse_int "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" 1)" + expected_results=$((rank_size * loop_count)) + expected_marker_matches=$((rank_size * (rank_size - 1) * loop_count)) + actual_results="$(grep -h -c "tilexr_ccu_alltoall result passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" + actual_marker_matches="$(grep -h -c "tilexr_ccu_alltoall peerLoopMarker .*matched=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" + echo "tilexr_ccu_direct_smoke_runner alltoallCounts expectedResults=${expected_results} actualResults=${actual_results} expectedMarkerMatches=${expected_marker_matches} actualMarkerMatches=${actual_marker_matches}" + if [ "${actual_results}" -ne "${expected_results}" ]; then + echo "ERROR: direct CCU alltoall result count mismatch expected=${expected_results} actual=${actual_results}" >&2 + exit 9 + fi + if alltoall_mesh_mode_enabled && [ "${actual_marker_matches}" -ne "${expected_marker_matches}" ]; then + echo "ERROR: direct CCU alltoall marker count mismatch expected=${expected_marker_matches} actual=${actual_marker_matches}" >&2 + exit 9 + fi elif signal_wait_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then echo "ERROR: direct CCU signal/wait result did not pass in ${log}" >&2 exit 9 @@ -944,9 +959,9 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]; then delay_rank="${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK:-0}" min_sync_ms="${TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS:-100}" if [ "${delay_rank}" = "0" ]; then - wait_log="${rank1_log}" + wait_log="${rank_logs[1]}" else - wait_log="${rank0_log}" + wait_log="${rank_logs[0]}" fi wait_sync_ms="$( awk ' @@ -976,7 +991,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then exit 11 fi p2p_passed_count=0 - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy" "${log}"; then echo "ERROR: direct CCU P2P CCU-copy result missing in ${log}" >&2 exit 12 diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index cad3e2ba..568380f6 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -17,6 +17,87 @@ class TileXRCcuDirectSmokeRunnerTest(unittest.TestCase): + def run_fake_mesh_runner(self, devices="4,5,6,7", rank_size="4", loop_count="10"): + temp_dir = tempfile.TemporaryDirectory() + temp_path = Path(temp_dir.name) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cxx = fake_bin / "c++" + fake_cxx.write_text( + "#!/usr/bin/env bash\n" + "out=''\n" + "while [ $# -gt 0 ]; do\n" + " if [ \"$1\" = -o ]; then out=$2; shift 2; else shift; fi\n" + "done\n" + "cat > \"$out\" <<'PROBE'\n" + "#!/usr/bin/env bash\n" + "rank=${TILEXR_CCU_PROBE_RANK}\n" + "rank_size=${TILEXR_CCU_PROBE_RANK_SIZE}\n" + "loops=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}\n" + "echo \"tilexr_ccu_alltoall prepare ret=0 installSucceeded=1 submitReady=1\"\n" + "for ((loop=0; loop "${thread_log}" 2>&1', source) self.assertIn("ccu_thread.log", source) - self.assertIn("ccu_rank0.log", source) - self.assertIn("ccu_rank1.log", source) + self.assertIn('ccu_rank${rank}.log', source) self.assertIn("installSucceeded=1", source) self.assertIn("submitReady=1", source) self.assertLess(source.index("installSucceeded=1"), source.index("submitReady=1")) self.assertIn("${repo_root}/install/lib64/libtile-comm.so", source) - self.assertIn( - 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0', - source, - ) - self.assertIn( - 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1', - source, - ) + self.assertIn('timeout "${timeout_s}s" env "${common_env[@]}" "${rank_env[@]}"', source) self.assertNotIn('bash -c "wait', source) self.assertIn("npu-smi rc=", source) self.assertIn("TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU", source) @@ -104,10 +174,8 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): self.assertIn("TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI", source) self.assertIn("ccu_npu_smi_busy_guard.py", source) self.assertIn("tilexr_ccu_direct_smoke_runner summary", source) - self.assertIn("rank0Status=", source) - self.assertIn("rank1Status=", source) - self.assertIn("rank0Log=", source) - self.assertIn("rank1Log=", source) + self.assertIn('rank${rank}Status=', source) + self.assertIn('rank${rank}Log=', source) self.assertIn("submitTiming", source) self.assertIn("syncMs=", source) self.assertIn("p2pCcuCopy", source) @@ -119,8 +187,7 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): "npu-smi info", '"${probe_bin}"', "TILEXR_CCU_DIRECT_SMOKE_ENABLE=1", - "TILEXR_CCU_PROBE_RANK=0", - "TILEXR_CCU_PROBE_RANK=1", + 'TILEXR_CCU_PROBE_RANK="${rank}"', ]: with self.subTest(needle=needle): self.assertLess(gate, source.index(needle)) @@ -189,7 +256,7 @@ def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): source = RUNNER.read_text(encoding="utf-8") submit_check = source[ - source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]', source.index('if [ "${rank0_status}"')): + source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]', source.index("any_rank_failed=0")): source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]') ] @@ -287,10 +354,8 @@ def test_runner_passes_rank_specific_resource_window_eid_index(self): source = RUNNER.read_text(encoding="utf-8") self.assertIn("EID_INDEX", source[source.index("resource_window_token_fields=("):]) - self.assertIn('rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0"', source) - self.assertIn('rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1"', source) - self.assertIn('echo "dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}"', source) - self.assertIn('echo "dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}"', source) + self.assertIn('rank_var="${generic}_RANK${rank}"', source) + self.assertIn('echo "dryRun rank${rank} TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank_token_value}"', source) def test_runner_dry_run_shows_repository_install_diagnostic_variants(self): with tempfile.TemporaryDirectory() as temp_dir: From b9d42d26de67b098eabd76ef0f814dde23240d06 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Thu, 23 Jul 2026 19:54:24 +0800 Subject: [PATCH 36/41] feat(ccu): enable 4-rank all-to-all 128KB --- src/comm/CMakeLists.txt | 2 + src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 546 +++++++++--- src/comm/ccu/tilexr_ccu_alltoall_program.h | 8 + .../ccu/tilexr_ccu_collective_planner.cpp | 126 +-- src/comm/ccu/tilexr_ccu_collective_planner.h | 1 + .../ccu/tilexr_ccu_direct_orchestrator.cpp | 190 ++-- src/comm/ccu/tilexr_ccu_direct_runtime.cpp | 814 +++++++++++++++++- src/comm/ccu/tilexr_ccu_direct_runtime.h | 37 + src/comm/ccu/tilexr_ccu_driver_adapter.cpp | 41 +- src/comm/ccu/tilexr_ccu_driver_adapter.h | 1 + src/comm/ccu/tilexr_ccu_hccp_loader.cpp | 4 + src/comm/ccu/tilexr_ccu_hccp_loader.h | 2 + src/comm/ccu/tilexr_ccu_hccp_types.h | 39 + .../tilexr_ccu_lower_layer_plan_builder.cpp | 98 ++- .../ccu/tilexr_ccu_lower_layer_plan_builder.h | 2 + src/comm/ccu/tilexr_ccu_microcode.cpp | 55 ++ src/comm/ccu/tilexr_ccu_microcode.h | 18 + .../ccu/tilexr_ccu_resource_allocator.cpp | 5 +- src/comm/ccu/tilexr_ccu_resource_allocator.h | 1 + src/comm/ccu/tilexr_ccu_topology.cpp | 293 +++++++ src/comm/ccu/tilexr_ccu_topology.h | 34 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 66 +- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 70 +- tests/ccu/test_tilexr_ccu_alltoall_program.py | 149 ++-- tests/ccu/test_tilexr_ccu_backend_boundary.py | 6 +- .../test_tilexr_ccu_direct_orchestrator.py | 29 +- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 54 +- .../test_tilexr_ccu_direct_smoke_runner.py | 21 +- tests/ccu/test_tilexr_ccu_driver_adapter.py | 13 + ...est_tilexr_ccu_lower_layer_plan_builder.py | 315 ++++++- ...est_tilexr_ccu_ra_custom_channel_loader.py | 55 +- 31 files changed, 2622 insertions(+), 473 deletions(-) create mode 100644 src/comm/ccu/tilexr_ccu_topology.cpp create mode 100644 src/comm/ccu/tilexr_ccu_topology.h diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index 1a881156..5066604d 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -149,6 +149,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_runtime.cpp ccu/tilexr_ccu_signal_wait_program.h ccu/tilexr_ccu_signal_wait_program.cpp + ccu/tilexr_ccu_topology.h + ccu/tilexr_ccu_topology.cpp ccu/tilexr_ccu_runtime_session.h ccu/tilexr_ccu_runtime_session.cpp ccu/tilexr_ccu_executor.h diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 8f8fc4b6..7e32cf0a 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -6,12 +6,23 @@ #include "ccu/tilexr_ccu_alltoall_program.h" #include -#include #include namespace TileXR { namespace { +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_SQE_ARGS_TO_X_HEADER = 0x0001U; +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER = 0x0003U; +constexpr uint16_t TILEXR_CCU_TRACE_SET_CKE_HEADER = 0x0802U; +constexpr uint16_t TILEXR_CCU_TRACE_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER = 0x1000U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MS_TO_LOC_MEM_HEADER = 0x1002U; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; +constexpr uint16_t TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK = 0xffffU; + uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) { (void)spec; @@ -204,6 +215,33 @@ int AppendRemoteNotify( return TILEXR_SUCCESS; } +int AppendSyncXnNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + uint16_t localXn, + uint16_t remoteXn, + uint16_t mask, + const char* phase, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuSyncXnSpec notify; + notify.remoteXn = remoteXn; + notify.localXn = localXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = mask; + notify.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + + " SyncXn notify"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + int AppendRemoteMarkerNotify( uint16_t remoteNotifyCke, uint16_t channelId, @@ -252,12 +290,14 @@ int AppendPreSyncPhase( spec.preSyncLocalTokenXn == 0 ? spec.lengthXn : spec.preSyncLocalTokenXn; const uint16_t tokenChannelId = spec.preSyncTokenChannelId == 0 ? outputChannelId : spec.preSyncTokenChannelId; + const uint16_t markerChannelId = + spec.preSyncMarkerChannelId == 0 ? outputChannelId : spec.preSyncMarkerChannelId; const uint16_t tokenNotifyCke = spec.preSyncRemoteTokenNotifyCke == 0 ? remoteNotifyCke : spec.preSyncRemoteTokenNotifyCke; if (spec.preSyncMarkerEnabled && AppendRemoteMarkerNotify( remoteNotifyCke, - outputChannelId, + markerChannelId, spec, program, report) != TILEXR_SUCCESS) { @@ -368,6 +408,42 @@ int AppendCopyBlock( return TILEXR_SUCCESS; } +int AppendMeshRemoteCopyBlock( + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + uint16_t completionCke, + uint16_t completionMask, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuMemoryCopySpec copy; + copy.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; + copy.localGsa = spec.localGsa; + copy.localXn = spec.localXn; + copy.remoteGsa = spec.remoteGsa; + copy.remoteXn = spec.remoteXn; + copy.lengthXn = spec.lengthXn; + copy.localAddr = spec.localSendAddr + offset; + copy.localToken = spec.localSendToken; + copy.remoteAddr = spec.remoteRecvAddr + offset; + copy.remoteToken = spec.remoteRecvToken; + copy.lengthBytes = bytesPerBlock; + copy.channelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + copy.completionCke = completionCke; + copy.completionMask = completionMask; + + std::vector block; + TileXRCcuMemoryProgramReport memoryReport; + if (TileXRCcuBuildMemoryCopyProgram(copy, &block, &memoryReport) != TILEXR_SUCCESS || block.size() != 7U) { + return Fail(program, report, memoryReport.message.empty() ? + "failed to build direct CCU alltoall mesh remote copy block" : memoryReport.message); + } + block.pop_back(); + program->insert(program->end(), block.begin(), block.end()); + return TILEXR_SUCCESS; +} + int AppendFinish( const TileXRCcuAllToAll2RankProgramSpec& spec, std::vector* program, @@ -402,33 +478,6 @@ void FillReport( report->message = "ok"; } -bool InsertUnique(std::set* ids, std::initializer_list values) -{ - for (const uint16_t value : values) { - if (value == 0 || !ids->insert(value).second) { - return false; - } - } - return true; -} - -bool InsertPeerUnique(std::set* ids, std::initializer_list values) -{ - std::set peerIds; - for (const uint16_t value : values) { - if (value == 0) { - return false; - } - peerIds.insert(value); - } - for (const uint16_t value : peerIds) { - if (!ids->insert(value).second) { - return false; - } - } - return true; -} - int ValidateMeshSpec( const TileXRCcuAllToAllMeshProgramSpec& spec, std::vector* program, @@ -446,16 +495,16 @@ int ValidateMeshSpec( return Fail(program, report, "invalid direct CCU alltoall mesh local buffer"); } if (spec.selfSourceGsa == 0 || spec.selfDestinationGsa == 0 || spec.selfSourceXn == 0 || - spec.selfDestinationXn == 0 || spec.selfLengthXn == 0 || spec.selfChannelId == 0 || - spec.selfCompletionCke == 0) { + spec.selfDestinationXn == 0 || spec.selfLengthXn == 0 || + spec.selfCompletionCke == 0 || spec.remoteCompletionCke == 0) { return Fail(program, report, "missing direct CCU alltoall mesh self-copy resource"); } bool peerRanks[4] = {}; - std::set localXnIds; - std::set remoteXnIds; std::set channelIds; - std::set localCkeIds; - std::set remoteCkeIds; + const auto& sharedRoute = spec.peers.front().route; + if (spec.remoteCompletionCke == sharedRoute.sourceCke) { + return Fail(program, report, "alltoall mesh completion CKE overlaps source CKE"); + } for (const auto& peer : spec.peers) { if (peer.peerRank >= spec.rankSize || peer.peerRank == spec.localRank || peerRanks[peer.peerRank]) { return Fail(program, report, "invalid direct CCU alltoall mesh peer rank"); @@ -464,24 +513,18 @@ int ValidateMeshSpec( if (peer.route.localRank != spec.localRank || peer.route.localSendAddr != spec.localSendAddr || peer.route.localSendToken != spec.localSendToken || peer.route.localRecvAddr != spec.localRecvAddr || peer.route.localRecvToken != spec.localRecvToken || peer.route.bytes != spec.chunkBytes || - !peer.route.preSyncMarkerEnabled || !peer.route.preSyncNotify || !peer.route.preSyncWait || + peer.route.preSyncMarkerEnabled || !peer.route.preSyncNotify || !peer.route.preSyncWait || !peer.route.postSyncNotify || !peer.route.postSyncWait) { return Fail(program, report, "invalid direct CCU alltoall mesh peer route"); } - if (!InsertPeerUnique(&localXnIds, - {peer.route.localXn, peer.route.lengthXn, peer.route.preSyncLocalAddrXn, - peer.route.preSyncLocalTokenXn, peer.route.preSyncLocalMarkerXn}) || - !InsertPeerUnique(&remoteXnIds, - {peer.route.remoteXn, peer.route.preSyncRemoteAddrXn, - peer.route.preSyncRemoteTokenXn, peer.route.preSyncRemoteMarkerXn}) || - !InsertUnique(&channelIds, - {peer.route.preSyncChannelId, peer.route.preSyncTokenChannelId, peer.route.copyChannelId}) || - !InsertPeerUnique(&localCkeIds, - {peer.route.copyCompletionCke, peer.route.preSyncLocalWaitCke, - peer.route.postSyncLocalWaitCke}) || - !InsertPeerUnique(&remoteCkeIds, - {peer.route.preSyncRemoteNotifyCke, peer.route.preSyncRemoteTokenNotifyCke, - peer.route.postSyncRemoteNotifyCke})) { + if (peer.route.preSyncLocalAddrXn != sharedRoute.preSyncLocalAddrXn || + peer.route.preSyncLocalTokenXn != sharedRoute.preSyncLocalTokenXn || + peer.route.preSyncChannelId != peer.route.copyChannelId || + peer.route.preSyncTokenChannelId != peer.route.copyChannelId || + peer.route.postSyncChannelId != peer.route.copyChannelId || + peer.route.copyCompletionCke != spec.remoteCompletionCke || + peer.route.ckeMask != TILEXR_CCU_ALLTOALL_POST_SYNC_MASK || + !channelIds.insert(peer.route.copyChannelId).second) { return Fail(program, report, "duplicate direct CCU alltoall mesh peer resource"); } TileXRCcuAllToAll2RankProgramSpec validationRoute = peer.route; @@ -496,42 +539,54 @@ int ValidateMeshSpec( } int AppendMeshPeerPosts( - const TileXRCcuAllToAll2RankProgramSpec& route, + const std::vector& peers, std::vector* program, TileXRCcuAllToAllProgramReport* report) { - if (AppendRemoteMarkerNotify( - route.preSyncRemoteNotifyCke, - route.preSyncChannelId, - route, - program, - report) != TILEXR_SUCCESS) { - return TILEXR_ERROR_PARA_CHECK_FAIL; + if (peers.empty()) { + return Fail(program, report, "missing direct CCU alltoall mesh peers"); } - if (AppendRemoteNotify( - route.preSyncRemoteNotifyCke, - route.preSyncChannelId, - route.preSyncLocalAddrXn, - route.preSyncRemoteAddrXn, - route.localRecvAddr, - 0, - PreSyncSignalMask(route), - "mesh PreSync output", - program, - report) != TILEXR_SUCCESS) { + const auto& shared = peers.front().route; + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(shared.preSyncLocalAddrXn, shared.localRecvAddr, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh output variable"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(shared.preSyncLocalTokenXn, shared.localRecvToken, 1, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh token variable"); + } + program->push_back(instr); + + if (AppendSetSourceCke(shared, TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK, program, report) != TILEXR_SUCCESS) { return TILEXR_ERROR_PARA_CHECK_FAIL; } - return AppendRemoteNotify( - route.preSyncRemoteTokenNotifyCke, - route.preSyncTokenChannelId, - route.preSyncLocalTokenXn, - route.preSyncRemoteTokenXn, - route.localRecvToken, - 1, - PreSyncTokenMask(route), - "mesh PreSync token", - program, - report); + + for (const auto& peer : peers) { + const auto& route = peer.route; + if (AppendSyncXnNotify( + route.preSyncRemoteNotifyCke, + route.preSyncChannelId, + shared.preSyncLocalAddrXn, + route.preSyncRemoteAddrXn, + PreSyncSignalMask(route), + "mesh PreSync output", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AppendSyncXnNotify( + route.preSyncRemoteTokenNotifyCke, + route.preSyncTokenChannelId, + shared.preSyncLocalTokenXn, + route.preSyncRemoteTokenXn, + PreSyncTokenMask(route), + "mesh PreSync token", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; } int AppendLocalCopyBlock( @@ -563,17 +618,16 @@ int AppendLocalCopyBlock( } program->push_back(instr); - TileXRCcuMemTransferSpec transfer; + TileXRCcuLocalMsTransferSpec transfer; transfer.localGsa = spec.selfSourceGsa; transfer.localXn = spec.selfSourceXn; - transfer.remoteGsa = spec.selfDestinationGsa; - transfer.remoteXn = spec.selfDestinationXn; + transfer.localMs = 0; transfer.lengthXn = spec.selfLengthXn; - transfer.channelId = spec.selfChannelId; + transfer.channelId = 0; transfer.setCkeId = spec.selfCompletionCke; transfer.setCkeMask = 1U; - if (TileXRCcuEncodeTransLocMemToLocMem(transfer, &instr) != TILEXR_SUCCESS) { - return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer"); + if (TileXRCcuEncodeTransLocMemToLocMs(transfer, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer to local MS"); } program->push_back(instr); @@ -584,6 +638,17 @@ int AppendLocalCopyBlock( return Fail(program, report, "failed to wait direct CCU alltoall mesh self transfer"); } program->push_back(instr); + + transfer.localGsa = spec.selfDestinationGsa; + transfer.localXn = spec.selfDestinationXn; + if (TileXRCcuEncodeTransLocMsToLocMem(transfer, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer from local MS"); + } + program->push_back(instr); + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to wait direct CCU alltoall mesh self transfer from local MS"); + } + program->push_back(instr); return TILEXR_SUCCESS; } @@ -592,9 +657,6 @@ int AppendMeshPostNotify( std::vector* program, TileXRCcuAllToAllProgramReport* report) { - if (AppendSetSourceCke(route, route.ckeMask, program, report) != TILEXR_SUCCESS) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } TileXRCcuSyncCkeSpec post; post.remoteCke = route.postSyncRemoteNotifyCke; post.localCke = route.sourceCke; @@ -609,8 +671,249 @@ int AppendMeshPostNotify( return TILEXR_SUCCESS; } +uint16_t InstructionSlot(const TileXRCcuInstr& instr, uint32_t slot) +{ + return static_cast( + (instr.words[slot / 4U] >> ((slot % 4U) * 16U)) & 0xffffU); +} + +int FailBindingValidation(TileXRCcuAllToAllProgramReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = "direct CCU alltoall encoded binding validation failed: " + message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool MatchesSyncXn( + const TileXRCcuInstr& instr, + uint16_t remoteXn, + uint16_t localXn, + uint16_t channelId, + uint16_t notifyCke, + uint16_t notifyMask) +{ + return InstructionSlot(instr, 0) == TILEXR_CCU_TRACE_SYNC_XN_HEADER && + InstructionSlot(instr, 1) == remoteXn && + InstructionSlot(instr, 2) == localXn && + InstructionSlot(instr, 4) == channelId && + InstructionSlot(instr, 5) == notifyCke && + InstructionSlot(instr, 6) == notifyMask; +} + +bool MatchesWait( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t waitCke, + uint16_t waitMask) +{ + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, 4) == waitCke && + InstructionSlot(instr, 5) == waitMask; +} + +bool MatchesTransfer( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t remoteGsa, + uint16_t remoteXn, + uint16_t localGsa, + uint16_t localXn, + uint16_t lengthXn, + uint16_t channelId, + uint16_t completionCke, + uint16_t completionMask) +{ + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, 1) == remoteGsa && + InstructionSlot(instr, 2) == remoteXn && + InstructionSlot(instr, 3) == localGsa && + InstructionSlot(instr, 4) == localXn && + InstructionSlot(instr, 5) == lengthXn && + InstructionSlot(instr, 6) == channelId && + InstructionSlot(instr, 12) == completionCke && + InstructionSlot(instr, 13) == completionMask; +} + +bool MatchesLocalMsTransfer( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t localGsa, + uint16_t localXn, + uint16_t localMs, + uint16_t lengthXn, + uint16_t channelId, + uint16_t completionCke, + uint16_t completionMask) +{ + const bool memToMs = header == TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER; + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, memToMs ? 1U : 3U) == localMs && + InstructionSlot(instr, memToMs ? 2U : 1U) == localGsa && + InstructionSlot(instr, memToMs ? 3U : 2U) == localXn && + InstructionSlot(instr, 4) == lengthXn && + InstructionSlot(instr, 5) == channelId && + InstructionSlot(instr, 12) == completionCke && + InstructionSlot(instr, 13) == completionMask; +} + +int ValidateMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + auto peers = spec.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuAllToAllMeshPeerSpec& lhs, + const TileXRCcuAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + const uint32_t blocksPerChunk = static_cast( + spec.chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES); + const size_t expectedSize = 12U + static_cast(blocksPerChunk) * 28U + 6U + 1U; + if (peers.size() != 3U || blocksPerChunk == 0 || program.size() != expectedSize) { + return FailBindingValidation(report, "unexpected mesh program shape"); + } + + const auto& sharedRoute = peers.front().route; + if (InstructionSlot(program[0], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[0], 1) != sharedRoute.preSyncLocalAddrXn || + InstructionSlot(program[1], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[1], 1) != sharedRoute.preSyncLocalTokenXn || + InstructionSlot(program[2], 0) != TILEXR_CCU_TRACE_SET_CKE_HEADER || + InstructionSlot(program[2], 2) != sharedRoute.sourceCke || + InstructionSlot(program[2], 3) != TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK) { + return FailBindingValidation(report, "pre-sync variable loads do not match the shared mesh resources"); + } + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& route = peers[ordinal].route; + if (!MatchesSyncXn( + program[3U + ordinal * 2U], + route.preSyncRemoteAddrXn, + sharedRoute.preSyncLocalAddrXn, + route.preSyncChannelId, + route.preSyncRemoteNotifyCke, + PreSyncSignalMask(route))) { + return FailBindingValidation(report, "output SyncXn does not match its peer route"); + } + if (!MatchesSyncXn( + program[4U + ordinal * 2U], + route.preSyncRemoteTokenXn, + sharedRoute.preSyncLocalTokenXn, + route.preSyncTokenChannelId, + route.preSyncRemoteTokenNotifyCke, + PreSyncTokenMask(route))) { + return FailBindingValidation(report, "token SyncXn does not match its peer route"); + } + const size_t wait = 9U + ordinal; + if (!MatchesWait( + program[wait], + TILEXR_CCU_TRACE_SET_CKE_HEADER, + route.preSyncLocalWaitCke, + static_cast(PreSyncSignalMask(route) | PreSyncTokenMask(route)))) { + return FailBindingValidation(report, "pre-sync waits do not match their peer route"); + } + } + + size_t instruction = 12U; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& route = peers[ordinal].route; + if (!MatchesTransfer( + program[instruction + 5U], + TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER, + route.remoteGsa, + route.remoteXn, + route.localGsa, + route.localXn, + route.lengthXn, + route.copyChannelId, + spec.remoteCompletionCke, + static_cast(1U << ordinal))) { + return FailBindingValidation(report, "remote copy does not match its peer route"); + } + instruction += 6U; + } + if (!MatchesLocalMsTransfer( + program[instruction + 5U], + TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER, + spec.selfSourceGsa, + spec.selfSourceXn, + 0, + spec.selfLengthXn, + 0, + spec.selfCompletionCke, + 1U) || + !MatchesWait( + program[instruction + 6U], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.selfCompletionCke, + 1U) || + !MatchesLocalMsTransfer( + program[instruction + 7U], + TILEXR_CCU_TRACE_TRANS_LOC_MS_TO_LOC_MEM_HEADER, + spec.selfDestinationGsa, + spec.selfDestinationXn, + 0, + spec.selfLengthXn, + 0, + spec.selfCompletionCke, + 1U) || + !MatchesWait( + program[instruction + 8U], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.selfCompletionCke, + 1U)) { + return FailBindingValidation(report, "self copy does not match its local route"); + } + instruction += 9U; + if (!MatchesWait( + program[instruction], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.remoteCompletionCke, + 0x7U)) { + return FailBindingValidation(report, "combined remote copy wait does not match the mesh completion CKE"); + } + ++instruction; + } + for (const auto& peer : peers) { + const auto& route = peer.route; + if (InstructionSlot(program[instruction], 0) != TILEXR_CCU_TRACE_SYNC_CKE_HEADER || + InstructionSlot(program[instruction], 1) != route.postSyncRemoteNotifyCke || + InstructionSlot(program[instruction], 2) != route.sourceCke || + InstructionSlot(program[instruction], 3) != route.ckeMask || + InstructionSlot(program[instruction], 4) != route.postSyncChannelId) { + return FailBindingValidation(report, "post-sync notify does not match its peer route"); + } + ++instruction; + } + for (const auto& peer : peers) { + if (!MatchesWait( + program[instruction], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + peer.route.postSyncLocalWaitCke, + peer.route.ckeMask)) { + return FailBindingValidation(report, "post-sync wait does not match its peer route"); + } + ++instruction; + } + if (instruction + 1U != program.size() || + InstructionSlot(program[instruction], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[instruction], 1) != spec.selfSourceXn) { + return FailBindingValidation(report, "finish instruction does not match the mesh program"); + } + return TILEXR_SUCCESS; +} + } // namespace +int TileXRCcuValidateAllToAllMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + return ValidateMeshProgramBindings(spec, program, report); +} + int TileXRCcuBuildAllToAll2RankProgram( const TileXRCcuAllToAll2RankProgramSpec& spec, std::vector* program, @@ -700,34 +1003,44 @@ int TileXRCcuBuildAllToAllMeshProgram( }); const uint64_t bytesPerBlock = TILEXR_CCU_ALLTOALL_BLOCK_BYTES; const uint32_t blocksPerChunk = static_cast(spec.chunkBytes / bytesPerBlock); - program->reserve(3U * 7U + 4U * blocksPerChunk * 7U + 3U * 3U + 1U); + program->reserve(12U + blocksPerChunk * 28U + 6U + 1U); - for (const auto& peer : peers) { - ret = AppendMeshPeerPosts(peer.route, program, report); - if (ret != TILEXR_SUCCESS) { - return ret; - } + ret = AppendMeshPeerPosts(peers, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; } for (const auto& peer : peers) { - ret = AppendNotifyWait(peer.route.preSyncLocalWaitCke, 0x7U, "mesh PreSync", false, program, report); + ret = AppendNotifyWait( + peer.route.preSyncLocalWaitCke, + static_cast(PreSyncSignalMask(peer.route) | PreSyncTokenMask(peer.route)), + "mesh PreSync", + false, + program, + report); if (ret != TILEXR_SUCCESS) { return ret; } } - for (const auto& peer : peers) { - TileXRCcuAllToAll2RankProgramSpec route = peer.route; - route.localSendAddr = spec.localSendAddr + static_cast(peer.peerRank) * spec.chunkBytes; - route.remoteRecvAddr += static_cast(spec.localRank) * spec.chunkBytes; - for (uint32_t block = 0; block < blocksPerChunk; ++block) { - ret = AppendCopyBlock(route, static_cast(block) * bytesPerBlock, bytesPerBlock, program, report); + const uint64_t selfBaseOffset = static_cast(spec.localRank) * spec.chunkBytes; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& peer = peers[ordinal]; + TileXRCcuAllToAll2RankProgramSpec route = peer.route; + route.localSendAddr = spec.localSendAddr + static_cast(peer.peerRank) * spec.chunkBytes; + route.remoteRecvAddr += static_cast(spec.localRank) * spec.chunkBytes; + ret = AppendMeshRemoteCopyBlock( + route, + static_cast(block) * bytesPerBlock, + bytesPerBlock, + spec.remoteCompletionCke, + static_cast(1U << ordinal), + program, + report); if (ret != TILEXR_SUCCESS) { return ret; } } - } - const uint64_t selfBaseOffset = static_cast(spec.localRank) * spec.chunkBytes; - for (uint32_t block = 0; block < blocksPerChunk; ++block) { ret = AppendLocalCopyBlock( spec, selfBaseOffset + static_cast(block) * bytesPerBlock, @@ -737,6 +1050,16 @@ int TileXRCcuBuildAllToAllMeshProgram( if (ret != TILEXR_SUCCESS) { return ret; } + ret = AppendNotifyWait( + spec.remoteCompletionCke, + 0x7U, + "mesh Copy", + true, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } } for (const auto& peer : peers) { @@ -764,16 +1087,21 @@ int TileXRCcuBuildAllToAllMeshProgram( } program->push_back(finish); + if (TileXRCcuValidateAllToAllMeshProgramBindings(spec, *program, report) != TILEXR_SUCCESS) { + program->clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (report != nullptr) { - report->preSyncInstructionCount = 21U; + report->preSyncInstructionCount = 12U; report->blockCount = blocksPerChunk; report->bytesPerBlock = static_cast(bytesPerBlock); - report->copyInstructionCount = 4U * blocksPerChunk * 7U; - report->postSyncInstructionCount = 9U; + report->copyInstructionCount = blocksPerChunk * 28U; + report->postSyncInstructionCount = 6U; report->finishInstructionCount = 1U; report->totalInstructionCount = static_cast(program->size()); report->peerCount = 3U; - report->syncResourceCount = 9U; + report->syncResourceCount = 3U; report->remoteBlockCount = 3U * blocksPerChunk; report->selfBlockCount = blocksPerChunk; report->message = "ok"; diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index d73db788..93783ba2 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -22,6 +22,7 @@ constexpr uint16_t TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_TOKEN_XN_ID = 2U; constexpr uint16_t TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_ID = 3U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_MASK = 0x8U; constexpr uint16_t TILEXR_CCU_ALLTOALL_SIGNAL_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK = 2U; @@ -50,6 +51,7 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t preSyncRemoteMarkerXn = 0; uint16_t preSyncMarkerArgIndex = 0; uint16_t channelId = 0; + uint16_t preSyncMarkerChannelId = 0; uint16_t preSyncChannelId = 0; uint16_t preSyncTokenChannelId = 0; uint16_t copyChannelId = 0; @@ -93,6 +95,7 @@ struct TileXRCcuAllToAllMeshProgramSpec { uint16_t selfLengthXn = 0; uint16_t selfChannelId = 0; uint16_t selfCompletionCke = 0; + uint16_t remoteCompletionCke = 0; std::vector peers; }; @@ -121,6 +124,11 @@ int TileXRCcuBuildAllToAllMeshProgram( std::vector* program, TileXRCcuAllToAllProgramReport* report); +int TileXRCcuValidateAllToAllMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report); + } // namespace TileXR #endif // TILEXR_CCU_ALLTOALL_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 7d8caa88..ad537e2e 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -28,7 +28,7 @@ namespace TileXR { constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = 7U + 64U * 7U; -constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; @@ -554,14 +554,29 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } const size_t peerRouteCount = static_cast(rankSize - 1); const size_t syncRouteCount = allocation.remoteXn.num; - const size_t routesPerPeer = syncRouteCount / peerRouteCount; + size_t routedPeerCount = peerRouteCount; + int selectedDiagnosticPeer = -1; +#ifdef TILEXR_CCU_TESTING + for (const auto &override : directCcuRemoteRouteMemoryOverrides_) { + if (!override.allRoutes && override.syncRouteIndex == 0U && + override.buffer.peerRank < static_cast(rankSize) && + override.buffer.peerRank != static_cast(rank)) { + selectedDiagnosticPeer = static_cast(override.buffer.peerRank); + if (syncRouteCount == 1U) { + routedPeerCount = 1U; + } + break; + } + } +#endif + const size_t routesPerPeer = syncRouteCount / routedPeerCount; if (allocation.localXn.num == 0 || allocation.localWaitCke.num == 0 || allocation.remoteNotifyCke.num == 0 || - allocation.remoteXn.num < static_cast(rankSize - 1) || + allocation.remoteXn.num < routedPeerCount || allocation.localWaitCke.num < allocation.remoteXn.num || allocation.remoteNotifyCke.num < allocation.remoteXn.num || - allocation.channels.num == 0 || routesPerPeer == 0 || syncRouteCount % peerRouteCount != 0 || + allocation.channels.num == 0 || routesPerPeer == 0 || syncRouteCount % routedPeerCount != 0 || remoteCcuBuffers->size() != peerRouteCount) { if (report != nullptr) { report->message = "invalid direct CCU peer XN/CKE exchange shape"; @@ -604,18 +619,42 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( std::vector peerRanks; peerRanks.reserve(peerRouteCount); + if (selectedDiagnosticPeer >= 0) { + peerRanks.push_back(selectedDiagnosticPeer); + } for (int peer = 0; peer < rankSize; ++peer) { - if (peer != rank) { + if (peer != rank && peer != selectedDiagnosticPeer && routedPeerCount > 1U) { peerRanks.push_back(peer); } } - if (peerRanks.size() != peerRouteCount) { + if (peerRanks.size() != routedPeerCount) { if (report != nullptr) { report->message = "invalid direct CCU peer XN/CKE exchange shape"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } - std::vector peerCcuBuffers = *remoteCcuBuffers; + const std::vector peerCcuBuffers = *remoteCcuBuffers; + std::vector peerCcuBuffersByRank( + static_cast(rankSize), nullptr); + for (const auto &peerCcuBuffer : peerCcuBuffers) { + if (peerCcuBuffer.peerRank >= static_cast(rankSize) || + peerCcuBuffer.peerRank == static_cast(rank) || + peerCcuBuffersByRank[peerCcuBuffer.peerRank] != nullptr) { + if (report != nullptr) { + report->message = "invalid direct CCU peer buffer rank mapping"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + peerCcuBuffersByRank[peerCcuBuffer.peerRank] = &peerCcuBuffer; + } + for (const int peer : peerRanks) { + if (peerCcuBuffersByRank[static_cast(peer)] == nullptr) { + if (report != nullptr) { + report->message = "incomplete direct CCU peer buffer rank mapping"; + } + return TILEXR_ERROR_NOT_FOUND; + } + } remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); size_t routeIndex = 0; @@ -624,7 +663,8 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( const size_t routeWithinPeer = syncIndex % routesPerPeer; const int peer = peerRanks[peerBufferIndex]; const PeerResourceExchange &peerResources = all[peer]; - const size_t peerLocalIndex = static_cast(rank < peer ? rank : rank - 1); + const size_t peerLocalIndex = selectedDiagnosticPeer >= 0 ? + 0U : static_cast(rank < peer ? rank : rank - 1); const uint32_t peerLocalResourceOffset = static_cast( peerLocalIndex * routesPerPeer + routeWithinPeer); const uint32_t peerLocalXnOffset = peerLocalResourceOffset; @@ -645,14 +685,14 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } return TILEXR_ERROR_NOT_FOUND; } - uint16_t channelBoundRemoteXnId = static_cast( - static_cast(peerResources.remoteXnStartId) + peerLocalResourceOffset); const uint16_t peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); + const uint16_t channelBoundRemoteXnId = static_cast( + static_cast(peerResources.remoteXnStartId) + selectedRemoteXnOffset); uint16_t remoteNotifyCke = static_cast(static_cast(peerResources.localWaitCkeStartId) + peerLocalWaitCkeOffset); - (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; + (*remoteCcuBuffers)[routeIndex] = *peerCcuBuffersByRank[static_cast(peer)]; (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; const bool peerLocalXnOwnerVerified = @@ -696,6 +736,7 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( directCcuRemoteRouteMemoryOverrides_.clear(); DirectCcuRemoteRouteMemoryOverride override; override.allRoutes = true; + override.applyMemory = true; override.buffer.peerRank = peerRank; override.buffer.remoteCcuVa = remoteCcuVa; override.buffer.memoryTokenId = memoryTokenId; @@ -721,9 +762,7 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRou override.buffer.memoryTokenId = memoryTokenId; override.buffer.rawMemoryTokenId = rawMemoryTokenId; override.buffer.memoryTokenValue = memoryTokenValue; - if (remoteCcuVa == 0 || memoryTokenId == 0 || memoryTokenValue == 0) { - return; - } + override.applyMemory = remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; for (auto &existing : directCcuRemoteRouteMemoryOverrides_) { if (!existing.allRoutes && existing.syncRouteIndex == syncRouteIndex) { existing = override; @@ -751,6 +790,9 @@ void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( (!override.allRoutes && override.syncRouteIndex != routeIndex)) { continue; } + if (!override.applyMemory) { + break; + } remoteCcuBuffer.remoteCcuVa = override.buffer.remoteCcuVa; remoteCcuBuffer.memoryTokenId = override.buffer.memoryTokenId; remoteCcuBuffer.rawMemoryTokenId = override.buffer.rawMemoryTokenId; @@ -1487,13 +1529,6 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( alltoall.localRecvAddr = localEndpoint.destinationAddr; alltoall.localRecvToken = localEndpoint.destinationToken; alltoall.chunkBytes = chunkBytes; - struct ImportedPeer { - uint32_t peerRank = 0; - TileXRCcuImportedRemoteMemoryBufferInfo imported; - TileXRCcuRemoteMemoryBufferImportRequest request; - }; - std::vector importedPeers; - importedPeers.reserve(3U); for (uint32_t peerRank = 0; peerRank < static_cast(rankSize); ++peerRank) { if (peerRank == static_cast(rank)) { continue; @@ -1507,10 +1542,9 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - ImportedPeer importedPeer; - importedPeer.peerRank = peerRank; - importedPeer.request = endpoint.destinationRemoteImport; - ret = session.ImportRemoteMemoryBuffer(importedPeer.request, &importedPeer.imported); + const TileXRCcuRemoteMemoryBufferImportRequest &remoteImport = endpoint.destinationRemoteImport; + TileXRCcuImportedRemoteMemoryBufferInfo imported; + ret = session.ImportRemoteMemoryBuffer(remoteImport, &imported); if (ret != TILEXR_SUCCESS) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; @@ -1520,11 +1554,10 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( } TileXRCcuDirectAllToAllMeshPeerSpec peer; peer.peerRank = peerRank; - peer.remoteRecvAddr = importedPeer.request.addr; + peer.remoteRecvAddr = remoteImport.addr; peer.remoteRecvToken = TileXRCcuPackMemoryToken( - importedPeer.request.tokenId, importedPeer.request.tokenValue, true); + remoteImport.tokenId, remoteImport.tokenValue, true); alltoall.peers.push_back(peer); - importedPeers.push_back(importedPeer); } TileXRCcuDriverAdapter adapter; @@ -1548,25 +1581,13 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; next.lowerLayerPlanUserData = &callbackContext; next.sqeArgCount = 0; - next.syncResourceCount = 9U; + next.syncResourceCount = 3U; next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; if (next.provider.empty()) { next.provider = "tilexr-comm-direct-ccu-alltoall-mesh"; } ClearDirectCcuRemoteRouteMemoryOverride(); - for (uint32_t peerOrdinal = 0; peerOrdinal < importedPeers.size(); ++peerOrdinal) { - const ImportedPeer &peer = importedPeers[peerOrdinal]; - for (uint32_t routeWithinPeer = 0; routeWithinPeer < 3U; ++routeWithinPeer) { - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - peerOrdinal * 3U + routeWithinPeer, - peer.peerRank, - peer.imported.targetSegVa, - peer.request.tokenId, - peer.request.rawTokenId, - peer.request.tokenValue); - } - } ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt(next, alltoall, attempt, report); ClearDirectCcuRemoteRouteMemoryOverride(); return ret; @@ -1591,7 +1612,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( } const int rank = session.Rank(); const int rankSize = session.RankSize(); - if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + if ((rankSize != 2 && rankSize != 4) || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; @@ -1674,17 +1695,6 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( return TILEXR_ERROR_PARA_CHECK_FAIL; } - TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; - TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; - ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to import direct CCU SyncXn ping remote endpoint buffer"; - } - return ret; - } - TileXRCcuDriverAdapter adapter; TileXRCcuDriverAdapterReport adapterReport; ret = session.CreateDriverAdapter(&adapter, &adapterReport); @@ -1707,7 +1717,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; next.lowerLayerPlanUserData = &callbackContext; next.sqeArgCount = 0; - next.syncResourceCount = 1; + next.syncResourceCount = rankSize == 4 ? 3U : 1U; next.syncInstructionCount = std::max( next.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); @@ -1725,13 +1735,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( syncXnPing.localWaitMask = SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK"); - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - 0U, - peerRank, - importedRemoteBuffer.targetSegVa, - remoteImportRequest.tokenId, - remoteImportRequest.rawTokenId, - remoteImportRequest.tokenValue); + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(0U, peerRank, 0, 0, 0, 0); ret = TileXRCcuRunDirectSyncXnPingInstallAttempt(next, syncXnPing, attempt, report); ClearDirectCcuRemoteRouteMemoryOverride(); return ret; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index 182cd533..97f841ed 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -151,6 +151,7 @@ class TileXRCcuCollectivePlanner { struct DirectCcuRemoteRouteMemoryOverride { uint32_t syncRouteIndex = 0; bool allRoutes = false; + bool applyMemory = false; TileXRCcuRemoteCcuBufferInfo buffer; }; std::vector directCcuRemoteRouteMemoryOverrides_; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index d94b28fe..1df19674 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -40,11 +40,19 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = 7U + 64U * 7U; -constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT = 9U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; -constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; + +uint32_t SyncXnPingAllocationInstructionCount(uint32_t syncResourceCount) +{ + if (syncResourceCount > std::numeric_limits::max() / 2U) { + return std::numeric_limits::max(); + } + return syncResourceCount * 2U; +} uint32_t DirectAllToAllMeshInstructionCount(uint64_t chunkBytes) { @@ -52,7 +60,7 @@ uint32_t DirectAllToAllMeshInstructionCount(uint64_t chunkBytes) return 0; } const uint64_t blocks = chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; - const uint64_t instructions = 21ULL + 4ULL * blocks * 7ULL + 9ULL + 1ULL; + const uint64_t instructions = 11ULL + blocks * 28ULL + 7ULL + 1ULL; return instructions > std::numeric_limits::max() ? 0U : static_cast(instructions); } @@ -90,7 +98,7 @@ bool DirectAllToAllMeshCapacityFits( (resources.xnCount > TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT ? resources.xnCount - TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : 0U) : resources.remoteXnCount) && - require("local CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT + 1U, + require("local CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT + 2U, localWaitCkeCount) && require("remote CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, remoteNotifyCkeCount) && @@ -889,7 +897,7 @@ int ConfigureDirectAllToAllMeshResources( attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || attempt->plan.taskWindows.size() != 1U) { if (report != nullptr) { - report->message = "alltoall mesh direct CCU plan requires nine sync resources and one task"; + report->message = "alltoall mesh direct CCU plan requires three peer sync resources and one task"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -904,7 +912,7 @@ int ConfigureDirectAllToAllMeshResources( attempt->allocation.localWaitCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || attempt->allocation.remoteNotifyCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || attempt->allocation.channels.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.sourceCke.num != 1U) { + attempt->allocation.sourceCke.num < 2U) { if (report != nullptr) { report->message = "alltoall mesh direct CCU allocation is missing XN/CKE/channel resources"; } @@ -1078,6 +1086,7 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.channelId = copyResource.channelId; alltoallSpec.preSyncChannelId = preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId; + alltoallSpec.preSyncMarkerChannelId = alltoallSpec.preSyncChannelId; alltoallSpec.preSyncTokenChannelId = preResource.channelId; alltoallSpec.copyChannelId = copyResource.channelId; alltoallSpec.postSyncChannelId = postResource.channelId; @@ -1187,6 +1196,45 @@ int BuildDirectAllToAll2RankLaunchPackage( return TILEXR_SUCCESS; } +int ValidateDirectAllToAllMeshRouteResources( + const TileXRCcuAllToAllMeshProgramSpec& mesh, + const TileXRCcuProducerPlan& plan, + TileXRCcuDirectInstallReport* report) +{ + if (mesh.peers.size() != 3U || + plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT) { + if (report != nullptr) { + report->message = "alltoall mesh route binding validation has an invalid shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + for (size_t ordinal = 0; ordinal < mesh.peers.size(); ++ordinal) { + const auto& resource = plan.syncResources[ordinal]; + const auto& route = mesh.peers[ordinal].route; + if (route.preSyncMarkerEnabled || + route.preSyncChannelId != resource.channelId || + route.preSyncTokenChannelId != resource.channelId || + route.copyChannelId != resource.channelId || + route.postSyncChannelId != resource.channelId || + route.preSyncLocalWaitCke != resource.localWaitCke || + route.preSyncTokenLocalWaitCke != resource.localWaitCke || + route.postSyncLocalWaitCke != resource.localWaitCke || + route.preSyncRemoteNotifyCke != resource.notifyCke || + route.preSyncRemoteTokenNotifyCke != resource.notifyCke || + route.postSyncRemoteNotifyCke != resource.notifyCke || + route.copyCompletionCke != mesh.remoteCompletionCke) { + if (report != nullptr) { + std::ostringstream stream; + stream << "alltoall mesh route binding mismatch peerRank=" << mesh.peers[ordinal].peerRank + << " ordinal=" << ordinal; + report->message = stream.str(); + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; +} + int BuildDirectAllToAllMeshLaunchPackage( const TileXRCcuDirectAllToAllMeshSpec& alltoall, TileXRCcuDirectInstallAttempt* attempt, @@ -1217,11 +1265,11 @@ int BuildDirectAllToAllMeshLaunchPackage( mesh.selfSourceGsa = attempt->plan.kernelLocalGsa.startId; mesh.selfDestinationGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + const uint16_t localXnStart = attempt->allocation.localXn.startId; + const uint16_t remoteXnStart = attempt->allocation.remoteXn.startId; + mesh.remoteCompletionCke = static_cast(attempt->allocation.sourceCke.startId + 1U); for (uint32_t ordinal = 0; ordinal < peers.size(); ++ordinal) { - const uint32_t base = ordinal * 3U; - const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[base]; - const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[base + 1U]; - const TileXRCcuSyncResource& tokenResource = attempt->plan.syncResources[base + 2U]; + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[ordinal]; TileXRCcuAllToAllMeshPeerSpec peer; peer.peerRank = peers[ordinal].peerRank; auto& route = peer.route; @@ -1235,30 +1283,27 @@ int BuildDirectAllToAllMeshLaunchPackage( route.bytes = alltoall.chunkBytes; route.localGsa = attempt->plan.kernelLocalGsa.startId; route.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); - route.localXn = copyResource.localXn; - route.remoteXn = copyResource.remoteXn; - route.lengthXn = tokenResource.localXn; - route.preSyncLocalAddrXn = preResource.localXn; - route.preSyncLocalTokenXn = tokenResource.localXn; - route.preSyncLocalMarkerXn = copyResource.localXn; - route.preSyncRemoteMarkerXn = copyResource.remoteXn; - route.preSyncRemoteAddrXn = preResource.remoteXn; - route.preSyncRemoteTokenXn = tokenResource.remoteXn; - route.preSyncMarkerArgIndex = 0; - route.preSyncMarkerEnabled = true; - route.preSyncChannelId = preResource.channelId; - route.preSyncTokenChannelId = tokenResource.channelId; - route.copyChannelId = copyResource.channelId; - route.postSyncChannelId = copyResource.channelId; - route.copyCompletionCke = copyResource.localWaitCke; - route.preSyncLocalWaitCke = preResource.localWaitCke; - route.preSyncRemoteNotifyCke = preResource.notifyCke; - route.preSyncTokenLocalWaitCke = tokenResource.localWaitCke; - route.preSyncRemoteTokenNotifyCke = preResource.notifyCke; - route.postSyncLocalWaitCke = copyResource.localWaitCke; - route.postSyncRemoteNotifyCke = copyResource.notifyCke; + route.localXn = localXnStart; + route.remoteXn = static_cast(remoteXnStart + 2U); + route.lengthXn = static_cast(localXnStart + 2U); + route.preSyncLocalAddrXn = localXnStart; + route.preSyncLocalTokenXn = static_cast(localXnStart + 1U); + route.preSyncRemoteAddrXn = remoteXnStart; + route.preSyncRemoteTokenXn = static_cast(remoteXnStart + 1U); + route.preSyncMarkerEnabled = false; + route.preSyncChannelId = resource.channelId; + route.preSyncTokenChannelId = resource.channelId; + route.copyChannelId = resource.channelId; + route.postSyncChannelId = resource.channelId; + route.copyCompletionCke = mesh.remoteCompletionCke; + route.preSyncLocalWaitCke = resource.localWaitCke; + route.preSyncRemoteNotifyCke = resource.notifyCke; + route.preSyncTokenLocalWaitCke = resource.localWaitCke; + route.preSyncRemoteTokenNotifyCke = resource.notifyCke; + route.postSyncLocalWaitCke = resource.localWaitCke; + route.postSyncRemoteNotifyCke = resource.notifyCke; route.sourceCke = attempt->allocation.sourceCke.startId; - route.ckeMask = 1U; + route.ckeMask = static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); route.preSyncNotify = true; route.preSyncWait = true; route.postSyncNotify = true; @@ -1266,14 +1311,15 @@ int BuildDirectAllToAllMeshLaunchPackage( route.emitFinish = false; mesh.peers.push_back(peer); } - const TileXRCcuSyncResource& selfCopy = attempt->plan.syncResources[0]; - const TileXRCcuSyncResource& selfPre = attempt->plan.syncResources[1]; - const TileXRCcuSyncResource& selfToken = attempt->plan.syncResources[2]; - mesh.selfSourceXn = selfCopy.localXn; - mesh.selfDestinationXn = selfToken.localXn; - mesh.selfLengthXn = selfPre.localXn; - mesh.selfChannelId = selfCopy.channelId; - mesh.selfCompletionCke = selfCopy.localWaitCke; + mesh.selfSourceXn = localXnStart; + mesh.selfDestinationXn = static_cast(localXnStart + 1U); + mesh.selfLengthXn = static_cast(localXnStart + 2U); + mesh.selfChannelId = 0; + mesh.selfCompletionCke = attempt->plan.syncResources[0].localWaitCke; + + if (ValidateDirectAllToAllMeshRouteResources(mesh, attempt->plan, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } TileXRCcuProgram program; TileXRCcuAllToAllProgramReport alltoallReport; @@ -1468,29 +1514,25 @@ int BuildDirectSyncXnPingLaunchPackage( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - if (attempt == nullptr || attempt->plan.syncResources.size() != 1 || attempt->plan.taskWindows.size() != 1) { + if (attempt == nullptr || attempt->plan.syncResources.empty() || attempt->plan.taskWindows.size() != 1) { if (report != nullptr) { report->message = "missing direct CCU SyncXn ping producer resources"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } - if (syncXnPing.localRank > 1U || syncXnPing.peerRank > 1U || syncXnPing.localRank == syncXnPing.peerRank) { + if (syncXnPing.localRank > 3U || syncXnPing.peerRank > 3U || syncXnPing.localRank == syncXnPing.peerRank) { if (report != nullptr) { - report->message = "direct CCU SyncXn ping requires two distinct rank ids"; + report->message = "direct CCU SyncXn ping requires distinct rank ids in the range [0, 3]"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; - const uint16_t localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; const uint16_t defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank); - const uint16_t defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank); const uint16_t remoteNotifyMask = syncXnPing.remoteNotifyMask == 0 ? defaultRemoteNotifyMask : syncXnPing.remoteNotifyMask; - const uint16_t localWaitMask = - syncXnPing.localWaitMask == 0 ? defaultLocalWaitMask : syncXnPing.localWaitMask; if (resource.localXn == 0 || resource.remoteXn == 0 || resource.channelId == 0 || - resource.notifyCke == 0 || localWaitCke == 0 || resource.sourceCke == 0) { + resource.notifyCke == 0) { if (report != nullptr) { report->message = "missing direct CCU SyncXn ping XN/CKE/channel resource"; } @@ -1522,44 +1564,6 @@ int BuildDirectSyncXnPingLaunchPackage( return TILEXR_ERROR_PARA_CHECK_FAIL; } program.sync.push_back(instr); - - TileXRCcuCkeSpec source; - source.ckeId = resource.sourceCke; - source.mask = remoteNotifyMask; - source.clearWait = true; - if (TileXRCcuEncodeSetCke(source, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping source CKE set"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); - - TileXRCcuSyncCkeSpec syncCke; - syncCke.remoteCke = resource.notifyCke; - syncCke.localCke = resource.sourceCke; - syncCke.localCkeMask = remoteNotifyMask; - syncCke.channelId = resource.channelId; - syncCke.clearWait = true; - if (TileXRCcuEncodeSyncCke(syncCke, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping SyncCke notify"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); - - TileXRCcuCkeSpec wait; - wait.waitCkeId = localWaitCke; - wait.waitMask = localWaitMask; - wait.clearWait = true; - if (TileXRCcuEncodeSetCke(wait, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping wait"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); TileXRCcuRepositoryImage repository; @@ -1827,6 +1831,7 @@ int RunDirectInstallAttemptImpl( attempt->resourceRequest.syncResourceCount = alltoallMesh != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : + syncXnPing != nullptr ? options.syncResourceCount : customProgram ? 1U : options.syncResourceCount; attempt->resourceRequest.syncInstructionCount = memoryCopy != nullptr ? @@ -1839,13 +1844,16 @@ int RunDirectInstallAttemptImpl( signalWait != nullptr ? std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : syncXnPing != nullptr ? - std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT) : + std::max( + options.syncInstructionCount, + SyncXnPingAllocationInstructionCount(options.syncResourceCount)) : options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.barrierMode = + attempt->resourceRequest.sourceCkeCount = alltoallMesh != nullptr ? 2U : 1U; + attempt->resourceRequest.barrierMode = alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : - syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn : signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); TileXRCcuResourceAllocator allocator; @@ -1897,7 +1905,7 @@ int RunDirectInstallAttemptImpl( attempt->plan.barrierMode = alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : - syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn : signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp index 26b91e71..1183e2a2 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -5,6 +5,8 @@ #include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_topology.h" + #include #include #include @@ -30,6 +32,7 @@ constexpr uint32_t TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH = 64; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_MAX_POLLS = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_SLEEP_US = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS = 8; +constexpr uint8_t TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT = 16; constexpr int TILEXR_CCU_DIRECT_MAX_RANK_SIZE = 128; constexpr int TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL = 2; constexpr int TILEXR_CCU_HCCP_ASYNC_EAGAIN = 128301; @@ -44,6 +47,8 @@ constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"; constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID"; constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE"; constexpr const char* TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE_ENV = @@ -52,11 +57,19 @@ constexpr const char* TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE_ENV = "TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE"; constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE"; +constexpr const char* TILEXR_CCU_HCCL_ROOT_INFO_PATH = "/etc/hccl_rootinfo.json"; constexpr const char* TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET_ENV = "TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET"; +constexpr const char* TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE_ENV = + "TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE"; constexpr uint8_t TILEXR_CCU_DIRECT_DEFAULT_DIE_ID = 0; constexpr uint64_t TILEXR_CCU_UB_MEM_PAGE_SIZE = 4096ULL; constexpr uint32_t TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT = 8; +constexpr uint32_t TILEXR_CCU_TP_HANDLE_REQUEST_NUM = 8; +constexpr uint32_t TILEXR_CCU_TP_ATTR_BITMAP_SL = 1U << 10U; +constexpr uint32_t TILEXR_CCU_TP_ATTR_BITMAP_SL_AVAILABLE = 1U << 17U; +constexpr uint32_t TILEXR_CCU_DEFAULT_HCCL_QOS = 4; +constexpr uint32_t TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT = 19U; struct TileXRCcuEndpointTpHandleExchange { uint64_t tpHandles[TILEXR_CCU_DIRECT_MAX_RANK_SIZE] = {}; @@ -109,12 +122,52 @@ bool HasCompleteEndpointRoute(const TileXRCcuLowerLayerTransportRoute& route) route.sqDepth != 0; } +struct TileXRCcuPeerEndpointOffer { + uint64_t resourceAddr = 0; + uint32_t resourceTokenId = 0; + uint32_t resourceRawTokenId = 0; + uint32_t resourceTokenValue = 0; + uint32_t jettyTokenValue = 0; + std::array eid {}; + TileXRCcuHccpQpKey qpKey {}; + uint32_t psn = 0; + uint32_t funcId = 0; + bool funcIdValid = false; + bool valid = false; +}; + +bool SameEid( + const std::array& expected, + const TileXRCcuHccpEid& actual) +{ + return std::memcmp(expected.data(), actual.raw, expected.size()) == 0; +} + bool UseImportedPeerEndpointRoute() { const char* mode = std::getenv(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV); return mode == nullptr || mode[0] == '\0' || std::strcmp(mode, "imported_peer") == 0; } +uint8_t SelectDirectCcuCleanupDieId() +{ + const char* value = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); + if (value == nullptr || value[0] == '\0') { + return TILEXR_CCU_DIRECT_DEFAULT_DIE_ID; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + return end != value && *end == '\0' && parsed <= UINT8_MAX ? + static_cast(parsed) : + TILEXR_CCU_DIRECT_DEFAULT_DIE_ID; +} + +bool RecoverTaskKillState() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE_ENV); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + uint64_t SelectResourceWindowBytes(const TileXRCcuBasicInfo& basicInfo) { (void)basicInfo; @@ -213,12 +266,16 @@ void TraceEndpointRouteStep(const std::string& message) } } -void TraceTaskKillCleanup(uint8_t dieId, int ret, const TileXRCcuDriverAdapterReport& report) +void TraceTaskKillStep( + const char* step, + uint8_t dieId, + int ret, + const TileXRCcuDriverAdapterReport& report) { if (!TraceEndpointRoute()) { return; } - std::cerr << "TileXRDirectCcuTrace taskKillCleanup" + std::cerr << "TileXRDirectCcuTrace taskKill" << step << " dieId=" << static_cast(dieId) << " ret=" << ret << " opcode=" << report.opcode @@ -237,37 +294,63 @@ void TraceRaCtxEidInfos(const std::vector& eidInfos) std::cerr << "TileXRDirectCcuTrace endpointRoute raCtxEidInfo" << " ordinal=" << i << " eidIndex=" << eidInfos[i].eidIndex + << " dieId=" << eidInfos[i].dieId << " funcId=" << eidInfos[i].funcId + << " devFeature=0x" << std::hex << eidInfos[i].resv << std::dec << " eid=" << FormatEndpointEid(CopyRawEid(eidInfos[i].eid)) << std::endl; } } -bool SelectRaCtxResourceWindowEidInfo( +bool ParseEndpointEid(const char* value, std::array* eid); + +bool BuildRaCtxResourceWindowEidCandidates( int rank, + uint8_t dieId, const std::vector& eidInfos, - TileXRCcuHccpDevEidInfo* selectedEid) + std::vector* candidates) { - if (eidInfos.empty() || selectedEid == nullptr) { + if (eidInfos.empty() || candidates == nullptr) { return false; } + candidates->clear(); TraceRaCtxEidInfos(eidInfos); - const char* configured = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, rank); - if (configured == nullptr) { - *selectedEid = eidInfos[0]; - return true; + const char* configuredEid = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV, rank); + if (configuredEid != nullptr) { + std::array expected {}; + if (!ParseEndpointEid(configuredEid, &expected)) { + return false; + } + for (const auto& eidInfo : eidInfos) { + if (CopyRawEid(eidInfo.eid) == expected) { + candidates->push_back(eidInfo); + return true; + } + } + return false; } - uint64_t configuredIndex = 0; - if (!ParseUnsignedEnv(configured, &configuredIndex) || configuredIndex > 0xffffffffULL) { + const char* configured = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, rank); + if (configured != nullptr) { + uint64_t configuredIndex = 0; + if (!ParseUnsignedEnv(configured, &configuredIndex) || configuredIndex > 0xffffffffULL) { + return false; + } + for (const auto& eidInfo : eidInfos) { + if (eidInfo.eidIndex == static_cast(configuredIndex)) { + candidates->push_back(eidInfo); + return true; + } + } return false; } - for (const auto& eidInfo : eidInfos) { - if (eidInfo.eidIndex == static_cast(configuredIndex)) { - *selectedEid = eidInfo; - return true; + for (auto it = eidInfos.rbegin(); it != eidInfos.rend(); ++it) { + const bool uboeOnly = + ((it->resv >> TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT) & 1U) != 0U; + if (it->dieId == dieId && !uboeOnly) { + candidates->push_back(*it); } } - return false; + return !candidates->empty(); } bool IsRaCtxLoopEndpointRouteCollectionMode() @@ -316,6 +399,55 @@ bool HasRaCtxEndpointRouteSymbols(const TileXRCcuHccpLoader& loader) loader.RaGetAsyncReqResult != nullptr; } +uint32_t CountAvailableSl(uint16_t mask) +{ + uint32_t count = 0; + for (uint32_t bit = 0; bit < 16U; ++bit) { + count += (mask & (1U << bit)) != 0U ? 1U : 0U; + } + return count; +} + +uint8_t SlAtRank(uint16_t mask, uint32_t rank) +{ + uint32_t seen = 0; + for (uint8_t bit = 0; bit < 16U; ++bit) { + if ((mask & (1U << bit)) != 0U && seen++ == rank) { + return bit; + } + } + return 0; +} + +bool MapQosToTpAndSl( + uint32_t qos, + uint32_t tpCount, + uint16_t slMask, + uint32_t* tpIndex, + uint8_t* mappedSl) +{ + if (tpIndex == nullptr || mappedSl == nullptr || tpCount == 0) { + return false; + } + const uint32_t slCount = CountAvailableSl(slMask); + const uint32_t k = std::min(tpCount, slCount); + if (k == 0) { + return false; + } + const uint32_t groupCount = std::min(8U, k); + const uint32_t q = qos & 7U; + const uint32_t group = k == 3U ? (q < 3U ? 0U : (q < 5U ? 1U : 2U)) : + (q * groupCount) / 8U; + const uint32_t slot = (group * k) / groupCount; + if (slot >= k || slot >= tpCount) { + return false; + } + const uint32_t slRank = (slCount - 1U) - slot; + *tpIndex = (k - 1U) - slot; + *mappedSl = SlAtRank(slMask, slRank); + return true; +} + uint32_t SelectEndpointRouteSqDepth() { return TILEXR_CCU_DEFAULT_DIRECT_SQ_DEPTH; @@ -586,11 +718,25 @@ int TileXRCcuDirectRuntime::Init( initialized_ = true; TileXRCcuDriverAdapter adapter; TileXRCcuDriverAdapterReport adapterReport; + const uint8_t cleanupDieId = SelectDirectCcuCleanupDieId(); int cleanupRet = CreateDriverAdapter(&adapter, &adapterReport); + if (cleanupRet == TILEXR_SUCCESS && RecoverTaskKillState()) { + cleanupRet = adapter.SetTaskKill(cleanupDieId, &adapterReport); + TraceTaskKillStep("Set", cleanupDieId, cleanupRet, adapterReport); + // hcomm treats SET_TASKKILL as a best-effort trigger and gates recovery + // on the following CLEAN_TASKKILL_STATE result. + cleanupRet = TILEXR_SUCCESS; + } if (cleanupRet == TILEXR_SUCCESS) { - cleanupRet = adapter.CleanTaskKillState(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, &adapterReport); + cleanupRet = adapter.CleanTaskKillState(cleanupDieId, &adapterReport); + } + TraceTaskKillStep("Cleanup", cleanupDieId, cleanupRet, adapterReport); + if (RecoverTaskKillState() && cleanupRet != TILEXR_SUCCESS) { + const std::string message = "failed to clean direct CCU task-kill state after explicit recovery: " + + adapterReport.message; + Shutdown(); + return Fail(report, message, cleanupRet); } - TraceTaskKillCleanup(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, cleanupRet, adapterReport); if (report != nullptr) { report->initialized = true; report->raInitialized = true; @@ -899,27 +1045,64 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( if (ret != 0 || queriedEidNum == 0) { return ret == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_ERROR_MKIRT; } + eidInfos.resize(queriedEidNum); void* ctxHandle = nullptr; void* tokenIdHandle = nullptr; void* lmemHandle = nullptr; - TileXRCcuHccpDevEidInfo selectedEid {}; - if (!SelectRaCtxResourceWindowEidInfo(options_.rank, eidInfos, &selectedEid)) { + std::vector eidCandidates; + if (!BuildRaCtxResourceWindowEidCandidates( + options_.rank, + SelectDirectCcuCleanupDieId(), + eidInfos, + &eidCandidates)) { return TILEXR_ERROR_PARA_CHECK_FAIL; } TileXRCcuHccpCtxInitCfg ctxCfg {}; ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; ctxCfg.rdma.disabledLiteThread = false; - TileXRCcuHccpCtxInitAttr ctxAttr {}; - ctxAttr.phyId = devicePhyId_; - ctxAttr.ub.eidIndex = selectedEid.eidIndex; - ctxAttr.ub.eid = selectedEid.eid; - - ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); - if (ret != 0 || ctxHandle == nullptr) { - return TILEXR_ERROR_MKIRT; + TileXRCcuHccpDevEidInfo selectedEid {}; + const bool explicitEid = + HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV, options_.rank) || + HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, options_.rank); + const bool canProbeLoopTp = !explicitEid && + loader_.RaGetTpInfoListAsync != nullptr && loader_.RaGetAsyncReqResult != nullptr; + for (const auto& candidate : eidCandidates) { + TileXRCcuHccpCtxInitAttr ctxAttr {}; + ctxAttr.phyId = devicePhyId_; + ctxAttr.ub.eidIndex = candidate.eidIndex; + ctxAttr.ub.eid = candidate.eid; + ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); + if (ret != 0 || ctxHandle == nullptr) { + ctxHandle = nullptr; + continue; + } + uint64_t loopTpHandle = 0; + const auto candidateEid = CopyRawEid(candidate.eid); + const int probeRet = canProbeLoopTp ? + QueryTpHandleForPeer(ctxHandle, candidateEid, candidateEid, &loopTpHandle) : + TILEXR_SUCCESS; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute loopEidCandidate" + << " eidIndex=" << candidate.eidIndex + << " dieId=" << candidate.dieId + << " funcId=" << candidate.funcId + << " eid=" << FormatEndpointEid(candidateEid) + << " probeRet=" << probeRet + << " tpHandle=0x" << std::hex << loopTpHandle << std::dec + << std::endl; + } + if (probeRet == TILEXR_SUCCESS) { + selectedEid = candidate; + break; + } + (void)loader_.RaCtxDeinit(ctxHandle); + ctxHandle = nullptr; + } + if (ctxHandle == nullptr) { + return TILEXR_ERROR_NOT_FOUND; } TileXRCcuHccpTokenId allocatedToken {}; @@ -1060,8 +1243,51 @@ void TileXRCcuDirectRuntime::ReleasePeerEndpointImports() endpointPeerRemoteQpHandles_.clear(); } +void TileXRCcuDirectRuntime::ReleasePeerEndpointState(TileXRCcuPeerEndpointState* state) +{ + if (state == nullptr) { + return; + } + if (state->remoteQpHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxQpUnimport != nullptr) { + (void)loader_.RaCtxQpUnimport(state->resourceWindow.raCtxHandle, state->remoteQpHandle); + } + if (state->qpHandle != nullptr && loader_.RaCtxQpDestroy != nullptr) { + (void)loader_.RaCtxQpDestroy(state->qpHandle); + } + if (state->cqHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxCqDestroy != nullptr) { + (void)loader_.RaCtxCqDestroy(state->resourceWindow.raCtxHandle, state->cqHandle); + } + if (state->resourceWindow.lmemHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister( + state->resourceWindow.raCtxHandle, + state->resourceWindow.lmemHandle); + } + if (state->resourceWindow.tokenIdHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree( + state->resourceWindow.raCtxHandle, + state->resourceWindow.tokenIdHandle); + } + if (state->resourceWindow.raCtxHandle != nullptr && loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(state->resourceWindow.raCtxHandle); + } + *state = TileXRCcuPeerEndpointState {}; +} + +void TileXRCcuDirectRuntime::ReleasePeerEndpointRoutes() +{ + for (auto it = peerEndpointStates_.rbegin(); it != peerEndpointStates_.rend(); ++it) { + ReleasePeerEndpointState(&*it); + } + peerEndpointStates_.clear(); +} + void TileXRCcuDirectRuntime::ReleaseLocalEndpointRoute() { + ReleasePeerEndpointRoutes(); ReleasePeerEndpointImports(); if (endpointRouteBound_ && endpointQpHandle_ != nullptr && loader_.RaCtxQpUnbind != nullptr) { (void)loader_.RaCtxQpUnbind(endpointQpHandle_); @@ -1189,6 +1415,7 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( qpAttr.ub.jfsFlag.bs.errorSuspend = 1; qpAttr.ub.priority = 2; qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; + qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; qpAttr.ub.extMode.cstmFlag.value = 0; qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; qpAttr.ub.extMode.sq.buffVa = sqVa; @@ -1322,6 +1549,9 @@ int TileXRCcuDirectRuntime::RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRun return Fail(report, "direct CCU resource window is not registered for endpoint route collection", TILEXR_ERROR_NOT_INITIALIZED); } + if (options_.rankSize > 2) { + return PreparePeerEndpointRoutes(report); + } TileXRCcuLowerLayerTransportRoute route; int ret = TILEXR_ERROR_NOT_FOUND; @@ -1413,12 +1643,498 @@ int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindow return TILEXR_SUCCESS; } +int TileXRCcuDirectRuntime::CreatePeerEndpointState( + uint32_t peerRank, + uint32_t peerDevicePhyId, + const std::array& localEid, + const std::array& peerEid, + uint32_t peerOrdinal, + TileXRCcuPeerEndpointState* state) +{ + if (state == nullptr || !resourceWindowRegistered_ || localResourceWindow_.addr == 0 || + !HasRaCtxResourceWindowSymbols(loader_) || !HasRaCtxEndpointRouteSymbols(loader_)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *state = TileXRCcuPeerEndpointState {}; + state->peerRank = peerRank; + state->peerDevicePhyId = peerDevicePhyId; + + TileXRCcuRaInfo raInfo {}; + raInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + raInfo.phyId = devicePhyId_; + uint32_t eidNum = 0; + int ret = loader_.RaGetDevEidInfoNum(raInfo, &eidNum); + if (ret != 0 || eidNum == 0) { + return TILEXR_ERROR_MKIRT; + } + std::vector eidInfos(eidNum); + uint32_t queriedEidNum = eidNum; + ret = loader_.RaGetDevEidInfoList(raInfo, eidInfos.data(), &queriedEidNum); + if (ret != 0 || queriedEidNum == 0) { + return TILEXR_ERROR_MKIRT; + } + const auto eidIt = std::find_if( + eidInfos.begin(), + eidInfos.begin() + queriedEidNum, + [&localEid](const TileXRCcuHccpDevEidInfo& info) { + return SameEid(localEid, info.eid); + }); + if (eidIt == eidInfos.begin() + queriedEidNum) { + return TILEXR_ERROR_NOT_FOUND; + } + state->eidInfo = *eidIt; + + TileXRCcuHccpCtxInitCfg ctxCfg {}; + ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; + ctxCfg.rdma.disabledLiteThread = false; + TileXRCcuHccpCtxInitAttr ctxAttr {}; + ctxAttr.phyId = devicePhyId_; + ctxAttr.ub.eidIndex = state->eidInfo.eidIndex; + ctxAttr.ub.eid = state->eidInfo.eid; + ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &state->resourceWindow.raCtxHandle); + if (ret != 0 || state->resourceWindow.raCtxHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuHccpTokenId allocatedToken {}; + ret = loader_.RaCtxTokenIdAlloc( + state->resourceWindow.raCtxHandle, + &allocatedToken, + &state->resourceWindow.tokenIdHandle); + if (ret != 0 || state->resourceWindow.tokenIdHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + TileXRCcuRaInfo randomInfo {}; + randomInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + randomInfo.phyId = devicePhyId_; + ret = loader_.RaGetSecRandom(&randomInfo, &state->resourceWindow.tokenValue); + if (ret != 0) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + const uint64_t alignedAddr = AlignResourceWindowAddr(localResourceWindow_.addr); + TileXRCcuHccpMrRegInfo mr {}; + mr.in.mem.addr = alignedAddr; + mr.in.mem.size = localResourceWindow_.bytes + (localResourceWindow_.addr - alignedAddr); + mr.in.ub.flags.value = 0; + mr.in.ub.flags.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + mr.in.ub.flags.bs.tokenIdValid = 1; + mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; + mr.in.ub.flags.bs.nonPin = 1; + mr.in.ub.tokenValue = state->resourceWindow.tokenValue; + mr.in.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; + ret = loader_.RaCtxLmemRegister( + state->resourceWindow.raCtxHandle, + &mr, + &state->resourceWindow.lmemHandle); + if (ret != 0 || state->resourceWindow.lmemHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + const uint32_t rawTokenId = mr.out.ub.tokenId != 0 ? mr.out.ub.tokenId : allocatedToken.tokenId; + state->resourceWindow.addr = localResourceWindow_.addr; + state->resourceWindow.bytes = localResourceWindow_.bytes; + state->resourceWindow.rawTokenId = rawTokenId; + state->resourceWindow.tokenId = rawTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + state->resourceWindow.targetSegHandle = mr.out.ub.targetSegHandle; + state->resourceWindow.eid = localEid; + state->resourceWindow.eidIndex = state->eidInfo.eidIndex; + state->resourceWindow.funcId = state->eidInfo.funcId; + state->resourceWindow.funcIdValid = true; + state->resourceWindow.raCtxRegistered = true; + + ret = SelectTpRouteForPeer( + state->resourceWindow.raCtxHandle, + localEid, + peerEid, + &state->localTpHandle, + &state->mappedJettyPriority); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointState(state); + return ret; + } + + TileXRCcuHccpCqInfo cqInfo {}; + cqInfo.in.chanHandle = nullptr; + cqInfo.in.depth = TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH; + cqInfo.in.ub.userCtx = 0; + cqInfo.in.ub.mode = TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL; + cqInfo.in.ub.ceqn = 0; + cqInfo.in.ub.flag.value = 0; + ret = loader_.RaCtxCqCreate(state->resourceWindow.raCtxHandle, &cqInfo, &state->cqHandle); + if (ret != 0 || state->cqHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + const uint32_t sqDepth = SelectEndpointRouteSqDepth(); + TileXRCcuHccpQpCreateAttr qpAttr {}; + qpAttr.scqHandle = state->cqHandle; + qpAttr.rcqHandle = state->cqHandle; + qpAttr.srqHandle = state->cqHandle; + qpAttr.sqDepth = sqDepth; + qpAttr.rqDepth = TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT; + qpAttr.transportMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + qpAttr.ub.mode = static_cast(TILEXR_CCU_HCCP_JETTY_MODE_CCU); + qpAttr.ub.jettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + peerOrdinal); + qpAttr.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; + qpAttr.ub.tokenValue = state->resourceWindow.tokenValue; + qpAttr.ub.flag.value = 0; + qpAttr.ub.flag.bs.shareJfr = 1; + qpAttr.ub.jfsFlag.bs.errorSuspend = 1; + qpAttr.ub.priority = state->mappedJettyPriority; + qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; + qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; + qpAttr.ub.extMode.cstmFlag.value = 0; + qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; + qpAttr.ub.extMode.sq.buffVa = SelectEndpointRouteSqVa(localResourceWindow_) + + static_cast(peerOrdinal) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; + qpAttr.ub.extMode.sq.buffSize = SelectEndpointRouteSqBytes(sqDepth); + qpAttr.ub.extMode.sqebbNum = sqDepth; + ret = loader_.RaCtxQpCreate( + state->resourceWindow.raCtxHandle, + &qpAttr, + &state->qpInfo, + &state->qpHandle); + if (ret != 0 || state->qpHandle == nullptr || state->qpInfo.key.size == 0) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + state->psn = endpointPsn_++; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace peerEndpoint created" + << " rank=" << options_.rank + << " peerRank=" << peerRank + << " peerDevice=" << peerDevicePhyId + << " localEid=" << FormatEndpointEid(localEid) + << " eidIndex=" << state->eidInfo.eidIndex + << " funcId=" << state->eidInfo.funcId + << " tpHandle=0x" << std::hex << state->localTpHandle << std::dec + << " priority=" << static_cast(state->mappedJettyPriority) + << " qpId=" << state->qpInfo.ub.id + << " psn=" << state->psn + << std::endl; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeReport* report) +{ + ReleasePeerEndpointRoutes(); + std::vector allDevicePhyIds(static_cast(options_.rankSize), 0); + int ret = options_.allGather( + &devicePhyId_, + sizeof(devicePhyId_), + allDevicePhyIds.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "failed to exchange physical device ids for CCU endpoint routes", ret); + } + std::vector peerDevicePhyIds; + std::vector peerRanks; + for (int peer = 0; peer < options_.rankSize; ++peer) { + if (peer != options_.rank) { + peerRanks.push_back(static_cast(peer)); + peerDevicePhyIds.push_back(allDevicePhyIds[static_cast(peer)]); + } + } + std::vector topologyRoutes; + std::string topologyMessage; + ret = TileXRCcuResolvePeerEidRoutes( + TILEXR_CCU_HCCL_ROOT_INFO_PATH, + devicePhyId_, + peerDevicePhyIds, + &topologyRoutes, + &topologyMessage); + if (ret != TILEXR_SUCCESS || topologyRoutes.size() != peerRanks.size()) { + return Fail(report, topologyMessage.empty() ? + "failed to resolve peer-specific CCU EIDs" : topologyMessage, ret); + } + + std::vector> localEidsByPeer( + static_cast(options_.rankSize)); + for (uint32_t ordinal = 0; ordinal < peerRanks.size(); ++ordinal) { + localEidsByPeer[peerRanks[ordinal]] = topologyRoutes[ordinal].localEid; + } + std::vector> allLocalEidsByPeer( + static_cast(options_.rankSize) * static_cast(options_.rankSize)); + ret = options_.allGather( + localEidsByPeer.data(), + localEidsByPeer.size() * sizeof(localEidsByPeer.front()), + allLocalEidsByPeer.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "failed to exchange peer-specific CCU topology EIDs", ret); + } + + peerEndpointStates_.reserve(peerRanks.size()); + for (uint32_t ordinal = 0; ordinal < peerRanks.size(); ++ordinal) { + const auto& peerEid = allLocalEidsByPeer[ + static_cast(peerRanks[ordinal]) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + if (IsEmptyEid(peerEid)) { + ReleasePeerEndpointRoutes(); + return Fail(report, "missing reciprocal peer-specific CCU topology EID", TILEXR_ERROR_NOT_FOUND); + } + TileXRCcuPeerEndpointState state; + ret = CreatePeerEndpointState( + peerRanks[ordinal], + peerDevicePhyIds[ordinal], + topologyRoutes[ordinal].localEid, + peerEid, + ordinal, + &state); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to create peer-specific CCU endpoint", ret); + } + peerEndpointStates_.push_back(state); + } + + std::vector localOffers(static_cast(options_.rankSize)); + for (const auto& state : peerEndpointStates_) { + auto& offer = localOffers[state.peerRank]; + offer.resourceAddr = state.resourceWindow.addr; + offer.resourceTokenId = state.resourceWindow.tokenId; + offer.resourceRawTokenId = state.resourceWindow.rawTokenId; + offer.resourceTokenValue = state.resourceWindow.tokenValue; + offer.jettyTokenValue = state.resourceWindow.tokenValue; + offer.eid = state.resourceWindow.eid; + offer.qpKey = state.qpInfo.key; + if (offer.qpKey.size == 0 || offer.qpKey.size > TILEXR_CCU_HCCP_QP_KEY_BYTES) { + ReleasePeerEndpointRoutes(); + return Fail(report, "peer-specific CCU QP key has an invalid size", TILEXR_ERROR_MKIRT); + } + offer.psn = state.psn; + offer.funcId = state.resourceWindow.funcId; + offer.funcIdValid = state.resourceWindow.funcIdValid; + offer.valid = true; + } + std::vector allOffers( + static_cast(options_.rankSize) * static_cast(options_.rankSize)); + ret = options_.allGather( + localOffers.data(), + localOffers.size() * sizeof(TileXRCcuPeerEndpointOffer), + allOffers.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to exchange peer-specific CCU endpoint offers", ret); + } + + std::vector localTpHandles(static_cast(options_.rankSize), 0); + for (auto& state : peerEndpointStates_) { + const auto& peerOffer = allOffers[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + if (!peerOffer.valid || peerOffer.qpKey.size == 0) { + ReleasePeerEndpointRoutes(); + return Fail(report, "missing reciprocal peer-specific CCU endpoint offer", TILEXR_ERROR_NOT_FOUND); + } + localTpHandles[state.peerRank] = state.localTpHandle; + } + std::vector allTpHandles( + static_cast(options_.rankSize) * static_cast(options_.rankSize), 0); + ret = options_.allGather( + localTpHandles.data(), + localTpHandles.size() * sizeof(uint64_t), + allTpHandles.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to exchange peer-specific CCU TP handles", ret); + } + + for (uint32_t ordinal = 0; ordinal < peerEndpointStates_.size(); ++ordinal) { + auto& state = peerEndpointStates_[ordinal]; + const auto& peerOffer = allOffers[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + const uint64_t localTpHandle = localTpHandles[state.peerRank]; + const uint64_t peerTpHandle = allTpHandles[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + TileXRCcuHccpQpImportInfo importInfo {}; + importInfo.in.key = peerOffer.qpKey; + if (importInfo.in.key.size == 0 || importInfo.in.key.size > TILEXR_CCU_HCCP_QP_KEY_BYTES) { + ReleasePeerEndpointRoutes(); + return Fail(report, "peer-specific remote CCU QP key has an invalid size", TILEXR_ERROR_MKIRT); + } + importInfo.in.ub.mode = TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP; + importInfo.in.ub.tokenValue = peerOffer.jettyTokenValue; + importInfo.in.ub.policy = TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR; + importInfo.in.ub.type = TILEXR_CCU_HCCP_TARGET_TYPE_JETTY; + importInfo.in.ub.flag.value = 0; + importInfo.in.ub.flag.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + importInfo.in.ub.expImportCfg.tpHandle = localTpHandle; + importInfo.in.ub.expImportCfg.peerTpHandle = peerTpHandle; + importInfo.in.ub.expImportCfg.txPsn = state.psn; + importInfo.in.ub.expImportCfg.rxPsn = peerOffer.psn; + importInfo.in.ub.tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; + ret = loader_.RaCtxQpImport( + state.resourceWindow.raCtxHandle, + &importInfo, + &state.remoteQpHandle); + if (ret != 0 || state.remoteQpHandle == nullptr) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to import peer-specific CCU QP", TILEXR_ERROR_MKIRT); + } + state.route.remoteEid = ReverseEndpointEid(peerOffer.eid); + state.route.tpn = importInfo.out.ub.tpn; + state.route.doorbellVa = state.qpInfo.ub.dbAddr; + state.route.doorbellTokenId = + state.qpInfo.ub.dbTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + state.route.doorbellTokenValue = state.resourceWindow.tokenValue; + state.route.sqDepth = SelectEndpointRouteSqDepth(); + state.route.startJettyId = static_cast(state.qpInfo.ub.id); + state.route.remoteCcuVa = peerOffer.resourceAddr; + state.route.memoryTokenId = peerOffer.resourceTokenId; + state.route.memoryTokenValue = peerOffer.resourceTokenValue; + state.route.endpointRouteVerified = true; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace peerEndpoint imported" + << " rank=" << options_.rank + << " peerRank=" << state.peerRank + << " localEid=" << FormatEndpointEid(state.resourceWindow.eid) + << " peerEid=" << FormatEndpointEid(peerOffer.eid) + << " localTpHandle=0x" << std::hex << localTpHandle + << " peerTpHandle=0x" << peerTpHandle + << std::dec + << " localMemoryTokenId=0x" << std::hex << state.resourceWindow.tokenId + << " peerMemoryTokenId=0x" << peerOffer.resourceTokenId + << " localCcuResourceTokenId=0x" << state.resourceWindow.tokenId + << std::dec + << " localPsn=" << state.psn + << " peerPsn=" << peerOffer.psn + << " tpn=0x" << std::hex << state.route.tpn + << " doorbellVa=0x" << state.route.doorbellVa + << " remoteCcuVa=0x" << state.route.remoteCcuVa + << std::dec + << " taJettyId=" << state.route.startJettyId + << std::endl; + } + } + localVerifiedEndpointRoute_ = peerEndpointStates_.front().route; + localVerifiedEndpointRouteValid_ = true; + if (report != nullptr) { + report->initialized = initialized_; + report->message = "peer-specific direct CCU endpoint routes prepared"; + } + return TILEXR_SUCCESS; +} + int TileXRCcuDirectRuntime::QueryTpHandleForPeer( const std::array& peerEid, uint64_t* tpHandle) { - if (tpHandle == nullptr || IsEmptyEid(localResourceWindow_.eid) || IsEmptyEid(peerEid) || - localResourceWindow_.raCtxHandle == nullptr || loader_.RaGetTpInfoListAsync == nullptr || + return QueryTpHandleForPeer( + localResourceWindow_.raCtxHandle, + localResourceWindow_.eid, + peerEid, + tpHandle); +} + +int TileXRCcuDirectRuntime::SelectTpRouteForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle, + uint8_t* mappedJettyPriority) +{ + if (ctxHandle == nullptr || tpHandle == nullptr || mappedJettyPriority == nullptr || + IsEmptyEid(localEid) || IsEmptyEid(peerEid) || + loader_.RaGetTpInfoListAsync == nullptr || loader_.RaGetTpAttrAsync == nullptr || + loader_.RaSetTpAttrAsync == nullptr || loader_.RaGetAsyncReqResult == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *tpHandle = 0; + *mappedJettyPriority = 0; + + TileXRCcuHccpGetTpCfg tpCfg {}; + tpCfg.flag.bs.rtp = 1; + tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + std::copy(localEid.begin(), localEid.end(), tpCfg.localEid.raw); + std::copy(peerEid.begin(), peerEid.end(), tpCfg.peerEid.raw); + + std::array tpInfos {}; + uint32_t tpInfoNum = static_cast(tpInfos.size()); + void* reqHandle = nullptr; + int ret = loader_.RaGetTpInfoListAsync( + ctxHandle, &tpCfg, tpInfos.data(), &tpInfoNum, &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS || tpInfoNum == 0 || tpInfoNum > tpInfos.size()) { + return ret == TILEXR_SUCCESS ? TILEXR_ERROR_NOT_FOUND : ret; + } + + TileXRCcuHccpTpAttr attr {}; + uint32_t attrBitmap = TILEXR_CCU_TP_ATTR_BITMAP_SL_AVAILABLE | + TILEXR_CCU_TP_ATTR_BITMAP_SL; + reqHandle = nullptr; + ret = loader_.RaGetTpAttrAsync( + ctxHandle, tpInfos[0].tpHandle, &attrBitmap, &attr, &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + uint32_t tpIndex = 0; + uint8_t mappedSl = 0; + if (!MapQosToTpAndSl( + TILEXR_CCU_DEFAULT_HCCL_QOS, tpInfoNum, attr.slBitmap, &tpIndex, &mappedSl) || + tpInfos[tpIndex].tpHandle == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuHccpTpAttr setAttr {}; + setAttr.sl = mappedSl; + reqHandle = nullptr; + ret = loader_.RaSetTpAttrAsync( + ctxHandle, + tpInfos[tpIndex].tpHandle, + TILEXR_CCU_TP_ATTR_BITMAP_SL, + &setAttr, + &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute selectedTp" + << " localEid=" << FormatEndpointEid(localEid) + << " peerEid=" << FormatEndpointEid(peerEid) + << " tpCount=" << tpInfoNum + << " slBitmap=0x" << std::hex << attr.slBitmap + << " tpIndex=" << std::dec << tpIndex + << " tpHandle=0x" << std::hex << tpInfos[tpIndex].tpHandle + << std::dec << " mappedSl=" << static_cast(mappedSl) + << std::endl; + } + *tpHandle = tpInfos[tpIndex].tpHandle; + *mappedJettyPriority = mappedSl; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::QueryTpHandleForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle) +{ + if (tpHandle == nullptr || IsEmptyEid(localEid) || IsEmptyEid(peerEid) || + ctxHandle == nullptr || loader_.RaGetTpInfoListAsync == nullptr || loader_.RaGetAsyncReqResult == nullptr) { return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -1429,7 +2145,7 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( tpCfg.flag.bs.rtp = 1; tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { - tpCfg.localEid.raw[i] = localResourceWindow_.eid[i]; + tpCfg.localEid.raw[i] = localEid[i]; tpCfg.peerEid.raw[i] = peerEid[i]; } @@ -1437,7 +2153,7 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( uint32_t tpInfoNum = 1; void* reqHandle = nullptr; const int ret = loader_.RaGetTpInfoListAsync( - localResourceWindow_.raCtxHandle, + ctxHandle, &tpCfg, &tpInfo, &tpInfoNum, @@ -1460,6 +2176,13 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( if (tpInfo.tpHandle == 0) { return TILEXR_ERROR_NOT_FOUND; } + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute peerTpInfoReady" + << " localEid=" << FormatEndpointEid(localEid) + << " peerEid=" << FormatEndpointEid(peerEid) + << " tpHandle=0x" << std::hex << tpInfo.tpHandle + << std::dec << std::endl; + } *tpHandle = tpInfo.tpHandle; return TILEXR_SUCCESS; } @@ -1534,6 +2257,33 @@ int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers(std::vectorreserve(peerEndpointStates_.size()); + for (const auto& state : peerEndpointStates_) { + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = state.route.remoteCcuVa; + remote.peerRank = state.peerRank; + remote.memoryTokenId = state.route.memoryTokenId; + remote.rawMemoryTokenId = state.route.memoryTokenId << TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + remote.memoryTokenValue = state.route.memoryTokenValue; + remote.localPfeId = state.resourceWindow.funcId; + remote.localPfeIdValid = state.resourceWindow.funcIdValid; + remote.remoteEid = state.route.remoteEid; + remote.tpn = state.route.tpn; + remote.doorbellVa = state.route.doorbellVa; + remote.doorbellTokenId = state.route.doorbellTokenId; + remote.doorbellTokenValue = state.route.doorbellTokenValue; + remote.sqDepth = state.route.sqDepth; + remote.startJettyId = state.route.startJettyId; + remote.localDoorbellVa = state.route.doorbellVa; + remote.localDoorbellTokenId = state.route.doorbellTokenId; + remote.localDoorbellTokenValue = state.route.doorbellTokenValue; + remote.localSqDepth = state.route.sqDepth; + remote.endpointRouteVerified = true; + buffers->push_back(remote); + } + return TILEXR_SUCCESS; + } if (options_.rankSize <= 1) { return TILEXR_SUCCESS; } diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.h b/src/comm/ccu/tilexr_ccu_direct_runtime.h index 83f73dac..f664a89e 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.h +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.h @@ -127,6 +127,21 @@ struct TileXRCcuDirectRuntimeReport { std::string message; }; +struct TileXRCcuPeerEndpointState { + uint32_t peerRank = 0; + uint32_t peerDevicePhyId = 0; + TileXRCcuLocalResourceWindowInfo resourceWindow; + TileXRCcuHccpDevEidInfo eidInfo {}; + void* cqHandle = nullptr; + void* qpHandle = nullptr; + void* remoteQpHandle = nullptr; + TileXRCcuHccpQpCreateInfo qpInfo {}; + TileXRCcuLowerLayerTransportRoute route; + uint32_t psn = 0; + uint64_t localTpHandle = 0; + uint8_t mappedJettyPriority = 0; +}; + class TileXRCcuDirectRuntime { public: int Init(const TileXRCcuDirectRuntimeOptions& options, TileXRCcuDirectRuntimeReport* report); @@ -155,7 +170,26 @@ class TileXRCcuDirectRuntime { int CollectLocalEndpointRouteWithRaCtxOnce( TileXRCcuLowerLayerTransportRoute* route, bool* asyncWaitFailed); + int PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeReport* report); + int CreatePeerEndpointState( + uint32_t peerRank, + uint32_t peerDevicePhyId, + const std::array& localEid, + const std::array& peerEid, + uint32_t peerOrdinal, + TileXRCcuPeerEndpointState* state); + int SelectTpRouteForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle, + uint8_t* mappedJettyPriority); + int QueryTpHandleForPeer( + const std::array& peerEid, + uint64_t* tpHandle); int QueryTpHandleForPeer( + void* ctxHandle, + const std::array& localEid, const std::array& peerEid, uint64_t* tpHandle); int ImportPeerEndpointRoute( @@ -170,6 +204,8 @@ class TileXRCcuDirectRuntime { void ReleaseRegisteredMemoryBuffers(); void ReleaseRegisteredResourceWindow(); void ReleaseLocalEndpointRoute(); + void ReleasePeerEndpointState(TileXRCcuPeerEndpointState* state); + void ReleasePeerEndpointRoutes(); TileXRCcuDirectRuntimeOptions options_; TileXRCcuHccpLoader loader_; @@ -185,6 +221,7 @@ class TileXRCcuDirectRuntime { void* endpointQpHandle_ = nullptr; void* endpointRemoteQpHandle_ = nullptr; std::vector endpointPeerRemoteQpHandles_; + std::vector peerEndpointStates_; std::vector registeredMemoryBuffers_; std::vector importedRemoteMemoryBuffers_; TileXRCcuHccpQpKey endpointQpKey_ = {}; diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp index 71945955..9d507b64 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp @@ -480,6 +480,12 @@ int TileXRCcuDriverAdapter::InstallMsidToken( return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_MSID_TOKEN, in, &out, report); } +int TileXRCcuDriverAdapter::SetTaskKill(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const +{ + TileXRCcuCustomChannelOut out; + return Call(dieId, TILEXR_CCU_U_OP_SET_TASKKILL, &out, report); +} + int TileXRCcuDriverAdapter::CleanTaskKillState(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const { TileXRCcuCustomChannelOut out; @@ -515,21 +521,34 @@ int TileXRCcuDriverAdapter::InstallJettyCtx( if (ctxs == nullptr) { return Fail(report, "missing CCU local jetty context payloads"); } - if (count == 0 || count > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { + if (count == 0) { return Fail(report, "invalid CCU local jetty context count"); } - TileXRCcuCustomChannelIn in; - InitRequest(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, &in); - in.offsetStartIdx = startJettyCtxId; - in.data.dataInfo.dataArraySize = count; - in.data.dataInfo.dataLen = count * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; - for (uint32_t i = 0; i < count; ++i) { - CopyPayloadToSlot(ctxs[i], &in.data.dataInfo.dataArray[i]); - } + uint32_t remaining = count; + uint32_t offset = startJettyCtxId; + uint32_t inputOffset = 0; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; + for (uint32_t i = 0; i < batch; ++i) { + CopyPayloadToSlot(ctxs[inputOffset + i], &in.data.dataInfo.dataArray[i]); + } - TileXRCcuCustomChannelOut out; - return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, in, &out, report); + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + remaining -= batch; + offset += batch; + inputOffset += batch; + } + return TILEXR_SUCCESS; } int TileXRCcuDriverAdapter::InstallChannelCtxV1( diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.h b/src/comm/ccu/tilexr_ccu_driver_adapter.h index ce6ce16a..b2c9a9e5 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.h +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.h @@ -122,6 +122,7 @@ class TileXRCcuDriverAdapter { uint32_t tokenId, uint32_t tokenValue, TileXRCcuDriverAdapterReport* report) const; + int SetTaskKill(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const; int CleanTaskKillState(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const; int InstallPfeCtx( uint8_t dieId, diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.cpp b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp index fcf96e0d..6d788ce1 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_loader.cpp +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp @@ -155,6 +155,8 @@ int TileXRCcuHccpLoader::Load(TileXRCcuHccpLoaderReport* report) LoadOptionalSymbol(raHandle_, RaCtxQpBind, "RaCtxQpBind", "ra_ctx_qp_bind"); LoadOptionalSymbol(raHandle_, RaCtxQpUnbind, "RaCtxQpUnbind", "ra_ctx_qp_unbind"); LoadOptionalSymbol(raHandle_, RaGetTpInfoListAsync, "RaGetTpInfoListAsync", "ra_get_tp_info_list_async"); + LoadOptionalSymbol(raHandle_, RaGetTpAttrAsync, "RaGetTpAttrAsync", "ra_get_tp_attr_async"); + LoadOptionalSymbol(raHandle_, RaSetTpAttrAsync, "RaSetTpAttrAsync", "ra_set_tp_attr_async"); LoadOptionalSymbol(raHandle_, RaGetAsyncReqResult, "RaGetAsyncReqResult", "ra_get_async_req_result"); runtimeHandle_ = dlopen("libruntime.so", RTLD_NOW); @@ -246,6 +248,8 @@ void TileXRCcuHccpLoader::Unload() RaCtxQpBind = nullptr; RaCtxQpUnbind = nullptr; RaGetTpInfoListAsync = nullptr; + RaGetTpAttrAsync = nullptr; + RaSetTpAttrAsync = nullptr; RaGetAsyncReqResult = nullptr; CollectLocalEndpointRoute = nullptr; loaded_ = false; diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.h b/src/comm/ccu/tilexr_ccu_hccp_loader.h index ef0f30ef..42cb5f57 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_loader.h +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.h @@ -87,6 +87,8 @@ class TileXRCcuHccpLoader { TileXRCcuRaCtxQpBindFunc RaCtxQpBind = nullptr; TileXRCcuRaCtxQpUnbindFunc RaCtxQpUnbind = nullptr; TileXRCcuRaGetTpInfoListAsyncFunc RaGetTpInfoListAsync = nullptr; + TileXRCcuRaGetTpAttrAsyncFunc RaGetTpAttrAsync = nullptr; + TileXRCcuRaSetTpAttrAsyncFunc RaSetTpAttrAsync = nullptr; TileXRCcuRaGetAsyncReqResultFunc RaGetAsyncReqResult = nullptr; TileXRCcuEndpointRouteProviderFunc CollectLocalEndpointRoute = nullptr; diff --git a/src/comm/ccu/tilexr_ccu_hccp_types.h b/src/comm/ccu/tilexr_ccu_hccp_types.h index 58460d30..9cb9b918 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_types.h +++ b/src/comm/ccu/tilexr_ccu_hccp_types.h @@ -493,6 +493,33 @@ struct TileXRCcuHccpTpInfo { uint32_t resv; }; +#pragma pack(push, 1) +struct TileXRCcuHccpTpAttr { + uint8_t retryTimesInit : 3; + uint8_t at : 5; + uint8_t sip[16]; + uint8_t dip[16]; + uint8_t sma[6]; + uint8_t dma[6]; + uint16_t vlanId : 12; + uint8_t vlanEn : 1; + uint8_t dscp : 6; + uint8_t atTimes : 5; + uint8_t sl : 4; + uint8_t ttl; + uint16_t ackUdpSrcport; + uint16_t dataUdpSrcport; + uint8_t udpSrcportRange : 4; + uint8_t sprayEn : 1; + uint8_t udpGlobalEn : 1; + uint8_t reserve0 : 2; + uint16_t slBitmap; + uint8_t dscpConfigMode : 1; + uint8_t reserve1 : 7; + uint8_t reserved[70]; +}; +#pragma pack(pop) + struct TileXRCcuRaInitConfig { uint32_t phyId; uint32_t nicPosition; @@ -608,6 +635,18 @@ using TileXRCcuRaGetTpInfoListAsyncFunc = int (*)( TileXRCcuHccpTpInfo infoList[], uint32_t* num, void** reqHandle); +using TileXRCcuRaGetTpAttrAsyncFunc = int (*)( + void* ctx, + uint64_t tpHandle, + uint32_t* attrBitmap, + TileXRCcuHccpTpAttr* attr, + void** reqHandle); +using TileXRCcuRaSetTpAttrAsyncFunc = int (*)( + void* ctx, + uint64_t tpHandle, + uint32_t attrBitmap, + TileXRCcuHccpTpAttr* attr, + void** reqHandle); using TileXRCcuRaGetAsyncReqResultFunc = int (*)(void* reqHandle, int* reqResult); using TileXRCcuEndpointRouteProviderFunc = int (*)( diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index fe0825af..39515270 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace TileXR { namespace { @@ -19,6 +20,7 @@ constexpr uint16_t TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE = 4; constexpr uint16_t TILEXR_CCU_HCOMM_WQE_BASIC_BLOCKS_PER_ROUTE = 256; constexpr uint32_t TILEXR_CCU_HCOMM_PER_DIE_PFE_RESERVED_NUM = 16; constexpr uint16_t TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM = 23; +constexpr uint16_t TILEXR_CCU_HCOMM_PER_DIE_JETTY_NUM = 128; constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID = 92; constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM = 36; constexpr uint32_t TILEXR_CCU_HCOMM_MAX_INNER_FE_ID = 7; @@ -140,7 +142,11 @@ void NormalizeVerifiedEndpointRouteJettyWindow(TileXRCcuLowerLayerTransportSnaps ++explicitStartCount; } - if (minExplicitStart != 0) { + const bool configuredWindowContainsExplicitJettys = + snapshot->startJettyId != 0 && snapshot->pfeJettyCount != 0 && minExplicitStart != 0 && + minExplicitStart >= snapshot->startJettyId && + explicitEnd <= static_cast(snapshot->startJettyId) + snapshot->pfeJettyCount; + if (minExplicitStart != 0 && !configuredWindowContainsExplicitJettys) { snapshot->startJettyId = minExplicitStart; } @@ -193,7 +199,7 @@ void ApplyHcommOrderedPfePartition(TileXRCcuLowerLayerTransportSnapshot* snapsho return; } snapshot->startLocalJettyCtxId = 0; - snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_PER_DIE_JETTY_NUM; snapshot->startJettyId = TILEXR_CCU_DEFAULT_START_JETTY_ID; } @@ -326,6 +332,14 @@ void AppendRemoteXnClears( if (plan == nullptr) { return; } + if (snapshot.remoteXnStartId != 0 && snapshot.remoteXnCount != 0) { + plan->xnClears.push_back({ + snapshot.dieId, + snapshot.remoteXnStartId, + snapshot.remoteXnCount, + }); + return; + } std::vector remoteXns; remoteXns.reserve(snapshot.routes.size()); for (const auto& route : snapshot.routes) { @@ -403,7 +417,9 @@ int TileXRCcuBuildLowerLayerInstallPlan( TileXRCcuJettyInstall jettyInstall; jettyInstall.dieId = spec.pfe.dieId; - jettyInstall.startJettyCtxId = spec.pfe.startLocalJettyCtxId; + jettyInstall.startJettyCtxId = spec.jettys.front().startJettyCtxId == 0 ? + spec.pfe.startLocalJettyCtxId : + spec.jettys.front().startJettyCtxId; for (const auto& jettySpec : spec.jettys) { if (jettySpec.startJettyCtxId != 0 && jettySpec.startJettyCtxId != jettyInstall.startJettyCtxId + jettyInstall.ctxs.size()) { @@ -512,13 +528,15 @@ int TileXRCcuBuildLowerLayerTransportTemplate( ApplyLowerLayerPfePartition(result.pfeId, &result); result.xnStartId = allocation.localXn.startId; result.xnCount = allocation.localXn.num; + result.remoteXnStartId = allocation.remoteXn.startId; + result.remoteXnCount = allocation.remoteXn.num; result.ckeStartId = localWaitCke.startId; result.ckeCount = localWaitCke.num; result.routes.reserve(remoteCcuBuffers.size()); const uint16_t wqeBasicBlockStride = SelectLowerLayerWqeBasicBlockStride(); - uint16_t verifiedStartJettyId = 0; - uint32_t verifiedJettyEnd = 0; + std::map wqeStartByJettyId; + uint32_t nextVerifiedWqeStartId = 0; for (uint32_t i = 0; i < remoteCcuBuffers.size(); ++i) { const auto& remoteCcuBuffer = remoteCcuBuffers[i]; if (remoteCcuBuffer.remoteCcuVa == 0) { @@ -538,7 +556,6 @@ int TileXRCcuBuildLowerLayerTransportTemplate( if (i > std::numeric_limits::max() / wqeBasicBlockStride) { return Fail(nullptr, report, "lower-layer CCU WQE basic block start overflows"); } - route.wqeBasicBlockStartId = static_cast(i * wqeBasicBlockStride); route.remoteCcuVa = remoteCcuBuffer.remoteCcuVa; route.memoryTokenId = remoteCcuBuffer.memoryTokenId; route.memoryTokenValue = remoteCcuBuffer.memoryTokenValue; @@ -561,22 +578,42 @@ int TileXRCcuBuildLowerLayerTransportTemplate( route.localSqDepth = remoteCcuBuffer.localSqDepth; route.startJettyId = remoteCcuBuffer.startJettyId; route.endpointRouteVerified = true; - if (route.startJettyId != 0) { - verifiedStartJettyId = verifiedStartJettyId == 0 ? - route.startJettyId : - std::min(verifiedStartJettyId, route.startJettyId); - verifiedJettyEnd = std::max( - verifiedJettyEnd, - static_cast(route.startJettyId) + 1U); + } + uint32_t wqeOrdinal = i; + if (route.startJettyId != 0) { + const uint32_t sqDepth = route.localSqDepth == 0 ? route.sqDepth : route.localSqDepth; + const uint32_t wqeBasicBlockCount = sqDepth * TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE; + if (wqeBasicBlockCount == 0 || nextVerifiedWqeStartId > std::numeric_limits::max()) { + return Fail(nullptr, report, "invalid verified endpoint WQE basic block window"); + } + uint32_t wqeBasicBlockStartId = nextVerifiedWqeStartId; + const bool pfeWindowContainsJetty = result.pfeJettyCount != 0 && + route.startJettyId >= result.startJettyId && + static_cast(route.startJettyId) < + static_cast(result.startJettyId) + result.pfeJettyCount; + if (pfeWindowContainsJetty) { + const uint32_t localJettyOffset = + static_cast(route.startJettyId) - result.startJettyId; + wqeBasicBlockStartId = localJettyOffset * wqeBasicBlockCount; + } + if (wqeBasicBlockStartId > std::numeric_limits::max()) { + return Fail(nullptr, report, "verified endpoint WQE basic block start overflows"); + } + const auto inserted = wqeStartByJettyId.emplace( + route.startJettyId, + static_cast(wqeBasicBlockStartId)); + route.wqeBasicBlockStartId = inserted.first->second; + if (inserted.second && !pfeWindowContainsJetty) { + nextVerifiedWqeStartId += wqeBasicBlockCount; + } + } else { + if (wqeOrdinal > std::numeric_limits::max() / wqeBasicBlockStride) { + return Fail(nullptr, report, "lower-layer CCU WQE basic block start overflows"); } + route.wqeBasicBlockStartId = static_cast(wqeOrdinal * wqeBasicBlockStride); } result.routes.push_back(route); } - if (verifiedStartJettyId != 0) { - result.startJettyId = verifiedStartJettyId; - const uint32_t requiredJettyCount = verifiedJettyEnd - verifiedStartJettyId; - result.pfeJettyCount = CheckedU16(std::max(result.pfeJettyCount, requiredJettyCount)); - } NormalizeVerifiedEndpointRouteJettyWindow(&result); *snapshot = result; @@ -670,12 +707,28 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( spec.ckeClear.count = normalized.ckeCount; spec.ckeClear.valid = normalized.ckeCount != 0; - uint32_t routeIndex = 0; - for (const auto& route : normalized.routes) { + std::map jettyRoutes; + for (uint32_t i = 0; i < normalized.routes.size(); ++i) { + const auto& route = normalized.routes[i]; + const uint16_t jettyId = route.startJettyId == 0 ? + static_cast(normalized.startJettyId + i) : + route.startJettyId; + jettyRoutes.emplace(jettyId, &route); + } + for (const auto& entry : jettyRoutes) { + if (entry.first < normalized.startJettyId) { + return Fail(plan, report, "lower-layer CCU endpoint jetty ID precedes the PFE jetty window"); + } + const uint32_t localJettyOffset = static_cast(entry.first) - normalized.startJettyId; + if (localJettyOffset >= normalized.pfeJettyCount || + static_cast(normalized.startLocalJettyCtxId) + localJettyOffset >= 128U) { + return Fail(plan, report, "lower-layer CCU endpoint jetty ID is outside the PFE jetty window"); + } + const auto& route = *entry.second; TileXRCcuLowerLayerJettySpec jetty; jetty.dieId = normalized.dieId; jetty.pfeId = normalized.pfeId; - jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + routeIndex); + jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + localJettyOffset); jetty.doorbellVa = route.localDoorbellVa == 0 ? route.doorbellVa : route.localDoorbellVa; jetty.doorbellTokenId = route.localDoorbellTokenId == 0 ? route.doorbellTokenId : @@ -686,7 +739,10 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( jetty.sqDepth = route.localSqDepth == 0 ? route.sqDepth : route.localSqDepth; jetty.wqeBasicBlockStartId = route.wqeBasicBlockStartId; spec.jettys.push_back(jetty); + } + uint32_t routeIndex = 0; + for (const auto& route : normalized.routes) { TileXRCcuLowerLayerChannelSpec channel; channel.dieId = normalized.dieId; channel.channelId = route.channelId; diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h index c37f908a..5a71bce9 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h @@ -153,6 +153,8 @@ struct TileXRCcuLowerLayerTransportSnapshot { uint16_t startLocalJettyCtxId = 0; uint32_t xnStartId = 0; uint32_t xnCount = 0; + uint32_t remoteXnStartId = 0; + uint32_t remoteXnCount = 0; uint32_t ckeStartId = 0; uint32_t ckeCount = 0; std::vector routes; diff --git a/src/comm/ccu/tilexr_ccu_microcode.cpp b/src/comm/ccu/tilexr_ccu_microcode.cpp index 597465a3..1ba5e4e3 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.cpp +++ b/src/comm/ccu/tilexr_ccu_microcode.cpp @@ -13,6 +13,8 @@ constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_GSA_HEADER = 0x0002U; constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_XN_HEADER = 0x0003U; constexpr uint64_t TILEXR_CCU_SET_CKE_HEADER = 0x0802U; constexpr uint64_t TILEXR_CCU_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MS_HEADER = 0x1000U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MS_TO_LOC_MEM_HEADER = 0x1002U; constexpr uint64_t TILEXR_CCU_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; @@ -78,6 +80,23 @@ int ValidateTransferSpec(const TileXRCcuMemTransferSpec& spec) return TILEXR_SUCCESS; } +int ValidateLocalMsTransferSpec(const TileXRCcuLocalMsTransferSpec& spec) +{ + if (spec.localGsa == 0 || spec.localXn == 0 || spec.lengthXn == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if ((spec.setCkeId == 0) != (spec.setCkeMask == 0) || + (spec.waitCkeId == 0) != (spec.waitCkeMask == 0)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +uint16_t LocalMsTransferFlagSlot(const TileXRCcuLocalMsTransferSpec& spec) +{ + return static_cast((spec.clearWait ? 1U : 0U) | (spec.lengthFromXn ? 2U : 0U)); +} + void WriteLe16(uint8_t* bytes, size_t offset, uint16_t value) { bytes[offset] = static_cast(value & 0xffU); @@ -273,6 +292,42 @@ int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, Til return TILEXR_SUCCESS; } +int TileXRCcuEncodeTransLocMemToLocMs(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || + ValidateLocalMsTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MS_HEADER), + spec.localMs, + spec.localGsa, + spec.localXn); + instr->words[1] = PackSlots(spec.lengthXn, spec.channelId, 0, 0); + instr->words[2] = PackSlots(0, 0, 0, LocalMsTransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeTransLocMsToLocMem(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || + ValidateLocalMsTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MS_TO_LOC_MEM_HEADER), + spec.localGsa, + spec.localXn, + spec.localMs); + instr->words[1] = PackSlots(spec.lengthXn, spec.channelId, 0, 0); + instr->words[2] = PackSlots(0, 0, 0, LocalMsTransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + int TileXRCcuBuildSqeLoadProgram(uint16_t firstXnId, uint32_t argCount, std::vector* program) { if (program == nullptr || firstXnId == 0 || argCount == 0 || argCount > TILEXR_CCU_SQE_ARGS_LEN) { diff --git a/src/comm/ccu/tilexr_ccu_microcode.h b/src/comm/ccu/tilexr_ccu_microcode.h index 65cfdbd1..fbe59783 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.h +++ b/src/comm/ccu/tilexr_ccu_microcode.h @@ -68,6 +68,20 @@ struct TileXRCcuMemTransferSpec { bool reduceEnabled = false; }; +struct TileXRCcuLocalMsTransferSpec { + uint16_t localGsa = 0; + uint16_t localXn = 0; + uint16_t localMs = 0; + uint16_t lengthXn = 0; + uint16_t channelId = 0; + uint16_t setCkeId = 0; + uint16_t setCkeMask = 0; + uint16_t waitCkeId = 0; + uint16_t waitCkeMask = 0; + bool clearWait = true; + bool lengthFromXn = true; +}; + int TileXRCcuEncodeLoadSqeArgsToX(uint16_t xnId, uint32_t sqeArgId, TileXRCcuInstr* instr); int TileXRCcuEncodeLoadImdToXn(uint16_t xnId, uint64_t immediate, uint16_t secFlag, TileXRCcuInstr* instr); @@ -88,6 +102,10 @@ int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, Til int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); +int TileXRCcuEncodeTransLocMemToLocMs(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeTransLocMsToLocMem(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr); + int TileXRCcuBuildSqeLoadProgram( uint16_t firstXnId, uint32_t argCount, diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp index 282d9fe2..5b4840f7 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp @@ -196,7 +196,10 @@ int TileXRCcuResourceAllocator::Allocate( const uint32_t requiredPostWaitInstructionCount = postOnly ? request.syncResourceCount : request.syncResourceCount * 2U; const uint32_t sourceCkeInitCount = syncCkeMode ? 1U : 0U; - const uint32_t sourceCkeResourceCount = syncCkeMode ? 1U : 0U; + const uint32_t sourceCkeResourceCount = syncCkeMode ? request.sourceCkeCount : 0U; + if (syncCkeMode && sourceCkeResourceCount == 0) { + return Fail(report, "invalid CCU source CKE resource request"); + } const uint32_t task1PreludeInstructionCount = hcommStyleTask1Prelude ? TILEXR_CCU_HCOMM_TASK1_PRELUDE_INSTRUCTION_COUNT : 0U; const uint32_t requiredBarrierInstructionCount = diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.h b/src/comm/ccu/tilexr_ccu_resource_allocator.h index 706ff2b4..959e712d 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.h +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.h @@ -43,6 +43,7 @@ struct TileXRCcuResourceRequest { uint32_t syncResourceCount = 0; uint32_t syncInstructionCount = 0; uint32_t bindingsPerSyncResource = 1; + uint32_t sourceCkeCount = 1; TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; }; diff --git a/src/comm/ccu/tilexr_ccu_topology.cpp b/src/comm/ccu/tilexr_ccu_topology.cpp new file mode 100644 index 00000000..fc20242a --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_topology.cpp @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_topology.h" + +#include "tilexr_types.h" + +#include +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +struct RootInfo { + std::string topoPath; + std::unordered_map deviceToLocalId; + std::unordered_map>> portToEidByLocalId; +}; + +struct TopoEdge { + uint32_t localA = 0; + uint32_t localB = 0; + std::vector localAPorts; + std::vector localBPorts; +}; + +std::string ReadTextFile(const std::string& path) +{ + std::ifstream input(path); + if (!input.is_open()) { + return {}; + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +bool ParseUint(const std::string& value, uint32_t* out) +{ + if (value.empty() || out == nullptr) { + return false; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value.c_str(), &end, 10); + if (end == value.c_str() || *end != '\0' || parsed > UINT32_MAX) { + return false; + } + *out = static_cast(parsed); + return true; +} + +std::string JsonStringField(const std::string& object, const std::string& field) +{ + const std::regex pattern("\"" + field + "\"\\s*:\\s*\"([^\"]*)\""); + std::smatch match; + return std::regex_search(object, match, pattern) ? match[1].str() : std::string(); +} + +bool JsonUintField(const std::string& object, const std::string& field, uint32_t* out) +{ + const std::regex quoted("\"" + field + "\"\\s*:\\s*\"([0-9]+)\""); + const std::regex plain("\"" + field + "\"\\s*:\\s*([0-9]+)"); + std::smatch match; + if (std::regex_search(object, match, quoted) || std::regex_search(object, match, plain)) { + return ParseUint(match[1].str(), out); + } + return false; +} + +bool ParseEidHex(const std::string& text, std::array* eid) +{ + if (eid == nullptr || text.size() != eid->size() * 2U) { + return false; + } + for (size_t i = 0; i < eid->size(); ++i) { + const char hi = text[i * 2U]; + const char lo = text[i * 2U + 1U]; + if (!std::isxdigit(static_cast(hi)) || + !std::isxdigit(static_cast(lo))) { + return false; + } + (*eid)[i] = static_cast( + std::strtoul(text.substr(i * 2U, 2U).c_str(), nullptr, 16)); + } + return true; +} + +std::vector JsonStringArrayField(const std::string& object, const std::string& field) +{ + const std::regex arrayPattern("\"" + field + "\"\\s*:\\s*\\[([^\\]]*)\\]"); + std::smatch arrayMatch; + if (!std::regex_search(object, arrayMatch, arrayPattern)) { + return {}; + } + const std::string body = arrayMatch[1].str(); + std::vector values; + const std::regex valuePattern("\"([^\"]*)\""); + for (auto it = std::sregex_iterator(body.begin(), body.end(), valuePattern); + it != std::sregex_iterator(); ++it) { + values.push_back((*it)[1].str()); + } + return values; +} + +std::vector ExtractObjectsWithKey(const std::string& text, const std::string& key) +{ + std::vector objects; + const std::string needle = "\"" + key + "\""; + size_t pos = 0; + while ((pos = text.find(needle, pos)) != std::string::npos) { + const size_t begin = text.rfind('{', pos); + if (begin == std::string::npos) { + ++pos; + continue; + } + int depth = 0; + bool inString = false; + bool escaped = false; + for (size_t i = begin; i < text.size(); ++i) { + const char ch = text[i]; + if (inString) { + escaped = !escaped && ch == '\\'; + if (ch == '"' && !escaped) { + inString = false; + } else if (ch != '\\') { + escaped = false; + } + continue; + } + if (ch == '"') { + inString = true; + } else if (ch == '{') { + ++depth; + } else if (ch == '}') { + --depth; + if (depth == 0) { + objects.emplace_back(text.substr(begin, i - begin + 1U)); + pos = i + 1U; + break; + } + } + } + if (depth != 0) { + break; + } + } + return objects; +} + +bool ParseRootInfo(const std::string& path, RootInfo* root) +{ + if (root == nullptr) { + return false; + } + const std::string content = ReadTextFile(path); + root->topoPath = JsonStringField(content, "topo_file_path"); + if (content.empty() || root->topoPath.empty()) { + return false; + } + for (const auto& rankObject : ExtractObjectsWithKey(content, "device_id")) { + uint32_t deviceId = 0; + uint32_t localId = 0; + if (!JsonUintField(rankObject, "device_id", &deviceId) || + !JsonUintField(rankObject, "local_id", &localId)) { + continue; + } + root->deviceToLocalId[deviceId] = localId; + for (const auto& addressObject : ExtractObjectsWithKey(rankObject, "addr")) { + std::array eid {}; + if (!ParseEidHex(JsonStringField(addressObject, "addr"), &eid)) { + continue; + } + for (const auto& port : JsonStringArrayField(addressObject, "ports")) { + root->portToEidByLocalId[localId][port] = eid; + } + } + } + return !root->deviceToLocalId.empty(); +} + +std::vector ParseTopoInfo(const std::string& path) +{ + const std::string content = ReadTextFile(path); + std::vector edges; + for (const auto& edgeObject : ExtractObjectsWithKey(content, "local_a")) { + TopoEdge edge; + if (!JsonUintField(edgeObject, "local_a", &edge.localA) || + !JsonUintField(edgeObject, "local_b", &edge.localB)) { + continue; + } + edge.localAPorts = JsonStringArrayField(edgeObject, "local_a_ports"); + edge.localBPorts = JsonStringArrayField(edgeObject, "local_b_ports"); + if (!edge.localAPorts.empty() && !edge.localBPorts.empty()) { + edges.push_back(edge); + } + } + return edges; +} + +bool ResolveLocalPort( + const std::vector& edges, + uint32_t localId, + uint32_t peerLocalId, + std::string* localPort) +{ + if (localPort == nullptr) { + return false; + } + for (const auto& edge : edges) { + if (edge.localA == localId && edge.localB == peerLocalId) { + *localPort = edge.localAPorts.front(); + return true; + } + if (edge.localB == localId && edge.localA == peerLocalId) { + *localPort = edge.localBPorts.front(); + return true; + } + } + return false; +} + +} // namespace + +int TileXRCcuResolvePeerEidRoutes( + const std::string& rootInfoPath, + uint32_t localDevicePhyId, + const std::vector& peerDevicePhyIds, + std::vector* routes, + std::string* message) +{ + if (routes == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + routes->clear(); + RootInfo root; + if (!ParseRootInfo(rootInfoPath, &root)) { + if (message != nullptr) { + *message = "failed to parse HCCL root info"; + } + return TILEXR_ERROR_NOT_FOUND; + } + const auto localIdIt = root.deviceToLocalId.find(localDevicePhyId); + if (localIdIt == root.deviceToLocalId.end()) { + if (message != nullptr) { + *message = "local physical device is absent from HCCL root info"; + } + return TILEXR_ERROR_NOT_FOUND; + } + const auto edges = ParseTopoInfo(root.topoPath); + const auto eidMapIt = root.portToEidByLocalId.find(localIdIt->second); + if (edges.empty() || eidMapIt == root.portToEidByLocalId.end()) { + if (message != nullptr) { + *message = "HCCL topology has no local EID routes"; + } + return TILEXR_ERROR_NOT_FOUND; + } + for (const uint32_t peerDevicePhyId : peerDevicePhyIds) { + const auto peerIdIt = root.deviceToLocalId.find(peerDevicePhyId); + std::string localPort; + if (peerIdIt == root.deviceToLocalId.end() || + !ResolveLocalPort(edges, localIdIt->second, peerIdIt->second, &localPort)) { + if (message != nullptr) { + *message = "HCCL topology has no device-pair edge"; + } + routes->clear(); + return TILEXR_ERROR_NOT_FOUND; + } + const auto eidIt = eidMapIt->second.find(localPort); + if (eidIt == eidMapIt->second.end()) { + if (message != nullptr) { + *message = "HCCL root info has no EID for the selected local port"; + } + routes->clear(); + return TILEXR_ERROR_NOT_FOUND; + } + TileXRCcuPeerEidRoute route; + route.peerDevicePhyId = peerDevicePhyId; + route.localEid = eidIt->second; + route.localPort = localPort; + routes->push_back(route); + } + if (message != nullptr) { + *message = "ok"; + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_topology.h b/src/comm/ccu/tilexr_ccu_topology.h new file mode 100644 index 00000000..cbb68d96 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_topology.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_TOPOLOGY_H +#define TILEXR_CCU_TOPOLOGY_H + +#include "ccu/tilexr_ccu_hccp_types.h" +#include "tilexr_types.h" + +#include +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuPeerEidRoute { + uint32_t peerDevicePhyId = 0; + std::array localEid {}; + std::string localPort; +}; + +int TileXRCcuResolvePeerEidRoutes( + const std::string& rootInfoPath, + uint32_t localDevicePhyId, + const std::vector& peerDevicePhyIds, + std::vector* routes, + std::string* message); + +} // namespace TileXR + +#endif // TILEXR_CCU_TOPOLOGY_H diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 041e24cd..288b04c1 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -86,6 +86,7 @@ constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; constexpr const char* kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"; constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; +constexpr const char* kSyncXnPingPeerXorEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; @@ -635,7 +636,9 @@ int InitAllToAllMeshState(int rank, int rankSize, AllToAllState* state) state->chunkBytes = AllToAllBytesFromEnv(); state->rankSize = rankSize; state->bytes = static_cast(rankSize) * state->chunkBytes; - if (state->chunkBytes != 2U * 1024U * 1024U || + const bool supportedChunkBytes = + state->chunkBytes == 128U * 1024U || state->chunkBytes == 2U * 1024U * 1024U; + if (!supportedChunkBytes || state->bytes / state->chunkBytes != static_cast(rankSize) || AllToAllMemSlicePerLoopFromEnv() != 8) { state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; @@ -2101,10 +2104,10 @@ int RunAllToAllMeshLongMissionSmokeForRank( alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); - options.syncResourceCount = 9U; + options.syncResourceCount = 3U; options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { - options.syncInstructionCount = 1823U; + options.syncInstructionCount = 131U; } if (options.gsaStartId == 0) { options.gsaStartId = 1; @@ -2115,7 +2118,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( << " chunkBytes=" << alltoall.chunkBytes << " bytes=" << alltoall.bytes << " loopCount=" << loopCount - << " resourceCount=9" + << " resourceCount=3" << " mesh=1" << " longMission=1" << std::endl; @@ -2143,7 +2146,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( if (prepareRet != TileXR::TILEXR_SUCCESS) { finalRet = 6; } else if (attempt.submitTasks.size() != 1U || attempt.submitTasks.front().argSize != - TILEXR_DIRECT_CCU_SQE_ARGS_LEN || attempt.plan.syncResources.size() != 9U) { + TILEXR_DIRECT_CCU_SQE_ARGS_LEN || attempt.plan.syncResources.size() != 3U) { std::cerr << "tilexr_ccu_alltoall invalidMeshPreparedTask" << " rank=" << rank << " taskCount=" << attempt.submitTasks.size() @@ -2166,8 +2169,6 @@ int RunAllToAllMeshLongMissionSmokeForRank( const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { const int resetRet = ResetAllToAllMeshStateForLoop(rank, loopIndex, &alltoall); - const uint64_t localLoopMarker = BuildAllToAllLoopMarker(rank, loopIndex); - attempt.submitTasks.front().args[0] = localLoopMarker; const bool ready = WaitForCollectiveSubmitReadiness( rank, rankSize, @@ -2200,28 +2201,6 @@ int RunAllToAllMeshLongMissionSmokeForRank( } } - uint32_t peerOrdinal = 0; - for (int peerRank = 0; peerRank < rankSize; ++peerRank) { - if (peerRank == rank) { - continue; - } - const uint32_t routeIndex = peerOrdinal * 3U; - const auto& markerResource = attempt.plan.syncResources[routeIndex]; - if (finalRet == 0 && ReadAndValidatePeerLoopMarker( - context, - attempt.submitTasks.front().dieId, - markerResource.localXn, - rank, - peerRank, - loopIndex, - routeIndex, - markerResource.channelId, - markerResource.localWaitCke, - BuildAllToAllLoopMarker(peerRank, loopIndex)) != TileXR::TILEXR_SUCCESS) { - finalRet = 15; - } - ++peerOrdinal; - } if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { finalRet = 14; } @@ -2237,7 +2216,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( << " rank=" << rank << " loopIndex=" << loopIndex << " ret=" << finalRet - << " resourceCount=9" + << " resourceCount=3" << " selfCopyCompletionCke=" << attempt.plan.syncResources[0].localWaitCke << std::endl; PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); @@ -2246,7 +2225,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( attempt.submitTasks.front().dieId, options, "tilexr_ccu_alltoall", - 9U); + 3U); break; } std::cout << "tilexr_ccu_alltoall stableResources=1" @@ -2412,7 +2391,7 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, const int markerRet = ReadAndValidatePeerLoopMarker( context, attempt.submitTasks.front().dieId, - attempt.plan.syncResources[0].localXn, + attempt.plan.syncResources[0].remoteXn, rank, peer, loopIndex, @@ -2473,7 +2452,7 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (AllToAllLongMissionEnabled()) { + if (AllToAllMeshSmokeEnabled() || AllToAllLongMissionEnabled()) { return RunAllToAllLongMissionSmokeForRank(context, rank, rankSize, device); } if (rankSize != 2) { @@ -2526,23 +2505,30 @@ int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (rankSize != 2) { + if (rankSize != 2 && rankSize != 4) { std::cout << "tilexr_ccu_sync_xn_ping skipped rankSize=" << rankSize - << " reason=\"direct CCU SyncXn ping requires two ranks\"" << std::endl; + << " reason=\"direct CCU SyncXn ping requires two or four ranks\"" << std::endl; return 0; } - const int peer = 1 - rank; + const int peerXor = EnvInt(kSyncXnPingPeerXorEnv, 1); + if (peerXor < 1 || peerXor >= rankSize) { + std::cerr << "tilexr_ccu_sync_xn_ping invalid peerXor=" << peerXor + << " rankSize=" << rankSize << std::endl; + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + const int peer = rank ^ peerXor; AllToAllState routeState; routeState.initRet = InitAllToAllState(rank, peer, &routeState); TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); options.syncResourceCount = 1; options.sqeArgCount = 0; - options.syncInstructionCount = 3; + options.syncInstructionCount = 2; std::cout << "tilexr_ccu_sync_xn_ping config" << " rank=" << rank << " peer=" << peer + << " peerXor=" << peerXor << std::endl; PrintConfig(options, rankSize); @@ -2614,6 +2600,12 @@ int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank if (!attempt.submitTasks.empty()) { PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_sync_xn_ping"); } + PrintCcuResourceState( + context, + attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_sync_xn_ping", + 1U); finalRet = 8; } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 8f7a9a98..806594ec 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -37,6 +37,7 @@ endpoint_fields=( ) resource_window_token_fields=( + EID EID_INDEX TOKEN_ID RAW_TOKEN_ID @@ -188,11 +189,17 @@ apply_sync_xn_ping_defaults() export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-2}" export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE:-hcomm_die}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION:-hcomm}" + export TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}" + export TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}" export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" @@ -216,18 +223,28 @@ apply_alltoall_defaults() fi export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" - export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + if alltoall_mesh_mode_enabled; then + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-131072}" + else + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + fi export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" export TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" if alltoall_mesh_mode_enabled; then + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE:-hcomm_die}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION:-hcomm}" export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" - export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-9}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-1823}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-131}" + export TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}" + export TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}" export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-16}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}" export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-16}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}" export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-16}" export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX:-3}" elif [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then @@ -409,22 +426,24 @@ cann_lib_dir="${cann_root}/lib64" driver_lib_dir="${ASCEND_DRIVER_PATH}/lib64/driver" probe_bin="${work_dir}/ccu_tilexr_direct_smoke_probe" -c++ -std=c++14 \ - -DTILEXR_CCU_TESTING=1 \ - -I "${repo_root}/src/include" \ - -I "${repo_root}/src/comm" \ - -I "${cann_root}/pkg_inc" \ - -I "${cann_root}/pkg_inc/runtime" \ - -I "${cann_root}/include" \ - "${repo_root}/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp" \ - -L "${tile_comm_dir}" \ - -L "${cann_lib_dir}" \ - -L "${driver_lib_dir}" \ - -Wl,-rpath-link,"${tile_comm_dir}" \ - -Wl,-rpath-link,"${cann_lib_dir}" \ - -Wl,-rpath-link,"${driver_lib_dir}" \ - -ltile-comm -lascendcl -lruntime -ldl -pthread \ - -o "${probe_bin}" +if [ "${TILEXR_CCU_SMOKE_REUSE_PROBE:-0}" != "1" ] || [ ! -x "${probe_bin}" ]; then + c++ -std=c++14 \ + -DTILEXR_CCU_TESTING=1 \ + -I "${repo_root}/src/include" \ + -I "${repo_root}/src/comm" \ + -I "${cann_root}/pkg_inc" \ + -I "${cann_root}/pkg_inc/runtime" \ + -I "${cann_root}/include" \ + "${repo_root}/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp" \ + -L "${tile_comm_dir}" \ + -L "${cann_lib_dir}" \ + -L "${driver_lib_dir}" \ + -Wl,-rpath-link,"${tile_comm_dir}" \ + -Wl,-rpath-link,"${cann_lib_dir}" \ + -Wl,-rpath-link,"${driver_lib_dir}" \ + -ltile-comm -lascendcl -lruntime -ldl -pthread \ + -o "${probe_bin}" +fi if command -v npu-smi >/dev/null 2>&1; then npu_smi_rc=0 @@ -582,6 +601,9 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION}") fi @@ -930,18 +952,12 @@ fi if alltoall_mode_enabled; then loop_count="$(parse_int "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" 1)" expected_results=$((rank_size * loop_count)) - expected_marker_matches=$((rank_size * (rank_size - 1) * loop_count)) actual_results="$(grep -h -c "tilexr_ccu_alltoall result passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" - actual_marker_matches="$(grep -h -c "tilexr_ccu_alltoall peerLoopMarker .*matched=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" - echo "tilexr_ccu_direct_smoke_runner alltoallCounts expectedResults=${expected_results} actualResults=${actual_results} expectedMarkerMatches=${expected_marker_matches} actualMarkerMatches=${actual_marker_matches}" + echo "tilexr_ccu_direct_smoke_runner alltoallCounts expectedResults=${expected_results} actualResults=${actual_results}" if [ "${actual_results}" -ne "${expected_results}" ]; then echo "ERROR: direct CCU alltoall result count mismatch expected=${expected_results} actual=${actual_results}" >&2 exit 9 fi - if alltoall_mesh_mode_enabled && [ "${actual_marker_matches}" -ne "${expected_marker_matches}" ]; then - echo "ERROR: direct CCU alltoall marker count mismatch expected=${expected_marker_matches} actual=${actual_marker_matches}" >&2 - exit 9 - fi elif signal_wait_mode_enabled; then for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index f1704841..7aeac4fe 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -120,6 +120,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.preSyncMarkerArgIndex = 0; spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; + spec.preSyncMarkerChannelId = 0x14; spec.preSyncChannelId = 0x13; spec.preSyncTokenChannelId = 0x13; spec.copyCompletionCke = 0x301; @@ -178,7 +179,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): Slot(program[1], 0) != kSyncXnHeader || Slot(program[1], 1) != spec.preSyncRemoteMarkerXn || Slot(program[1], 2) != spec.preSyncLocalMarkerXn || - Slot(program[1], 4) != spec.preSyncChannelId || + Slot(program[1], 4) != spec.preSyncMarkerChannelId || Slot(program[1], 5) != spec.preSyncRemoteNotifyCke || Slot(program[1], 6) != markerMask || Slot(program[2], 0) != kLoadImdToXnHeader || @@ -310,6 +311,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): spec.preSyncMarkerArgIndex = 0; spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; + spec.preSyncMarkerChannelId = spec.channelId; spec.copyCompletionCke = 0x301; spec.preSyncRemoteAddrXn = 0x211; spec.preSyncRemoteTokenXn = 0x212; @@ -560,31 +562,29 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) route.remoteRecvAddr = 0x30000000ULL + static_cast(peerRank) * 0x1000000ULL; route.remoteRecvToken = TileXRCcuPackMemoryToken(10 + peerRank, 20 + peerRank, true); route.bytes = 2ULL * 1024ULL * 1024ULL; - route.localGsa = static_cast(0x100 + ordinal * 2); - route.remoteGsa = static_cast(0x101 + ordinal * 2); - route.localXn = static_cast(0x200 + ordinal * 8); - route.remoteXn = static_cast(0x201 + ordinal * 8); - route.lengthXn = static_cast(0x202 + ordinal * 8); - route.preSyncLocalAddrXn = static_cast(0x203 + ordinal * 8); - route.preSyncLocalTokenXn = static_cast(0x204 + ordinal * 8); - route.preSyncLocalMarkerXn = static_cast(0x205 + ordinal * 8); - route.preSyncRemoteMarkerXn = static_cast(0x300 + ordinal * 3); - route.preSyncRemoteAddrXn = static_cast(0x301 + ordinal * 3); - route.preSyncRemoteTokenXn = static_cast(0x302 + ordinal * 3); - route.preSyncMarkerArgIndex = 0; - route.preSyncMarkerEnabled = true; - route.preSyncChannelId = static_cast(0x10 + ordinal * 3); - route.preSyncTokenChannelId = static_cast(0x11 + ordinal * 3); - route.copyChannelId = static_cast(0x12 + ordinal * 3); + route.localGsa = 0x100; + route.remoteGsa = 0x101; + route.localXn = 0x200; + route.remoteXn = 0x301; + route.lengthXn = 0x202; + route.preSyncLocalAddrXn = 0x200; + route.preSyncLocalTokenXn = 0x201; + route.preSyncRemoteAddrXn = 0x300; + route.preSyncRemoteTokenXn = 0x301; + route.preSyncMarkerEnabled = false; + route.preSyncChannelId = static_cast(0x10 + ordinal); + route.preSyncTokenChannelId = route.preSyncChannelId; + route.copyChannelId = route.preSyncChannelId; route.postSyncChannelId = route.preSyncChannelId; - route.copyCompletionCke = static_cast(0x400 + ordinal * 4); - route.preSyncLocalWaitCke = static_cast(0x401 + ordinal * 4); - route.preSyncRemoteNotifyCke = static_cast(0x500 + ordinal * 3); - route.preSyncRemoteTokenNotifyCke = static_cast(0x501 + ordinal * 3); - route.postSyncLocalWaitCke = static_cast(0x402 + ordinal * 4); - route.postSyncRemoteNotifyCke = static_cast(0x502 + ordinal * 3); - route.sourceCke = static_cast(0x403 + ordinal * 4); - route.ckeMask = 1; + route.copyCompletionCke = 0x491; + route.preSyncLocalWaitCke = static_cast(0x401 + ordinal); + route.preSyncTokenLocalWaitCke = route.preSyncLocalWaitCke; + route.preSyncRemoteNotifyCke = static_cast(0x500 + ordinal); + route.preSyncRemoteTokenNotifyCke = route.preSyncRemoteNotifyCke; + route.postSyncLocalWaitCke = route.preSyncLocalWaitCke; + route.postSyncRemoteNotifyCke = route.preSyncRemoteNotifyCke; + route.sourceCke = 0x490; + route.ckeMask = 0x8; return peer; } @@ -603,8 +603,9 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) spec.selfSourceXn = 0x280; spec.selfDestinationXn = 0x281; spec.selfLengthXn = 0x282; - spec.selfChannelId = 0x30; + spec.selfChannelId = 0; spec.selfCompletionCke = 0x480; + spec.remoteCompletionCke = 0x491; spec.peers = {Peer(2, 3, 2), Peer(2, 0, 0), Peer(2, 1, 1)}; std::vector program; @@ -614,41 +615,93 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) std::cerr << report.message << "\n"; return 1; } - if (report.peerCount != 3 || report.syncResourceCount != 9 || + if (report.peerCount != 3 || report.syncResourceCount != 3 || report.remoteBlockCount != 192 || report.selfBlockCount != 64 || - report.preSyncInstructionCount != 21 || report.copyInstructionCount != 1792 || - report.postSyncInstructionCount != 9 || report.finishInstructionCount != 1 || - report.totalInstructionCount != 1823 || program.size() != 1823) { + report.preSyncInstructionCount != 12 || report.copyInstructionCount != 1792 || + report.postSyncInstructionCount != 6 || report.finishInstructionCount != 1 || + report.totalInstructionCount != 1811 || program.size() != 1811) { std::cerr << "unexpected mesh counts total=" << program.size() << "\n"; return 2; } - // Three peers publish marker/address/token (6 instructions each) before any wait. - for (uint32_t i = 0; i < 18; ++i) { - if (Slot(program[i], 0) == 0x0802U || Slot(program[i], 0) == 0x0804U) { - std::cerr << "wait appeared before all peer posts\n"; - return 3; + // Match HCCL: load both values, initialize source CKE, then post output/token per channel. + if (Slot(program[0], 1) != 0x200U || Slot(program[1], 1) != 0x201U || + Slot(program[2], 0) != 0x0802U || Slot(program[2], 2) != 0x490U || + Slot(program[2], 3) != 0xffffU) { + std::cerr << "unexpected HCCL-style presync prelude\n"; + return 3; + } + for (uint32_t ordinal = 0; ordinal < 3; ++ordinal) { + const uint32_t output = 3 + ordinal * 2; + const uint32_t token = output + 1; + if (Slot(program[output], 0) != 0x100dU || Slot(program[output], 6) != 0x2U || + Slot(program[token], 0) != 0x100dU || Slot(program[token], 6) != 0x4U || + Slot(program[output], 4) != Slot(program[token], 4)) { + std::cerr << "presync output/token are not paired by channel\n"; + return 4; } } - for (uint32_t i = 18; i < 21; ++i) { - if (Slot(program[i], 0) != 0x0802U || Slot(program[i], 5) != 0x7U) { + for (uint32_t i = 9; i < 12; ++i) { + if (Slot(program[i], 0) != 0x0802U || Slot(program[i], 5) != 0x6U) { std::cerr << "missing presync wait mask\n"; - return 4; + return 5; } } // Sorted peer 0 copy: send[target=0] -> recv_peer0[source=2]. - if (Immediate(program[21]) != spec.localSendAddr || - Immediate(program[23]) != 0x30000000ULL + 2ULL * spec.chunkBytes || - Slot(program[26], 0) != 0x1009U) { + if (Immediate(program[12]) != spec.localSendAddr || + Immediate(program[14]) != 0x30000000ULL + 2ULL * spec.chunkBytes || + Slot(program[17], 0) != 0x1009U) { std::cerr << "unexpected first remote copy offsets\n"; - return 5; + return 6; } - const uint32_t selfStart = 21 + 3 * 64 * 7; + const uint32_t selfStart = 30; const uint64_t selfOffset = 2ULL * spec.chunkBytes; if (Immediate(program[selfStart]) != spec.localSendAddr + selfOffset || Immediate(program[selfStart + 2]) != spec.localRecvAddr + selfOffset || - Slot(program[selfStart + 5], 0) != 0x100aU) { + Slot(program[selfStart + 5], 0) != 0x1000U || + Slot(program[selfStart + 5], 1) != 0U || + Slot(program[selfStart + 5], 5) != 0U || + Slot(program[selfStart + 7], 0) != 0x1002U || + Slot(program[selfStart + 7], 3) != 0U || + Slot(program[selfStart + 7], 5) != 0U) { std::cerr << "unexpected CCU self copy offsets\n"; - return 6; + return 7; + } + auto corrupted = program; + corrupted[3].words[1] ^= 1ULL; + if (TileXRCcuValidateAllToAllMeshProgramBindings(spec, corrupted, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("output SyncXn") == std::string::npos) { + std::cerr << "corrupted output channel accepted: " << report.message << "\n"; + return 8; + } + auto sharedRemoteIds = spec; + for (uint32_t ordinal = 1; ordinal < sharedRemoteIds.peers.size(); ++ordinal) { + sharedRemoteIds.peers[ordinal].route.remoteXn = sharedRemoteIds.peers[0].route.remoteXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteAddrXn = + sharedRemoteIds.peers[0].route.preSyncRemoteAddrXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteTokenXn = + sharedRemoteIds.peers[0].route.preSyncRemoteTokenXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteNotifyCke = + sharedRemoteIds.peers[0].route.preSyncRemoteNotifyCke; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteTokenNotifyCke = + sharedRemoteIds.peers[0].route.preSyncRemoteTokenNotifyCke; + sharedRemoteIds.peers[ordinal].route.postSyncRemoteNotifyCke = + sharedRemoteIds.peers[0].route.postSyncRemoteNotifyCke; + } + if (TileXRCcuBuildAllToAllMeshProgram(sharedRemoteIds, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "per-peer remote resource IDs rejected: " << report.message << "\n"; + return 7; + } + auto overlappingCke = spec; + overlappingCke.remoteCompletionCke = overlappingCke.peers[0].route.sourceCke; + for (auto& peer : overlappingCke.peers) { + peer.route.copyCompletionCke = overlappingCke.remoteCompletionCke; + } + if (TileXRCcuBuildAllToAllMeshProgram(overlappingCke, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("overlaps source CKE") == std::string::npos) { + std::cerr << "overlapping source/completion CKE accepted: " << report.message << "\n"; + return 8; } auto duplicate = spec; duplicate.peers[1].route.copyChannelId = duplicate.peers[0].route.copyChannelId; @@ -656,7 +709,7 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) TILEXR_ERROR_PARA_CHECK_FAIL || report.message.find("duplicate") == std::string::npos) { std::cerr << "duplicate peer resource accepted: " << report.message << "\n"; - return 7; + return 8; } for (uint32_t localRank = 0; localRank < 4; ++localRank) { auto rankSpec = spec; @@ -670,10 +723,10 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) } if (TileXRCcuBuildAllToAllMeshProgram(rankSpec, &program, &report) != TILEXR_SUCCESS) { std::cerr << "rank " << localRank << " rejected: " << report.message << "\n"; - return 8; + return 9; } const uint64_t rankOffset = static_cast(localRank) * rankSpec.chunkBytes; - if (program.size() != 1823 || + if (program.size() != 1811 || Immediate(program[selfStart]) != rankSpec.localSendAddr + rankOffset || Immediate(program[selfStart + 2]) != rankSpec.localRecvAddr + rankOffset) { std::cerr << "rank " << localRank << " self offset mismatch\n"; diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index c300fdec..7c55de78 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -255,8 +255,10 @@ def test_four_rank_mesh_gathers_imports_and_maps_three_routes_per_peer(self): self.assertEqual(1, mesh_body.count("session.AllGather(")) self.assertIn("endpoint.rank != peerRank", mesh_body) self.assertIn("session.ImportRemoteMemoryBuffer", mesh_body) - self.assertIn("for (uint32_t routeWithinPeer = 0; routeWithinPeer < 3U", mesh_body) - self.assertIn("peerOrdinal * 3U + routeWithinPeer", mesh_body) + self.assertNotIn("routeWithinPeer", mesh_body) + self.assertNotIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute", mesh_body) + self.assertNotIn("peer.imported.targetSegVa", mesh_body) + self.assertIn("ClearDirectCcuRemoteRouteMemoryOverride", mesh_body) self.assertIn("TileXRCcuRunDirectAllToAllMeshInstallAttempt", mesh_body) exchange = planner[ diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index f19ded29..f30e01bf 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1783,6 +1783,7 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): self.assertIn("preSyncTokenChannelId = preResource.channelId", source) self.assertIn("preSyncLocalMarkerXn = copyResource.localXn", source) self.assertIn("preSyncRemoteMarkerXn = copyResource.remoteXn", source) + self.assertIn("preSyncMarkerChannelId = alltoallSpec.preSyncChannelId", source) self.assertIn("preSyncMarkerArgIndex = 0", source) self.assertIn("preSyncMarkerEnabled = true", source) self.assertNotIn("preSyncTokenChannelId = postResource.channelId", source) @@ -1830,7 +1831,7 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): with self.subTest(needle=needle): self.assertNotIn(needle, combined) - def test_direct_four_rank_mesh_builds_one_nine_route_launch_package(self): + def test_direct_four_rank_mesh_builds_one_three_channel_launch_package(self): code = textwrap.dedent( r''' #include "ccu/tilexr_ccu_direct_orchestrator.h" @@ -1890,18 +1891,18 @@ def test_direct_four_rank_mesh_builds_one_nine_route_launch_package(self): const int ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt( options, mesh, &attempt, &report); (void)ret; - if (!report.pipelineBuilt || attempt.plan.syncResources.size() != 9 || + if (!report.pipelineBuilt || attempt.plan.syncResources.size() != 3 || attempt.plan.taskWindows.size() != 1 || attempt.package.tasks.size() != 1 || - attempt.package.program.sync.size() != 1823 || - attempt.plan.taskWindows[0].instCnt != 1823 || - attempt.plan.kernelLocalGsa.num != 2 || attempt.allocation.sourceCke.num != 1 || + attempt.package.program.sync.size() != 1811 || + attempt.plan.taskWindows[0].instCnt != 1811 || + attempt.plan.kernelLocalGsa.num != 2 || attempt.allocation.sourceCke.num != 2 || attempt.plan.barrierMode != TileXRCcuBarrierMode::SyncCke) { std::cerr << "unexpected mesh package: " << report.message << " resources=" << attempt.plan.syncResources.size() << " instructions=" << attempt.package.program.sync.size() << "\n"; return 1; } - basic.caps.cap0 = (7U << 24) | (11U << 16) | 1799U; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 1599U; TileXRCcuDirectInstallAttempt smallAttempt; TileXRCcuDirectInstallReport smallReport; if (TileXRCcuRunDirectAllToAllMeshInstallAttempt( @@ -1962,7 +1963,7 @@ def test_collective_planner_has_private_alltoall_prepare_path(self): self.assertIn("tilexr-comm-direct-ccu-alltoall", source) self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) - def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): + def test_direct_sync_xn_ping_uses_one_mission_route_and_full_4p_transport_resources(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") planner_header = PLANNER_HEADER.read_text(encoding="utf-8") @@ -1975,13 +1976,23 @@ def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): self.assertIn("BuildDirectSyncXnPingLaunchPackage", source) self.assertIn("TileXRCcuEncodeSyncXn", source) self.assertIn("defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank)", source) - self.assertIn("defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank)", source) - self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U", source) + self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U", source) + self.assertIn("SyncXnPingAllocationInstructionCount(options.syncResourceCount)", source) + self.assertIn("syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn", source) + ping_body = source[ + source.index("int BuildDirectSyncXnPingLaunchPackage"): + source.index("void FillReportFromAttempt") + ] + self.assertNotIn("TileXRCcuEncodeSyncCke", ping_body) + self.assertNotIn("TileXRCcuEncodeSetCke", ping_body) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner_header) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK", planner) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK", planner) self.assertIn("RegisterCcuResourceRmaBuffer", planner) + self.assertIn("next.syncResourceCount = rankSize == 4 ? 3U : 1U", planner) + self.assertIn("attempt->plan.syncResources.empty()", source) + self.assertIn("syncXnPing != nullptr ? options.syncResourceCount", source) if __name__ == "__main__": diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 12aeaae1..ae8ad68f 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -703,6 +703,8 @@ def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex)", body) self.assertIn("adapter.ReadXnRange", source) self.assertIn("peerLoopMarker", source) + self.assertIn("attempt.plan.syncResources[0].remoteXn", body) + self.assertNotIn("attempt.plan.syncResources[0].localXn,", body) self.assertIn("loopIndex=", body) self.assertLess( body.index("PrepareDirectCcuAllToAll2RankInstallAttempt"), @@ -719,7 +721,18 @@ def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(s self.assertIn("InitAllToAllMeshState", source) self.assertIn("ResetAllToAllMeshStateForLoop", source) self.assertIn("BuildAllToAllMeshByte", source) + mesh_body = source[ + source.index("int RunAllToAllMeshLongMissionSmokeForRank"): + source.index("int RunAllToAllLongMissionSmokeForRank") + ] + self.assertNotIn("const uint32_t routeIndex = peerOrdinal * 3U", mesh_body) + self.assertNotIn("markerResource.remoteXn", mesh_body) self.assertIn("RunAllToAllMeshLongMissionSmokeForRank", source) + mesh_dispatch = source[source.index("int RunAllToAllSmokeForRank"):] + self.assertLess( + mesh_dispatch.index("AllToAllMeshSmokeEnabled()"), + mesh_dispatch.index("rankSize != 2"), + ) body = source[ source.index("int RunAllToAllMeshLongMissionSmokeForRank"): source.index("int RunAllToAllLongMissionSmokeForRank") @@ -731,15 +744,14 @@ def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(s self.assertIn(loop, body) self.assertLess(body.index("PrepareDirectCcuAllToAllMeshInstallAttempt"), body.index(loop)) self.assertLess(body.index("aclrtCreateStream"), body.index(loop)) - self.assertIn("attempt.submitTasks.front().args[0] = localLoopMarker", body) + self.assertNotIn("attempt.submitTasks.front().args[0] = localLoopMarker", body) self.assertIn("WaitForCollectiveSubmitReadiness", body) self.assertIn("WaitForCollectiveSubmitDone", body) - self.assertIn("for (int peerRank = 0; peerRank < rankSize; ++peerRank)", body) - self.assertIn("peerOrdinal * 3U", body) - self.assertIn("ReadAndValidatePeerLoopMarker", body) + self.assertNotIn("peerOrdinal * 3U", body) + self.assertNotIn("ReadAndValidatePeerLoopMarker", body) self.assertIn("CheckAllToAllState(&alltoall)", body) self.assertIn("PrintCcuResourceState", body) - self.assertIn("resourceCount=9", body) + self.assertIn("resourceCount=3", body) pattern = source[ source.index("uint8_t BuildAllToAllMeshByte"): @@ -757,6 +769,7 @@ def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): source = PROBE_SOURCE.read_text(encoding="utf-8") self.assertIn('kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"', source) + self.assertIn('kSyncXnPingPeerXorEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR"', source) self.assertIn("SyncXnPingSmokeEnabled", source) self.assertIn("RunSyncXnPingSmokeForRank", source) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", source) @@ -766,10 +779,41 @@ def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): ] self.assertIn("AllToAllState routeState", sync_ping_body) self.assertIn("InitAllToAllState(rank, peer, &routeState)", sync_ping_body) + self.assertIn("const int peer = rank ^ peerXor", sync_ping_body) + self.assertIn("peerXor < 1 || peerXor >= rankSize", sync_ping_body) + self.assertIn("options.syncInstructionCount = 2", sync_ping_body) + planner = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp").read_text( + encoding="utf-8" + ) + self.assertIn("selectedDiagnosticPeer", planner) + self.assertIn("override.syncRouteIndex == 0U", planner) + self.assertIn("peerRanks.push_back(selectedDiagnosticPeer)", planner) + self.assertIn("selectedDiagnosticPeer >= 0 ?", planner) + runtime = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp").read_text( + encoding="utf-8" + ) + peer_route_body = runtime[ + runtime.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes("): + runtime.index("int TileXRCcuDirectRuntime::QueryTpHandleForPeer(", + runtime.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(")) + ] + self.assertIn("offer.qpKey = state.qpInfo.key", peer_route_body) + self.assertIn("importInfo.in.key = peerOffer.qpKey", peer_route_body) + self.assertNotIn("std::copy(offer.eid.begin(), offer.eid.end(), offer.qpKey.value)", runtime) + self.assertIn( + "offer.eid = state.resourceWindow.eid", + runtime, + ) + self.assertNotIn( + "std::copy(peerOffer.eid.begin(), peerOffer.eid.end(), importInfo.in.key.value)", + runtime, + ) self.assertIn("tilexr_ccu_sync_xn_ping prepare", source) self.assertIn("tilexr_ccu_sync_xn_ping submit", source) self.assertIn("tilexr_ccu_sync_xn_ping timing", source) self.assertIn("aclrtSynchronizeStreamWithTimeout", source) + self.assertIn('PrintCcuResourceState(\n context,', sync_ping_body) + self.assertIn('"tilexr_ccu_sync_xn_ping",\n 1U', sync_ping_body) def test_alltoall_timeout_prints_xn_and_cke_readback(self): source = PROBE_SOURCE.read_text(encoding="utf-8") diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 568380f6..99278d67 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -87,7 +87,17 @@ def test_runner_executes_four_rank_mesh_loop10_with_exact_counts(self): self.assertEqual(0, result.returncode, result.stdout + result.stderr) self.assertIn("rank=3 device=7", result.stdout) self.assertIn("expectedResults=40 actualResults=40", result.stdout) - self.assertIn("expectedMarkerMatches=120 actualMarkerMatches=120", result.stdout) + self.assertNotIn("expectedMarkerMatches", result.stdout) + + def test_runner_mesh_defaults_cover_every_rank_resource_range(self): + source = RUNNER.read_text(encoding="utf-8") + mesh_defaults = source[source.index("if alltoall_mesh_mode_enabled;"):] + mesh_defaults = mesh_defaults[:mesh_defaults.index("elif")] + + self.assertIn('TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}"', mesh_defaults) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}"', mesh_defaults) + self.assertIn('TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}"', mesh_defaults) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}"', mesh_defaults) def test_runner_rejects_mesh_device_count_mismatch_and_duplicates(self): missing = self.run_fake_mesh_runner(devices="4,5,6") @@ -102,6 +112,8 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): source = RUNNER.read_text(encoding="utf-8") self.assertIn("TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE", source) + self.assertIn("TILEXR_CCU_SMOKE_REUSE_PROBE", source) + self.assertIn('[ ! -x "${probe_bin}" ]', source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ENABLE=1", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1", source) @@ -244,13 +256,18 @@ def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): self.assertIn("sync_xn_ping_mode_enabled", source) self.assertIn("apply_sync_xn_ping_defaults", source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR}")', source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}"', source) self.assertIn('common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}")', source) self.assertIn('TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}"', source) - self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}"', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-2}"', source) + self.assertIn('TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}"', source) + self.assertIn('TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}"', source) self.assertLess(source.index("apply_sync_xn_ping_defaults"), source.index("apply_alltoall_defaults")) def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py index ec65c46f..33cf6f95 100644 --- a/tests/ccu/test_tilexr_ccu_driver_adapter.py +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -1028,7 +1028,9 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertIn("GetDieEnabled", header) self.assertIn("InstallInstructions", header) self.assertIn("InstallMsidToken", header) + self.assertIn("SetTaskKill", header) self.assertIn("CleanTaskKillState", header) + self.assertIn("TILEXR_CCU_U_OP_SET_TASKKILL, &out, report", source) self.assertIn("TILEXR_CCU_U_OP_CLEAN_TASKKILL_STATE, &out, report", source) self.assertIn("InstallPfeCtx", header) self.assertIn("InstallJettyCtx", header) @@ -1056,5 +1058,16 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertNotIn(needle, combined) + def test_jetty_install_batches_payloads_larger_than_custom_channel_array(self): + source = DRIVER_SOURCE.read_text(encoding="utf-8") + install = source[source.index("int TileXRCcuDriverAdapter::InstallJettyCtx"):] + install = install[:install.index("int TileXRCcuDriverAdapter::InstallChannelCtxV1")] + + self.assertIn("while (remaining > 0)", install) + self.assertIn("std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE)", install) + self.assertIn("offset += batch", install) + self.assertIn("inputOffset += batch", install) + + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 0efe9101..5930615a 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -4,6 +4,7 @@ # import shutil +import json import os import subprocess import tempfile @@ -17,6 +18,7 @@ BUILDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_plan_builder.cpp" DIRECT_RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.h" DIRECT_RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp" +TOPOLOGY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_topology.cpp" PAYLOAD_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_payloads.cpp" SPECS_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp" ALLOCATOR_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_resource_allocator.cpp" @@ -44,6 +46,8 @@ def compile_and_run(self, code: str, env=None, extra_sources=None, extra_link_fl if compiler is None: self.skipTest("no local C++ compiler found") extra_sources = extra_sources or [] + if DIRECT_RUNTIME_SOURCE in extra_sources and TOPOLOGY_SOURCE not in extra_sources: + extra_sources = [*extra_sources, TOPOLOGY_SOURCE] extra_link_flags = extra_link_flags or [] with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) @@ -116,6 +120,75 @@ def compile_only(self, code: str): capture_output=True, ) + def test_topology_resolver_selects_peer_specific_hccs_eids(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_topology.h" + + #include + #include + #include + + using namespace TileXR; + + int main() + { + const char* rootPath = std::getenv("TILEXR_TEST_CCU_ROOT_INFO"); + std::vector routes; + std::string message; + const int ret = TileXRCcuResolvePeerEidRoutes( + rootPath == nullptr ? "" : rootPath, + 0, + {1, 2, 3}, + &routes, + &message); + if (ret != TILEXR_SUCCESS || routes.size() != 3) { + std::cerr << "resolve failed ret=" << ret << " message=" << message << "\n"; + return 1; + } + if (routes[0].localPort != "0/8" || routes[0].localEid[5] != 0x08 || + routes[1].localPort != "0/0" || routes[1].localEid[5] != 0x00 || + routes[2].localPort != "0/7" || routes[2].localEid[5] != 0x07) { + std::cerr << "peer-specific EID mapping mismatch\n"; + return 2; + } + return 0; + } + ''') + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + topo_path = temp_path / "topology.json" + root_path = temp_path / "rootinfo.json" + topo_path.write_text(json.dumps({ + "edge_list": [ + {"local_a": 0, "local_a_ports": ["0/8"], "local_b": 1, "local_b_ports": ["0/7"]}, + {"local_a": 0, "local_a_ports": ["0/0"], "local_b": 2, "local_b_ports": ["0/0"]}, + {"local_a": 0, "local_a_ports": ["0/7"], "local_b": 3, "local_b_ports": ["0/7"]}, + ] + }), encoding="utf-8") + root_path.write_text(json.dumps({ + "topo_file_path": str(topo_path), + "rank_list": [ + {"device_id": device, "local_id": device, "level_list": [{"rank_addr_list": addresses}]} + for device, addresses in [ + (0, [ + {"addr": "000000000000030000100000df160100", "ports": ["0/0"]}, + {"addr": "000000000008030000100000df160900", "ports": ["0/8"]}, + {"addr": "000000000007030000100000df160800", "ports": ["0/7"]}, + ]), + (1, []), + (2, []), + (3, []), + ] + ] + }), encoding="utf-8") + env = os.environ.copy() + env["TILEXR_TEST_CCU_ROOT_INFO"] = str(root_path) + result = self.compile_and_run(code, env=env, extra_sources=[TOPOLOGY_SOURCE]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_builds_lower_layer_install_plan_from_tilexr_owned_specs(self): code = textwrap.dedent( r''' @@ -457,7 +530,7 @@ def test_transport_template_can_use_hcomm_ordered_pfe_partition_for_direct_ccu_e if (snapshot.pfeId != 3 || snapshot.startLocalJettyCtxId != 0 || snapshot.startJettyId != 1024 || - snapshot.pfeJettyCount != 23 || + snapshot.pfeJettyCount != 128 || snapshot.routes.size() != 2) { std::cerr << "hcomm ordered pfe partition not applied: pfeId=" << snapshot.pfeId << " startLocalJettyCtxId=" << snapshot.startLocalJettyCtxId @@ -540,6 +613,75 @@ def test_transport_template_can_use_hcomm_fe_id_pfe_partition_for_direct_ccu_exp self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_hcomm_pfe_window_keeps_base_and_maps_sparse_verified_jetty_context(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + setenv("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", "hcomm", 1); + TileXRCcuBasicInfo basic; + basic.dieId = 0; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {0, 2, 1}; + allocation.localXn = {0, 1961, 1}; + allocation.remoteXn = {0, 2361, 1}; + allocation.notifyCke = {0, 332, 1}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0xf98000000000ULL; + remote.memoryTokenId = 0x1100; + remote.remoteEid[0] = 1; + remote.tpn = 0x51; + remote.doorbellVa = 0x3fffff85080ULL; + remote.doorbellTokenId = 0x1103; + remote.sqDepth = 8; + remote.startJettyId = 1026; + remote.endpointRouteVerified = true; + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, {remote}, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 1; + } + if (snapshot.startJettyId != 1024 || snapshot.pfeJettyCount != 128 || + snapshot.routes[0].wqeBasicBlockStartId != 64) { + std::cerr << "PFE window was narrowed\n"; + return 2; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + snapshot, &plan, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 3; + } + if (plan.pfes.size() != 1 || plan.jettys.size() != 1 || + plan.jettys[0].startJettyCtxId != 2 || plan.jettys[0].ctxs.size() != 1) { + std::cerr << "sparse jetty context mapping mismatch\n"; + return 4; + } + return 0; + } + ''') + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_transport_template_uses_local_pfe_id_from_ra_eid_func_id(self): code = textwrap.dedent( r''' @@ -1650,7 +1792,7 @@ def test_overlay_verified_endpoint_route_reuses_shared_jetty_for_multi_route_sna << " routeCount=" << snapshot.routes.size() << "\n"; return 3; } - if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 3 || plan.pfes.empty()) { + if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 1 || plan.pfes.empty()) { std::cerr << "install plan shape mismatch\n"; return 4; } @@ -1868,6 +2010,125 @@ def test_transport_template_carries_explicit_channel_owner_exchange_proof(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_shared_peer_jetty_routes_reuse_one_wqe_window_per_peer(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + #include + + using namespace TileXR; + + uint16_t DecodeWqeStart(const TileXRCcuLocalJettyCtxData& ctx) + { + const uint16_t word22 = static_cast(ctx.raw[22]) | + static_cast(ctx.raw[23] << 8U); + const uint16_t word24 = static_cast(ctx.raw[24]) | + static_cast(ctx.raw[25] << 8U); + return static_cast(((word22 >> 12U) & 0xfU) | ((word24 & 0xffU) << 4U)); + } + + uint16_t DecodeChannelJetty(const TileXRCcuChannelCtxDataV1& ctx) + { + const uint16_t word18 = static_cast(ctx.raw[18]) | + static_cast(ctx.raw[19] << 8U); + const uint16_t word20 = static_cast(ctx.raw[20]) | + static_cast(ctx.raw[21] << 8U); + return static_cast(((word18 >> 12U) & 0xfU) | ((word20 & 0xfffU) << 4U)); + } + + int main() + { + setenv("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE", "hcomm_cap", 1); + TileXRCcuBasicInfo basic; + basic.dieId = 0; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {0, 2, 9}; + allocation.localXn = {0, 1961, 9}; + allocation.remoteXn = {0, 2361, 9}; + allocation.notifyCke = {0, 332, 9}; + allocation.localWaitCke = {0, 332, 9}; + allocation.remoteNotifyCke = {0, 364, 9}; + + std::vector buffers(9); + for (uint32_t route = 0; route < buffers.size(); ++route) { + const uint32_t peerOrdinal = route / 3U; + auto& buffer = buffers[route]; + buffer.remoteCcuVa = 0x90000000ULL + route * 0x1000ULL; + buffer.memoryTokenId = 0x2000U + route; + buffer.memoryTokenValue = 0x3000U + route; + buffer.remoteXnId = static_cast(2361U + route); + buffer.remoteNotifyCke = static_cast(364U + route); + buffer.peerRank = peerOrdinal + 1U; + for (uint32_t byte = 0; byte < buffer.remoteEid.size(); ++byte) { + buffer.remoteEid[byte] = static_cast(0x20U + peerOrdinal * 0x10U + byte); + } + buffer.tpn = 0x50U + peerOrdinal; + buffer.doorbellVa = 0x10000000ULL + peerOrdinal * 0x10000ULL; + buffer.doorbellTokenId = 0x4000U + peerOrdinal; + buffer.sqDepth = 8; + buffer.localDoorbellVa = 0x20000000ULL + peerOrdinal * 0x10000ULL; + buffer.localDoorbellTokenId = 0x5000U + peerOrdinal; + buffer.localSqDepth = 8; + buffer.startJettyId = static_cast(1024U + peerOrdinal); + buffer.endpointRouteVerified = true; + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, buffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template failed: " << report.message << "\n"; + return 1; + } + for (uint32_t route = 0; route < snapshot.routes.size(); ++route) { + const uint16_t expected = static_cast((route / 3U) * 32U); + if (snapshot.routes[route].wqeBasicBlockStartId != expected) { + std::cerr << "route WQE mismatch route=" << route << " observed=" + << snapshot.routes[route].wqeBasicBlockStartId << "\n"; + return 2; + } + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "plan failed: " << report.message << "\n"; + return 3; + } + if (plan.jettys.size() != 1U || plan.jettys[0].ctxs.size() != 3U || + plan.channels.size() != 9U) { + std::cerr << "unexpected plan shape\n"; + return 4; + } + for (uint32_t peer = 0; peer < 3U; ++peer) { + if (DecodeWqeStart(plan.jettys[0].ctxs[peer]) != peer * 32U) { + std::cerr << "jetty context WQE mismatch peer=" << peer << "\n"; + return 5; + } + for (uint32_t route = 0; route < 3U; ++route) { + if (DecodeChannelJetty(plan.channels[peer * 3U + route].ctx) != 1024U + peer) { + std::cerr << "channel jetty mismatch peer=" << peer << " route=" << route << "\n"; + return 6; + } + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_plan_builder_rejects_incomplete_lower_layer_inputs(self): code = textwrap.dedent( r''' @@ -2237,16 +2498,13 @@ def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): "peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset)", compact_body) self.assertIn("selectedRemoteXnOffset >= peerResources.remoteXnCount", compact_body) - self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", compact_body) self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertNotIn("SelectDirectCcuRemoteBindingOverride", compact_body) - self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn("(*remoteCcuBuffers)[routeIndex].remoteCcuVa +=", compact_body) self.assertNotIn("static_cast(peerLocalXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET + static_cast(peerLocalXnId)", compact_body) - self.assertNotIn( - "uint16_t remoteXnId = static_cast(peerResources.localXnStartId + peerLocalIndex)", - compact_body) + self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn( "channelBoundRemoteXnId = static_cast(allocation.remoteXn.startId + routeIndex)", compact_body) @@ -2254,6 +2512,14 @@ def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): "static_cast((*remoteCcuBuffers)[routeIndex].remoteXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) + def test_lower_layer_clears_the_complete_allocated_remote_xn_range(self): + source = BUILDER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("result.remoteXnStartId = allocation.remoteXn.startId", source) + self.assertIn("result.remoteXnCount = allocation.remoteXn.num", source) + self.assertIn("snapshot.remoteXnStartId", source) + self.assertIn("snapshot.remoteXnCount", source) + def test_remote_notify_cke_targets_peer_local_wait_cke(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ @@ -2286,14 +2552,19 @@ def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): self.assertIn("const size_t peerRouteCount = static_cast(rankSize - 1)", compact_body) self.assertIn("const size_t syncRouteCount = allocation.remoteXn.num", compact_body) - self.assertIn("allocation.remoteXn.num < static_cast(rankSize - 1)", compact_body) + self.assertIn("allocation.remoteXn.num < routedPeerCount", compact_body) self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize - 1)", compact_body) - self.assertIn("std::vector peerCcuBuffers = *remoteCcuBuffers", compact_body) + self.assertIn("peerCcuBuffersByRank", compact_body) + self.assertIn("peerCcuBuffer.peerRank", compact_body) + self.assertIn("invalid direct CCU peer buffer rank mapping", compact_body) + self.assertIn("incomplete direct CCU peer buffer rank mapping", compact_body) self.assertIn("remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{})", compact_body) self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) - self.assertIn("const size_t peerBufferIndex = syncIndex % peerRouteCount", compact_body) - self.assertIn("(*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]", compact_body) - self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("const size_t peerBufferIndex = syncIndex / routesPerPeer", compact_body) + self.assertIn( + "(*remoteCcuBuffers)[routeIndex] = *peerCcuBuffersByRank[static_cast(peer)]", + compact_body) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", compact_body) self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) @@ -2318,6 +2589,15 @@ def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self): self.assertIn("remote.localDoorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId", compact_body) self.assertIn("remote.localDoorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue", compact_body) + def test_peer_endpoints_keep_per_peer_resource_and_jetty_tokens(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn("offer.resourceTokenId = state.resourceWindow.tokenId", source) + self.assertIn("offer.resourceTokenValue = state.resourceWindow.tokenValue", source) + self.assertIn("offer.jettyTokenValue = state.resourceWindow.tokenValue", source) + self.assertIn("importInfo.in.ub.tokenValue = peerOffer.jettyTokenValue", source) + self.assertIn("state.route.memoryTokenValue = peerOffer.resourceTokenValue", source) + def test_direct_ccu_runtime_can_override_resource_window_token_from_rank_env(self): code = textwrap.dedent( r''' @@ -2582,10 +2862,17 @@ def test_direct_runtime_source_supports_selecting_ra_ctx_resource_window_eid(sel source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX", source) - self.assertIn("SelectRaCtxResourceWindowEidInfo", source) + self.assertIn("BuildRaCtxResourceWindowEidCandidates", source) self.assertIn("SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV", source) + self.assertIn("SelectDirectCcuCleanupDieId()", source) + self.assertIn("it->dieId == dieId", source) + self.assertIn("TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT = 19U", source) + self.assertIn("QueryTpHandleForPeer(ctxHandle, candidateEid, candidateEid", source) + self.assertIn("loopEidCandidate", source) self.assertIn("TraceRaCtxEidInfos", source) - self.assertIn("ctxAttr.ub.eidIndex = selectedEid.eidIndex", source) + self.assertIn("ctxAttr.ub.eidIndex = candidate.eidIndex", source) + self.assertEqual(2, source.count( + "qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT")) def test_direct_ccu_runtime_can_select_ra_ctx_resource_window_eid_by_env(self): code = textwrap.dedent( diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py index 1a67354f..baa5791f 100644 --- a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py @@ -18,6 +18,7 @@ ABI_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_abi_constants.h" DIRECT_RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.h" DIRECT_RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp" +TOPOLOGY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_topology.cpp" DRIVER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.h" DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" @@ -28,6 +29,8 @@ def compile_and_run(self, code: str, extra_sources=None, extra_link_flags=None, if compiler is None: self.skipTest("no local C++ compiler found") extra_sources = extra_sources or [] + if DIRECT_RUNTIME_SOURCE in extra_sources and TOPOLOGY_SOURCE not in extra_sources: + extra_sources = [*extra_sources, TOPOLOGY_SOURCE] extra_link_flags = extra_link_flags or [] with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) @@ -138,6 +141,8 @@ def test_ccu_hccp_loader_exposes_public_ra_ctx_endpoint_route_symbols(self): "TileXRCcuRaCtxQpBindFunc", "TileXRCcuRaCtxQpUnbindFunc", "TileXRCcuRaGetTpInfoListAsyncFunc", + "TileXRCcuRaGetTpAttrAsyncFunc", + "TileXRCcuRaSetTpAttrAsyncFunc", "TileXRCcuRaGetAsyncReqResultFunc", "TileXRCcuHccpQpCreateAttr", "TileXRCcuHccpQpImportInfo", @@ -156,6 +161,8 @@ def test_ccu_hccp_loader_exposes_public_ra_ctx_endpoint_route_symbols(self): "RaCtxQpBind", "RaCtxQpUnbind", "RaGetTpInfoListAsync", + "RaGetTpAttrAsync", + "RaSetTpAttrAsync", "RaGetAsyncReqResult", ]: with self.subTest(needle=needle): @@ -1259,8 +1266,50 @@ def test_direct_ccu_runtime_init_cleans_sticky_taskkill_state(self): source.index("void TileXRCcuDirectRuntime::Shutdown()") ] self.assertIn("adapter.CleanTaskKillState", init_body) - self.assertIn("TILEXR_CCU_DIRECT_DEFAULT_DIE_ID", init_body) - self.assertIn("TraceTaskKillCleanup", source) + self.assertIn("adapter.SetTaskKill", init_body) + self.assertIn("TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE", source) + self.assertLess( + init_body.index("adapter.SetTaskKill"), + init_body.index("adapter.CleanTaskKillState"), + ) + self.assertIn("cleanupRet = TILEXR_SUCCESS", init_body) + self.assertIn("RecoverTaskKillState() && cleanupRet != TILEXR_SUCCESS", init_body) + self.assertIn("SelectDirectCcuCleanupDieId", init_body) + self.assertIn("TraceTaskKillStep", source) + + def test_direct_runtime_selects_tp_sl_before_creating_peer_qp(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + create_body = source[ + source.index("int TileXRCcuDirectRuntime::CreatePeerEndpointState("): + source.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(") + ] + + self.assertIn("TILEXR_CCU_TP_HANDLE_REQUEST_NUM = 8", source) + self.assertIn("RaGetTpAttrAsync", source) + self.assertIn("RaSetTpAttrAsync", source) + self.assertLess( + create_body.index("SelectTpRouteForPeer("), + create_body.index("RaCtxQpCreate("), + ) + self.assertIn("qpAttr.ub.priority = state->mappedJettyPriority", create_body) + self.assertNotIn("qpAttr.ub.priority = 2", create_body) + + def test_peer_endpoint_route_uses_the_driver_returned_jetty_id(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + prepare_body = source[ + source.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes("): + source.index("int TileXRCcuDirectRuntime::QueryTpHandleForPeer(") + ] + + self.assertIn( + "state.route.startJettyId = static_cast(state.qpInfo.ub.id)", + prepare_body, + ) + self.assertNotIn( + "state.route.startJettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + ordinal)", + prepare_body, + ) + def test_direct_ccu_runtime_keeps_ra_custom_channel_provider_alive_for_created_adapters(self): header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") @@ -1473,7 +1522,7 @@ def test_direct_runtime_uses_optional_tilexr_endpoint_route_provider_before_env( return 2; } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0xaa || + buffers[0].remoteEid[15] != 0xaa || buffers[0].tpn != 0x10203 || buffers[0].doorbellVa != 0x1122334455667788ULL || buffers[0].doorbellTokenId != 0x3456 || From 01d358dbd839e23ba4e561290cfea5802a4988c7 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Mon, 27 Jul 2026 20:58:26 +0800 Subject: [PATCH 37/41] feat(ccu): generalize direct all-to-all mesh **Summary** Generalize direct CCU AllToAll mesh planning and microcode generation from a fixed 4-rank shape to runtime rank sizes from 2 through 64, including grouped completion CKE masks and dynamic instruction/resource sizing. Align peer route exchange, channel-strided remote XN allocation, shared verified jetty use, and HCCL topology transport selection for multi-rank execution while preserving the 2-rank long-mission path. Update the hardware smoke runner and CCU source checks for runtime rank counts, payload-derived instruction capacity, RA retry handling, and generalized mesh contracts. **Validation** git diff --check Built tile-comm on 141.61.50.31 with /home/pkg/b110/cann-9.1.0 and CMake 3.31.12. Ran 2-rank 2 MB long-mission CCU AllToAll on devices 6,7: both ranks passed with zero mismatches. Ran 4-rank 2 MB mesh CCU AllToAll on devices 0,1,2,3: all four ranks passed with zero mismatches and stable resources. --- src/comm/ccu/tilexr_ccu_alltoall_program.cpp | 127 ++++++++++----- src/comm/ccu/tilexr_ccu_alltoall_program.h | 4 +- .../ccu/tilexr_ccu_collective_planner.cpp | 81 ++++++---- .../ccu/tilexr_ccu_direct_orchestrator.cpp | 148 ++++++++++++------ src/comm/ccu/tilexr_ccu_direct_orchestrator.h | 2 +- src/comm/ccu/tilexr_ccu_direct_runtime.cpp | 133 +++++++++------- src/comm/ccu/tilexr_ccu_direct_runtime.h | 4 + .../tilexr_ccu_lower_layer_plan_builder.cpp | 5 +- .../tilexr_ccu_ra_custom_channel_provider.cpp | 21 ++- .../ccu/tilexr_ccu_resource_allocator.cpp | 19 ++- src/comm/ccu/tilexr_ccu_resource_allocator.h | 2 + src/comm/ccu/tilexr_ccu_topology.cpp | 36 ++++- src/comm/ccu/tilexr_ccu_topology.h | 1 + tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 50 ++++-- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 35 ++++- tests/ccu/test_tilexr_ccu_alltoall_program.py | 68 +++++++- tests/ccu/test_tilexr_ccu_backend_boundary.py | 7 +- .../test_tilexr_ccu_direct_orchestrator.py | 95 +++++++++-- .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 8 +- .../test_tilexr_ccu_direct_smoke_runner.py | 24 ++- ...est_tilexr_ccu_lower_layer_plan_builder.py | 43 ++++- ...est_tilexr_ccu_ra_custom_channel_loader.py | 21 +++ ...t_tilexr_ccu_ra_custom_channel_provider.py | 62 ++++++++ .../ccu/test_tilexr_ccu_resource_allocator.py | 17 +- 24 files changed, 767 insertions(+), 246 deletions(-) diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 7e32cf0a..e54981f0 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -19,6 +19,32 @@ constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER = 0x1000U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MS_TO_LOC_MEM_HEADER = 0x1002U; +constexpr uint32_t TILEXR_CCU_ALLTOALL_MAX_RANK_SIZE = 64U; +constexpr uint32_t TILEXR_CCU_CKE_MASK_BITS = 16U; + +uint32_t CompletionCkeCount(size_t peerCount) +{ + return static_cast((peerCount + TILEXR_CCU_CKE_MASK_BITS - 1U) / TILEXR_CCU_CKE_MASK_BITS); +} + +uint16_t CompletionMaskForGroup(size_t peerCount, uint32_t group) +{ + const size_t begin = static_cast(group) * TILEXR_CCU_CKE_MASK_BITS; + const size_t remaining = peerCount > begin ? peerCount - begin : 0U; + const uint32_t bits = static_cast(std::min(remaining, TILEXR_CCU_CKE_MASK_BITS)); + return bits == TILEXR_CCU_CKE_MASK_BITS ? 0xffffU : static_cast((1U << bits) - 1U); +} + +size_t MeshPreSyncInstructionCount(size_t peerCount) +{ + return 3U + peerCount * 3U; +} + +size_t MeshCopyInstructionCountPerBlock(size_t peerCount) +{ + return peerCount * 6U + 9U + CompletionCkeCount(peerCount); +} + constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; constexpr uint16_t TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK = 0xffffU; @@ -486,8 +512,9 @@ int ValidateMeshSpec( if (program == nullptr) { return Fail(program, report, "missing output direct CCU alltoall mesh program"); } - if (spec.rankSize != 4U || spec.localRank >= spec.rankSize || spec.peers.size() != 3U) { - return Fail(program, report, "direct CCU alltoall mesh requires four ranks and three peers"); + if (spec.rankSize < 2U || spec.rankSize > TILEXR_CCU_ALLTOALL_MAX_RANK_SIZE || + spec.localRank >= spec.rankSize || spec.peers.size() != spec.rankSize - 1U) { + return Fail(program, report, "direct CCU alltoall mesh requires 2..64 ranks and rankSize-1 peers"); } if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.chunkBytes == 0 || @@ -496,16 +523,28 @@ int ValidateMeshSpec( } if (spec.selfSourceGsa == 0 || spec.selfDestinationGsa == 0 || spec.selfSourceXn == 0 || spec.selfDestinationXn == 0 || spec.selfLengthXn == 0 || - spec.selfCompletionCke == 0 || spec.remoteCompletionCke == 0) { + spec.selfCompletionCke == 0 || + spec.remoteCompletionCkes.size() != CompletionCkeCount(spec.peers.size())) { return Fail(program, report, "missing direct CCU alltoall mesh self-copy resource"); } - bool peerRanks[4] = {}; + std::vector peerRanks(spec.rankSize, false); std::set channelIds; + std::set completionCkes; const auto& sharedRoute = spec.peers.front().route; - if (spec.remoteCompletionCke == sharedRoute.sourceCke) { - return Fail(program, report, "alltoall mesh completion CKE overlaps source CKE"); + if (spec.selfSourceXn != sharedRoute.localXn || + spec.selfDestinationXn != sharedRoute.preSyncLocalTokenXn || + spec.selfLengthXn != sharedRoute.lengthXn) { + return Fail(program, report, "alltoall mesh self copy must share source, destination, and length XNs"); + } + for (uint16_t completionCke : spec.remoteCompletionCkes) { + if (completionCke == 0 || completionCke == sharedRoute.sourceCke || + !completionCkes.insert(completionCke).second) { + return Fail(program, report, + "alltoall mesh completion CKE overlaps source CKE or duplicates another completion CKE"); + } } - for (const auto& peer : spec.peers) { + for (size_t ordinal = 0; ordinal < spec.peers.size(); ++ordinal) { + const auto& peer = spec.peers[ordinal]; if (peer.peerRank >= spec.rankSize || peer.peerRank == spec.localRank || peerRanks[peer.peerRank]) { return Fail(program, report, "invalid direct CCU alltoall mesh peer rank"); } @@ -522,7 +561,7 @@ int ValidateMeshSpec( peer.route.preSyncChannelId != peer.route.copyChannelId || peer.route.preSyncTokenChannelId != peer.route.copyChannelId || peer.route.postSyncChannelId != peer.route.copyChannelId || - peer.route.copyCompletionCke != spec.remoteCompletionCke || + peer.route.copyCompletionCke != spec.remoteCompletionCkes[ordinal / TILEXR_CCU_CKE_MASK_BITS] || peer.route.ckeMask != TILEXR_CCU_ALLTOALL_POST_SYNC_MASK || !channelIds.insert(peer.route.copyChannelId).second) { return Fail(program, report, "duplicate direct CCU alltoall mesh peer resource"); @@ -769,8 +808,11 @@ int ValidateMeshProgramBindings( }); const uint32_t blocksPerChunk = static_cast( spec.chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES); - const size_t expectedSize = 12U + static_cast(blocksPerChunk) * 28U + 6U + 1U; - if (peers.size() != 3U || blocksPerChunk == 0 || program.size() != expectedSize) { + const size_t preSyncInstructions = MeshPreSyncInstructionCount(peers.size()); + const size_t copyInstructionsPerBlock = MeshCopyInstructionCountPerBlock(peers.size()); + const size_t expectedSize = preSyncInstructions + + static_cast(blocksPerChunk) * copyInstructionsPerBlock + peers.size() * 2U + 1U; + if (peers.size() != spec.rankSize - 1U || blocksPerChunk == 0 || program.size() != expectedSize) { return FailBindingValidation(report, "unexpected mesh program shape"); } @@ -804,7 +846,7 @@ int ValidateMeshProgramBindings( PreSyncTokenMask(route))) { return FailBindingValidation(report, "token SyncXn does not match its peer route"); } - const size_t wait = 9U + ordinal; + const size_t wait = 3U + peers.size() * 2U + ordinal; if (!MatchesWait( program[wait], TILEXR_CCU_TRACE_SET_CKE_HEADER, @@ -814,7 +856,7 @@ int ValidateMeshProgramBindings( } } - size_t instruction = 12U; + size_t instruction = preSyncInstructions; for (uint32_t block = 0; block < blocksPerChunk; ++block) { for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { const auto& route = peers[ordinal].route; @@ -827,8 +869,8 @@ int ValidateMeshProgramBindings( route.localXn, route.lengthXn, route.copyChannelId, - spec.remoteCompletionCke, - static_cast(1U << ordinal))) { + route.copyCompletionCke, + static_cast(1U << (ordinal % TILEXR_CCU_CKE_MASK_BITS)))) { return FailBindingValidation(report, "remote copy does not match its peer route"); } instruction += 6U; @@ -866,14 +908,16 @@ int ValidateMeshProgramBindings( return FailBindingValidation(report, "self copy does not match its local route"); } instruction += 9U; - if (!MatchesWait( - program[instruction], - TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, - spec.remoteCompletionCke, - 0x7U)) { - return FailBindingValidation(report, "combined remote copy wait does not match the mesh completion CKE"); + for (uint32_t group = 0; group < spec.remoteCompletionCkes.size(); ++group) { + if (!MatchesWait( + program[instruction], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.remoteCompletionCkes[group], + CompletionMaskForGroup(peers.size(), group))) { + return FailBindingValidation(report, "grouped remote copy wait does not match the mesh completion CKE"); + } + ++instruction; } - ++instruction; } for (const auto& peer : peers) { const auto& route = peer.route; @@ -1003,7 +1047,10 @@ int TileXRCcuBuildAllToAllMeshProgram( }); const uint64_t bytesPerBlock = TILEXR_CCU_ALLTOALL_BLOCK_BYTES; const uint32_t blocksPerChunk = static_cast(spec.chunkBytes / bytesPerBlock); - program->reserve(12U + blocksPerChunk * 28U + 6U + 1U); + const size_t preSyncInstructions = MeshPreSyncInstructionCount(peers.size()); + const size_t copyInstructionsPerBlock = MeshCopyInstructionCountPerBlock(peers.size()); + program->reserve(preSyncInstructions + + static_cast(blocksPerChunk) * copyInstructionsPerBlock + peers.size() * 2U + 1U); ret = AppendMeshPeerPosts(peers, program, report); if (ret != TILEXR_SUCCESS) { @@ -1033,8 +1080,8 @@ int TileXRCcuBuildAllToAllMeshProgram( route, static_cast(block) * bytesPerBlock, bytesPerBlock, - spec.remoteCompletionCke, - static_cast(1U << ordinal), + peer.route.copyCompletionCke, + static_cast(1U << (ordinal % TILEXR_CCU_CKE_MASK_BITS)), program, report); if (ret != TILEXR_SUCCESS) { @@ -1050,15 +1097,17 @@ int TileXRCcuBuildAllToAllMeshProgram( if (ret != TILEXR_SUCCESS) { return ret; } - ret = AppendNotifyWait( - spec.remoteCompletionCke, - 0x7U, - "mesh Copy", - true, - program, - report); - if (ret != TILEXR_SUCCESS) { - return ret; + for (uint32_t group = 0; group < spec.remoteCompletionCkes.size(); ++group) { + ret = AppendNotifyWait( + spec.remoteCompletionCkes[group], + CompletionMaskForGroup(peers.size(), group), + "mesh Copy", + true, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } } } @@ -1093,16 +1142,16 @@ int TileXRCcuBuildAllToAllMeshProgram( } if (report != nullptr) { - report->preSyncInstructionCount = 12U; + report->preSyncInstructionCount = static_cast(preSyncInstructions); report->blockCount = blocksPerChunk; report->bytesPerBlock = static_cast(bytesPerBlock); - report->copyInstructionCount = blocksPerChunk * 28U; - report->postSyncInstructionCount = 6U; + report->copyInstructionCount = static_cast(blocksPerChunk * copyInstructionsPerBlock); + report->postSyncInstructionCount = static_cast(peers.size() * 2U); report->finishInstructionCount = 1U; report->totalInstructionCount = static_cast(program->size()); - report->peerCount = 3U; - report->syncResourceCount = 3U; - report->remoteBlockCount = 3U * blocksPerChunk; + report->peerCount = static_cast(peers.size()); + report->syncResourceCount = static_cast(peers.size()); + report->remoteBlockCount = static_cast(peers.size()) * blocksPerChunk; report->selfBlockCount = blocksPerChunk; report->message = "ok"; } diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index 93783ba2..d02ab822 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -81,7 +81,7 @@ struct TileXRCcuAllToAllMeshPeerSpec { }; struct TileXRCcuAllToAllMeshProgramSpec { - uint32_t rankSize = 4; + uint32_t rankSize = 2; uint32_t localRank = 0; uint64_t localSendAddr = 0; uint64_t localSendToken = 0; @@ -95,7 +95,7 @@ struct TileXRCcuAllToAllMeshProgramSpec { uint16_t selfLengthXn = 0; uint16_t selfChannelId = 0; uint16_t selfCompletionCke = 0; - uint16_t remoteCompletionCke = 0; + std::vector remoteCompletionCkes; std::vector peers; }; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index ad537e2e..c160012d 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -26,13 +26,23 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; -constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 7U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; +constexpr uint32_t TILEXR_CCU_CHANNEL_XN_STRIDE = 8U; + +uint32_t DirectAllToAll2RankInstructionCapacity(uint64_t bytes) +{ + if (bytes == 0 || bytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return 0; + } + const uint64_t blocks = bytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint64_t instructions = 7ULL + blocks * 7ULL; + return instructions > std::numeric_limits::max() ? + 0U : static_cast(instructions); +} uint8_t SelectDirectCcuInstallDieId() { @@ -48,18 +58,15 @@ uint8_t SelectDirectCcuInstallDieId() return static_cast(parsed); } -uint32_t SelectDirectCcuPeerLocalXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) -{ - if (peerRouteCount == 0) { - return 0; - } - return static_cast(peerLocalIndex) + - static_cast(syncIndex / peerRouteCount) * static_cast(peerRouteCount); -} - -uint32_t SelectDirectCcuChannelBoundRemoteXnOffset(size_t peerLocalIndex, uint32_t syncIndex, size_t peerRouteCount) +uint32_t SelectDirectCcuChannelBoundRemoteXnOffset( + size_t peerLocalIndex, + size_t routeWithinPeer, + size_t routesPerPeer, + bool channelStrided) { - return SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); + const size_t stride = channelStrided ? TILEXR_CCU_CHANNEL_XN_STRIDE : routesPerPeer; + return static_cast(peerLocalIndex * stride) + + static_cast(routeWithinPeer); } uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) @@ -73,12 +80,11 @@ uint16_t DirectCcuRemoteXnProofSpan(uint16_t syncRouteCount) uint16_t SelectDirectCcuChannelBoundRemoteXnId( uint16_t remoteXnStartId, size_t peerLocalIndex, - uint32_t syncIndex, - size_t peerRouteCount) + size_t routeWithinPeer) { return static_cast( static_cast(remoteXnStartId) + - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount)); + SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, routeWithinPeer, 1U, true)); } TileXRCcuSignalWaitProgramRole ToDirectSignalWaitProgramRole(TileXRCcuSignalWaitRole role) @@ -553,7 +559,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( return TILEXR_ERROR_PARA_CHECK_FAIL; } const size_t peerRouteCount = static_cast(rankSize - 1); - const size_t syncRouteCount = allocation.remoteXn.num; + const size_t syncRouteCount = allocation.channels.num; size_t routedPeerCount = peerRouteCount; int selectedDiagnosticPeer = -1; #ifdef TILEXR_CCU_TESTING @@ -570,16 +576,28 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } #endif const size_t routesPerPeer = syncRouteCount / routedPeerCount; + const bool channelStridedRemoteXn = allocation.channels.num != 0U && + static_cast(allocation.remoteXn.num) >= + static_cast(allocation.channels.num) * TILEXR_CCU_CHANNEL_XN_STRIDE; if (allocation.localXn.num == 0 || allocation.localWaitCke.num == 0 || allocation.remoteNotifyCke.num == 0 || allocation.remoteXn.num < routedPeerCount || - allocation.localWaitCke.num < allocation.remoteXn.num || - allocation.remoteNotifyCke.num < allocation.remoteXn.num || + allocation.localWaitCke.num < syncRouteCount || + allocation.remoteNotifyCke.num < syncRouteCount || allocation.channels.num == 0 || routesPerPeer == 0 || syncRouteCount % routedPeerCount != 0 || + routesPerPeer > TILEXR_CCU_CHANNEL_XN_STRIDE || remoteCcuBuffers->size() != peerRouteCount) { if (report != nullptr) { - report->message = "invalid direct CCU peer XN/CKE exchange shape"; + report->message = "invalid direct CCU peer XN/CKE exchange shape" + " peerRoutes=" + std::to_string(peerRouteCount) + + " routedPeers=" + std::to_string(routedPeerCount) + + " syncRoutes=" + std::to_string(syncRouteCount) + + " localXn=" + std::to_string(allocation.localXn.num) + + " localWaitCke=" + std::to_string(allocation.localWaitCke.num) + + " remoteNotifyCke=" + std::to_string(allocation.remoteNotifyCke.num) + + " channels=" + std::to_string(allocation.channels.num) + + " peerBuffers=" + std::to_string(remoteCcuBuffers->size()); } return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -623,7 +641,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( peerRanks.push_back(selectedDiagnosticPeer); } for (int peer = 0; peer < rankSize; ++peer) { - if (peer != rank && peer != selectedDiagnosticPeer && routedPeerCount > 1U) { + if (peer != rank && peer != selectedDiagnosticPeer && peerRanks.size() < routedPeerCount) { peerRanks.push_back(peer); } } @@ -658,7 +676,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); size_t routeIndex = 0; - for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { + for (uint32_t syncIndex = 0; syncIndex < syncRouteCount; ++syncIndex) { const size_t peerBufferIndex = syncIndex / routesPerPeer; const size_t routeWithinPeer = syncIndex % routesPerPeer; const int peer = peerRanks[peerBufferIndex]; @@ -668,7 +686,11 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( const uint32_t peerLocalResourceOffset = static_cast( peerLocalIndex * routesPerPeer + routeWithinPeer); const uint32_t peerLocalXnOffset = peerLocalResourceOffset; - const uint32_t selectedRemoteXnOffset = peerLocalResourceOffset; + const uint32_t selectedRemoteXnOffset = SelectDirectCcuChannelBoundRemoteXnOffset( + peerLocalIndex, + routeWithinPeer, + routesPerPeer, + channelStridedRemoteXn); const uint32_t peerLocalWaitCkeOffset = peerLocalResourceOffset; if (peerResources.localXnCount == 0 || peerResources.remoteXnCount == 0 || @@ -1412,7 +1434,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( next.syncResourceCount = 3; next.syncInstructionCount = std::max( next.syncInstructionCount, - TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT); + DirectAllToAll2RankInstructionCapacity(bytes)); next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; if (next.provider.empty()) { next.provider = "tilexr-comm-direct-ccu-alltoall"; @@ -1448,15 +1470,16 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( } const int rank = session.Rank(); const int rankSize = session.RankSize(); - if (rankSize != 4 || rank < 0 || rank >= rankSize || localSourceAddr == 0 || localDestinationAddr == 0 || - chunkBytes == 0 || chunkBytes > std::numeric_limits::max() / 4ULL) { + if (rankSize < 2 || rankSize > 64 || rank < 0 || rank >= rankSize || + localSourceAddr == 0 || localDestinationAddr == 0 || chunkBytes == 0 || + chunkBytes > std::numeric_limits::max() / static_cast(rankSize)) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; report->message = "invalid direct CCU alltoall mesh endpoint"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } - const uint64_t bufferBytes = chunkBytes * 4ULL; + const uint64_t bufferBytes = chunkBytes * static_cast(rankSize); const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); if (!unavailableMessage.empty()) { if (report != nullptr) { @@ -1581,8 +1604,8 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; next.lowerLayerPlanUserData = &callbackContext; next.sqeArgCount = 0; - next.syncResourceCount = 3U; - next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + next.syncResourceCount = static_cast(rankSize - 1); + next.bindingsPerSyncResource = 1U; if (next.provider.empty()) { next.provider = "tilexr-comm-direct-ccu-alltoall-mesh"; } diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index 1df19674..953482ed 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -38,14 +38,37 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; -constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = - 7U + 64U * 7U; -constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT = 3U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MAX_RANK_SIZE = 64U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_CKE_MASK_BITS = 16U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_XN_BINDINGS_PER_CHANNEL = 3U; + +uint32_t DirectAllToAllMeshPeerCount(uint32_t rankSize) +{ + return rankSize >= 2U && rankSize <= TILEXR_CCU_DIRECT_ALLTOALL_MAX_RANK_SIZE ? rankSize - 1U : 0U; +} + +uint32_t DirectAllToAllMeshCompletionCkeCount(uint32_t rankSize) +{ + const uint32_t peers = DirectAllToAllMeshPeerCount(rankSize); + return peers == 0 ? 0U : (peers + TILEXR_CCU_DIRECT_ALLTOALL_CKE_MASK_BITS - 1U) / + TILEXR_CCU_DIRECT_ALLTOALL_CKE_MASK_BITS; +} constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; +uint32_t DirectAllToAll2RankInstructionCapacity(uint64_t bytes) +{ + if (bytes == 0 || bytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return 0; + } + const uint64_t blocks = bytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint64_t instructions = 7ULL + blocks * 7ULL; + return instructions > std::numeric_limits::max() ? + 0U : static_cast(instructions); +} + uint32_t SyncXnPingAllocationInstructionCount(uint32_t syncResourceCount) { if (syncResourceCount > std::numeric_limits::max() / 2U) { @@ -54,18 +77,24 @@ uint32_t SyncXnPingAllocationInstructionCount(uint32_t syncResourceCount) return syncResourceCount * 2U; } -uint32_t DirectAllToAllMeshInstructionCount(uint64_t chunkBytes) +uint32_t DirectAllToAllMeshInstructionCount(uint32_t rankSize, uint64_t chunkBytes) { - if (chunkBytes == 0 || chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + const uint64_t peers = DirectAllToAllMeshPeerCount(rankSize); + const uint64_t completionCkes = DirectAllToAllMeshCompletionCkeCount(rankSize); + if (peers == 0 || chunkBytes == 0 || chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { return 0; } const uint64_t blocks = chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; - const uint64_t instructions = 11ULL + blocks * 28ULL + 7ULL + 1ULL; + const uint64_t preSync = 3ULL + peers * 3ULL; + const uint64_t perBlock = peers * 6ULL + 9ULL + completionCkes; + const uint64_t postSync = peers * 2ULL; + const uint64_t instructions = preSync + blocks * perBlock + postSync + 1ULL; return instructions > std::numeric_limits::max() ? 0U : static_cast(instructions); } bool DirectAllToAllMeshCapacityFits( const TileXRCcuResourceSpec& resources, + uint32_t rankSize, uint32_t instructionCount, std::string* message) { @@ -89,20 +118,23 @@ bool DirectAllToAllMeshCapacityFits( resources.ckeCount : resources.localWaitCkeCount; const uint32_t remoteNotifyCkeCount = resources.remoteNotifyCkeCount == 0 ? resources.ckeCount : resources.remoteNotifyCkeCount; - return require("mission", 1U, resources.missionCount) && + const uint32_t peers = DirectAllToAllMeshPeerCount(rankSize); + const uint32_t completionCkes = DirectAllToAllMeshCompletionCkeCount(rankSize); + const uint32_t localXns = std::max(peers, TILEXR_CCU_DIRECT_ALLTOALL_XN_BINDINGS_PER_CHANNEL); + const uint32_t remoteXns = TILEXR_CCU_DIRECT_ALLTOALL_XN_BINDINGS_PER_CHANNEL; + return peers != 0U && completionCkes != 0U && require("mission", 1U, resources.missionCount) && require("instruction", repositoryPrefix + instructionCount, resources.instructionCount) && require("GSA", TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT, resources.gsaCount) && - require("local XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.xnCount) && - require("remote XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + require("local XN", localXns, resources.xnCount) && + require("remote XN", remoteXns, resources.remoteXnCount == 0 ? - (resources.xnCount > TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT ? - resources.xnCount - TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : 0U) : + (resources.xnCount > localXns ? resources.xnCount - localXns : 0U) : resources.remoteXnCount) && - require("local CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT + 2U, + require("local CKE", peers + 1U + completionCkes, localWaitCkeCount) && - require("remote CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + require("remote CKE", peers, remoteNotifyCkeCount) && - require("channel", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.channelCount); + require("channel", peers, resources.channelCount); } void ResetReport(TileXRCcuDirectInstallReport* report) @@ -842,10 +874,12 @@ int ConfigureDirectMemoryCopyResources( int ConfigureDirectAllToAll2RankResources( const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAll2RankSpec& alltoall, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - if (attempt == nullptr || + const uint32_t instructionCapacity = DirectAllToAll2RankInstructionCapacity(alltoall.bytes); + if (attempt == nullptr || instructionCapacity == 0 || attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT || attempt->plan.taskWindows.size() != 1) { if (report != nullptr) { @@ -882,7 +916,7 @@ int ConfigureDirectAllToAll2RankResources( attempt->plan.taskWindows[0].instCnt = static_cast(std::max( attempt->plan.taskWindows[0].instCnt, - TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT)); + instructionCapacity)); return TILEXR_SUCCESS; } @@ -892,12 +926,14 @@ int ConfigureDirectAllToAllMeshResources( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - const uint32_t instructionCount = DirectAllToAllMeshInstructionCount(alltoall.chunkBytes); + const uint32_t peerCount = DirectAllToAllMeshPeerCount(alltoall.rankSize); + const uint32_t completionCkeCount = DirectAllToAllMeshCompletionCkeCount(alltoall.rankSize); + const uint32_t instructionCount = DirectAllToAllMeshInstructionCount(alltoall.rankSize, alltoall.chunkBytes); if (attempt == nullptr || instructionCount == 0 || - attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.syncResources.size() != peerCount || attempt->plan.taskWindows.size() != 1U) { if (report != nullptr) { - report->message = "alltoall mesh direct CCU plan requires three peer sync resources and one task"; + report->message = "alltoall mesh direct CCU plan requires rankSize-1 peer resources and one task"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -907,12 +943,10 @@ int ConfigureDirectAllToAllMeshResources( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - if (attempt->allocation.localXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.remoteXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.localWaitCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.remoteNotifyCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.channels.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || - attempt->allocation.sourceCke.num < 2U) { + if (attempt->allocation.localXn.num < peerCount || attempt->allocation.remoteXn.num < peerCount || + attempt->allocation.localWaitCke.num < peerCount || attempt->allocation.remoteNotifyCke.num < peerCount || + attempt->allocation.channels.num < peerCount || + attempt->allocation.sourceCke.num < 1U + completionCkeCount) { if (report != nullptr) { report->message = "alltoall mesh direct CCU allocation is missing XN/CKE/channel resources"; } @@ -1201,8 +1235,7 @@ int ValidateDirectAllToAllMeshRouteResources( const TileXRCcuProducerPlan& plan, TileXRCcuDirectInstallReport* report) { - if (mesh.peers.size() != 3U || - plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT) { + if (mesh.peers.size() != mesh.rankSize - 1U || plan.syncResources.size() != mesh.peers.size()) { if (report != nullptr) { report->message = "alltoall mesh route binding validation has an invalid shape"; } @@ -1222,7 +1255,8 @@ int ValidateDirectAllToAllMeshRouteResources( route.preSyncRemoteNotifyCke != resource.notifyCke || route.preSyncRemoteTokenNotifyCke != resource.notifyCke || route.postSyncRemoteNotifyCke != resource.notifyCke || - route.copyCompletionCke != mesh.remoteCompletionCke) { + route.copyCompletionCke != + mesh.remoteCompletionCkes[ordinal / TILEXR_CCU_DIRECT_ALLTOALL_CKE_MASK_BITS]) { if (report != nullptr) { std::ostringstream stream; stream << "alltoall mesh route binding mismatch peerRank=" << mesh.peers[ordinal].peerRank @@ -1241,7 +1275,7 @@ int BuildDirectAllToAllMeshLaunchPackage( TileXRCcuDirectInstallReport* report) { if (attempt == nullptr || - attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.syncResources.size() != DirectAllToAllMeshPeerCount(alltoall.rankSize) || attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { if (report != nullptr) { report->message = "missing direct CCU alltoall mesh producer resources"; @@ -1267,7 +1301,11 @@ int BuildDirectAllToAllMeshLaunchPackage( const uint16_t localXnStart = attempt->allocation.localXn.startId; const uint16_t remoteXnStart = attempt->allocation.remoteXn.startId; - mesh.remoteCompletionCke = static_cast(attempt->allocation.sourceCke.startId + 1U); + const uint32_t completionCkeCount = DirectAllToAllMeshCompletionCkeCount(alltoall.rankSize); + for (uint32_t group = 0; group < completionCkeCount; ++group) { + mesh.remoteCompletionCkes.push_back( + static_cast(attempt->allocation.sourceCke.startId + 1U + group)); + } for (uint32_t ordinal = 0; ordinal < peers.size(); ++ordinal) { const TileXRCcuSyncResource& resource = attempt->plan.syncResources[ordinal]; TileXRCcuAllToAllMeshPeerSpec peer; @@ -1295,7 +1333,8 @@ int BuildDirectAllToAllMeshLaunchPackage( route.preSyncTokenChannelId = resource.channelId; route.copyChannelId = resource.channelId; route.postSyncChannelId = resource.channelId; - route.copyCompletionCke = mesh.remoteCompletionCke; + route.copyCompletionCke = + mesh.remoteCompletionCkes[ordinal / TILEXR_CCU_DIRECT_ALLTOALL_CKE_MASK_BITS]; route.preSyncLocalWaitCke = resource.localWaitCke; route.preSyncRemoteNotifyCke = resource.notifyCke; route.preSyncTokenLocalWaitCke = resource.localWaitCke; @@ -1819,7 +1858,8 @@ int RunDirectInstallAttemptImpl( std::string capacityMessage; if (!DirectAllToAllMeshCapacityFits( attempt->resourceSpec, - DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes), + alltoallMesh->rankSize, + DirectAllToAllMeshInstructionCount(alltoallMesh->rankSize, alltoallMesh->chunkBytes), &capacityMessage)) { return Fail(attempt, report, capacityMessage); } @@ -1829,7 +1869,7 @@ int RunDirectInstallAttemptImpl( signalWait != nullptr || syncXnPing != nullptr; attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = - alltoallMesh != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : + alltoallMesh != nullptr ? DirectAllToAllMeshPeerCount(alltoallMesh->rankSize) : alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : syncXnPing != nullptr ? options.syncResourceCount : customProgram ? 1U : options.syncResourceCount; @@ -1837,10 +1877,12 @@ int RunDirectInstallAttemptImpl( memoryCopy != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : alltoall != nullptr ? - std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : + std::max( + options.syncInstructionCount, + DirectAllToAll2RankInstructionCapacity(alltoall->bytes)) : alltoallMesh != nullptr ? std::max(options.syncInstructionCount, - DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes)) : + DirectAllToAllMeshInstructionCount(alltoallMesh->rankSize, alltoallMesh->chunkBytes)) : signalWait != nullptr ? std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : syncXnPing != nullptr ? @@ -1848,8 +1890,16 @@ int RunDirectInstallAttemptImpl( options.syncInstructionCount, SyncXnPingAllocationInstructionCount(options.syncResourceCount)) : options.syncInstructionCount; - attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.sourceCkeCount = alltoallMesh != nullptr ? 2U : 1U; + attempt->resourceRequest.bindingsPerSyncResource = alltoallMesh != nullptr ? + 1U : options.bindingsPerSyncResource; + attempt->resourceRequest.minimumLocalXnCount = + alltoallMesh != nullptr ? + TILEXR_CCU_DIRECT_ALLTOALL_XN_BINDINGS_PER_CHANNEL : 0U; + attempt->resourceRequest.minimumRemoteXnCount = + alltoallMesh != nullptr ? + TILEXR_CCU_DIRECT_ALLTOALL_XN_BINDINGS_PER_CHANNEL : 0U; + attempt->resourceRequest.sourceCkeCount = alltoallMesh != nullptr ? + 1U + DirectAllToAllMeshCompletionCkeCount(alltoallMesh->rankSize) : 1U; attempt->resourceRequest.barrierMode = alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : @@ -1882,7 +1932,7 @@ int RunDirectInstallAttemptImpl( report->message); } } else if (alltoall != nullptr) { - ret = ConfigureDirectAllToAll2RankResources(options, attempt, report); + ret = ConfigureDirectAllToAll2RankResources(options, *alltoall, attempt, report); if (ret != TILEXR_SUCCESS) { return Fail( attempt, @@ -2075,18 +2125,22 @@ int TileXRCcuRunDirectAllToAllMeshInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - bool peerRanks[4] = {}; - bool valid = alltoall.rankSize == 4U && alltoall.localRank < alltoall.rankSize && + bool valid = alltoall.rankSize >= 2U && alltoall.rankSize <= TILEXR_CCU_DIRECT_ALLTOALL_MAX_RANK_SIZE && + alltoall.localRank < alltoall.rankSize && alltoall.localSendAddr != 0 && alltoall.localSendToken != 0 && alltoall.localRecvAddr != 0 && alltoall.localRecvToken != 0 && - DirectAllToAllMeshInstructionCount(alltoall.chunkBytes) != 0 && alltoall.peers.size() == 3U; - for (const auto& peer : alltoall.peers) { - if (peer.peerRank >= alltoall.rankSize || peer.peerRank == alltoall.localRank || - peerRanks[peer.peerRank] || peer.remoteRecvAddr == 0 || peer.remoteRecvToken == 0) { - valid = false; - break; + DirectAllToAllMeshInstructionCount(alltoall.rankSize, alltoall.chunkBytes) != 0 && + alltoall.peers.size() == alltoall.rankSize - 1U; + if (valid) { + std::vector peerRanks(alltoall.rankSize, false); + for (const auto& peer : alltoall.peers) { + if (peer.peerRank >= alltoall.rankSize || peer.peerRank == alltoall.localRank || + peerRanks[peer.peerRank] || peer.remoteRecvAddr == 0 || peer.remoteRecvToken == 0) { + valid = false; + break; + } + peerRanks[peer.peerRank] = true; } - peerRanks[peer.peerRank] = true; } if (!valid) { ResetReport(report); diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index a98b945f..e5a32cfb 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -92,7 +92,7 @@ struct TileXRCcuDirectAllToAllMeshPeerSpec { }; struct TileXRCcuDirectAllToAllMeshSpec { - uint32_t rankSize = 4; + uint32_t rankSize = 2; uint32_t localRank = 0; uint64_t localSendAddr = 0; uint64_t localSendToken = 0; diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp index 1183e2a2..be8eafb9 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -26,6 +26,10 @@ constexpr uint32_t TILEXR_CCU_DIRECT_CCUM_SQE_BYTES = 64; constexpr uint32_t TILEXR_CCU_DIRECT_SQ_EBB_WORDS = 4; constexpr uint32_t TILEXR_CCU_DIRECT_LOOP_JETTY_ID = 1024; constexpr uint32_t TILEXR_CCU_DIRECT_LOOP_JETTY_CTX_ID = 0; +constexpr uint32_t TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM = 23; +constexpr uint32_t TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID = 92; +constexpr uint32_t TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM = 36; +constexpr uint32_t TILEXR_CCU_HCOMM_MAX_INNER_FE_ID = 7; constexpr uint64_t TILEXR_CCU_V1_WQE_BASIC_BLOCK_OFFSET = TILEXR_CCU_V1_CCUM_OFFSET + 0x800000ULL; constexpr uint64_t TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES = 256ULL * 1024ULL; constexpr uint32_t TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH = 64; @@ -33,6 +37,7 @@ constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_MAX_POLLS = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_SLEEP_US = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS = 8; constexpr uint8_t TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT = 16; +constexpr uint8_t TILEXR_CCU_DIRECT_CTP_ENDPOINT_ERR_TIMEOUT = 8; constexpr int TILEXR_CCU_DIRECT_MAX_RANK_SIZE = 128; constexpr int TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL = 2; constexpr int TILEXR_CCU_HCCP_ASYNC_EAGAIN = 128301; @@ -458,10 +463,30 @@ uint32_t SelectEndpointRouteSqBytes(uint32_t sqDepth) return sqDepth * TILEXR_CCU_DIRECT_SQ_EBB_WORDS * TILEXR_CCU_DIRECT_CCUM_SQE_BYTES; } -uint64_t SelectEndpointRouteSqVa(const TileXRCcuLocalResourceWindowInfo& localResourceWindow) +bool SelectEndpointRouteJettyCtxId(uint32_t pfeId, uint32_t peerOrdinal, uint16_t* jettyCtxId) +{ + if (jettyCtxId == nullptr) { + return false; + } + const uint32_t start = pfeId > TILEXR_CCU_HCOMM_MAX_INNER_FE_ID ? + TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID : + pfeId * TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + const uint32_t count = pfeId > TILEXR_CCU_HCOMM_MAX_INNER_FE_ID ? + TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM : + TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + if (peerOrdinal >= count || start + peerOrdinal >= 128U) { + return false; + } + *jettyCtxId = static_cast(start + peerOrdinal); + return true; +} + +uint64_t SelectEndpointRouteSqVa( + const TileXRCcuLocalResourceWindowInfo& localResourceWindow, + uint16_t jettyCtxId) { return localResourceWindow.addr + TILEXR_CCU_V1_WQE_BASIC_BLOCK_OFFSET + - static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_CTX_ID) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; + static_cast(jettyCtxId) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; } int WaitRaCtxAsyncRequest(TileXRCcuHccpLoader& loader, void* reqHandle) @@ -1371,7 +1396,9 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( ReleaseLocalEndpointRoute(); const uint32_t sqDepth = SelectEndpointRouteSqDepth(); - const uint64_t sqVa = SelectEndpointRouteSqVa(localResourceWindow_); + const uint64_t sqVa = SelectEndpointRouteSqVa( + localResourceWindow_, + TILEXR_CCU_DIRECT_LOOP_JETTY_CTX_ID); const uint32_t sqBytes = SelectEndpointRouteSqBytes(sqDepth); if (TraceEndpointRoute()) { std::cerr << "TileXRDirectCcuTrace endpointRoute begin" @@ -1648,6 +1675,7 @@ int TileXRCcuDirectRuntime::CreatePeerEndpointState( uint32_t peerDevicePhyId, const std::array& localEid, const std::array& peerEid, + uint32_t tpType, uint32_t peerOrdinal, TileXRCcuPeerEndpointState* state) { @@ -1658,6 +1686,10 @@ int TileXRCcuDirectRuntime::CreatePeerEndpointState( *state = TileXRCcuPeerEndpointState {}; state->peerRank = peerRank; state->peerDevicePhyId = peerDevicePhyId; + state->tpType = tpType; + if (tpType != TILEXR_CCU_HCCP_TP_TYPE_RTP && tpType != TILEXR_CCU_HCCP_TP_TYPE_CTP) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } TileXRCcuRaInfo raInfo {}; raInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; @@ -1709,47 +1741,34 @@ int TileXRCcuDirectRuntime::CreatePeerEndpointState( TileXRCcuRaInfo randomInfo {}; randomInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; randomInfo.phyId = devicePhyId_; - ret = loader_.RaGetSecRandom(&randomInfo, &state->resourceWindow.tokenValue); + ret = loader_.RaGetSecRandom(&randomInfo, &state->jettyTokenValue); if (ret != 0) { ReleasePeerEndpointState(state); return TILEXR_ERROR_MKIRT; } - - const uint64_t alignedAddr = AlignResourceWindowAddr(localResourceWindow_.addr); - TileXRCcuHccpMrRegInfo mr {}; - mr.in.mem.addr = alignedAddr; - mr.in.mem.size = localResourceWindow_.bytes + (localResourceWindow_.addr - alignedAddr); - mr.in.ub.flags.value = 0; - mr.in.ub.flags.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; - mr.in.ub.flags.bs.tokenIdValid = 1; - mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; - mr.in.ub.flags.bs.nonPin = 1; - mr.in.ub.tokenValue = state->resourceWindow.tokenValue; - mr.in.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; - ret = loader_.RaCtxLmemRegister( - state->resourceWindow.raCtxHandle, - &mr, - &state->resourceWindow.lmemHandle); - if (ret != 0 || state->resourceWindow.lmemHandle == nullptr) { - ReleasePeerEndpointState(state); - return TILEXR_ERROR_MKIRT; - } - const uint32_t rawTokenId = mr.out.ub.tokenId != 0 ? mr.out.ub.tokenId : allocatedToken.tokenId; state->resourceWindow.addr = localResourceWindow_.addr; state->resourceWindow.bytes = localResourceWindow_.bytes; - state->resourceWindow.rawTokenId = rawTokenId; - state->resourceWindow.tokenId = rawTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; - state->resourceWindow.targetSegHandle = mr.out.ub.targetSegHandle; + state->resourceWindow.rawTokenId = localResourceWindow_.rawTokenId; + state->resourceWindow.tokenId = localResourceWindow_.tokenId; + state->resourceWindow.tokenValue = localResourceWindow_.tokenValue; + state->resourceWindow.targetSegHandle = localResourceWindow_.targetSegHandle; state->resourceWindow.eid = localEid; state->resourceWindow.eidIndex = state->eidInfo.eidIndex; state->resourceWindow.funcId = state->eidInfo.funcId; state->resourceWindow.funcIdValid = true; - state->resourceWindow.raCtxRegistered = true; + state->resourceWindow.raCtxRegistered = false; + + uint16_t jettyCtxId = 0; + if (!SelectEndpointRouteJettyCtxId(state->eidInfo.funcId, peerOrdinal, &jettyCtxId)) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } ret = SelectTpRouteForPeer( state->resourceWindow.raCtxHandle, localEid, peerEid, + state->tpType, &state->localTpHandle, &state->mappedJettyPriority); if (ret != TILEXR_SUCCESS) { @@ -1779,19 +1798,19 @@ int TileXRCcuDirectRuntime::CreatePeerEndpointState( qpAttr.rqDepth = TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT; qpAttr.transportMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; qpAttr.ub.mode = static_cast(TILEXR_CCU_HCCP_JETTY_MODE_CCU); - qpAttr.ub.jettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + peerOrdinal); + qpAttr.ub.jettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + jettyCtxId); qpAttr.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; - qpAttr.ub.tokenValue = state->resourceWindow.tokenValue; + qpAttr.ub.tokenValue = state->jettyTokenValue; qpAttr.ub.flag.value = 0; qpAttr.ub.flag.bs.shareJfr = 1; qpAttr.ub.jfsFlag.bs.errorSuspend = 1; qpAttr.ub.priority = state->mappedJettyPriority; qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; - qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; + qpAttr.ub.errTimeout = state->tpType == TILEXR_CCU_HCCP_TP_TYPE_CTP ? + TILEXR_CCU_DIRECT_CTP_ENDPOINT_ERR_TIMEOUT : TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; qpAttr.ub.extMode.cstmFlag.value = 0; qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; - qpAttr.ub.extMode.sq.buffVa = SelectEndpointRouteSqVa(localResourceWindow_) + - static_cast(peerOrdinal) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; + qpAttr.ub.extMode.sq.buffVa = SelectEndpointRouteSqVa(localResourceWindow_, jettyCtxId); qpAttr.ub.extMode.sq.buffSize = SelectEndpointRouteSqBytes(sqDepth); qpAttr.ub.extMode.sqebbNum = sqDepth; ret = loader_.RaCtxQpCreate( @@ -1814,6 +1833,9 @@ int TileXRCcuDirectRuntime::CreatePeerEndpointState( << " funcId=" << state->eidInfo.funcId << " tpHandle=0x" << std::hex << state->localTpHandle << std::dec << " priority=" << static_cast(state->mappedJettyPriority) + << " tpType=" << state->tpType + << " jettyCtxId=" << jettyCtxId + << " sqVa=0x" << std::hex << qpAttr.ub.extMode.sq.buffVa << std::dec << " qpId=" << state->qpInfo.ub.id << " psn=" << state->psn << std::endl; @@ -1885,6 +1907,7 @@ int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeRepo peerDevicePhyIds[ordinal], topologyRoutes[ordinal].localEid, peerEid, + topologyRoutes[ordinal].tpType, ordinal, &state); if (ret != TILEXR_SUCCESS) { @@ -1901,7 +1924,7 @@ int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeRepo offer.resourceTokenId = state.resourceWindow.tokenId; offer.resourceRawTokenId = state.resourceWindow.rawTokenId; offer.resourceTokenValue = state.resourceWindow.tokenValue; - offer.jettyTokenValue = state.resourceWindow.tokenValue; + offer.jettyTokenValue = state.jettyTokenValue; offer.eid = state.resourceWindow.eid; offer.qpKey = state.qpInfo.key; if (offer.qpKey.size == 0 || offer.qpKey.size > TILEXR_CCU_HCCP_QP_KEY_BYTES) { @@ -1973,7 +1996,7 @@ int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeRepo importInfo.in.ub.expImportCfg.peerTpHandle = peerTpHandle; importInfo.in.ub.expImportCfg.txPsn = state.psn; importInfo.in.ub.expImportCfg.rxPsn = peerOffer.psn; - importInfo.in.ub.tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; + importInfo.in.ub.tpType = state.tpType; ret = loader_.RaCtxQpImport( state.resourceWindow.raCtxHandle, &importInfo, @@ -1987,7 +2010,7 @@ int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeRepo state.route.doorbellVa = state.qpInfo.ub.dbAddr; state.route.doorbellTokenId = state.qpInfo.ub.dbTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; - state.route.doorbellTokenValue = state.resourceWindow.tokenValue; + state.route.doorbellTokenValue = state.jettyTokenValue; state.route.sqDepth = SelectEndpointRouteSqDepth(); state.route.startJettyId = static_cast(state.qpInfo.ub.id); state.route.remoteCcuVa = peerOffer.resourceAddr; @@ -2041,6 +2064,7 @@ int TileXRCcuDirectRuntime::SelectTpRouteForPeer( void* ctxHandle, const std::array& localEid, const std::array& peerEid, + uint32_t tpType, uint64_t* tpHandle, uint8_t* mappedJettyPriority) { @@ -2054,7 +2078,8 @@ int TileXRCcuDirectRuntime::SelectTpRouteForPeer( *mappedJettyPriority = 0; TileXRCcuHccpGetTpCfg tpCfg {}; - tpCfg.flag.bs.rtp = 1; + tpCfg.flag.bs.rtp = tpType == TILEXR_CCU_HCCP_TP_TYPE_RTP ? 1 : 0; + tpCfg.flag.bs.ctp = tpType == TILEXR_CCU_HCCP_TP_TYPE_CTP ? 1 : 0; tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; std::copy(localEid.begin(), localEid.end(), tpCfg.localEid.raw); std::copy(peerEid.begin(), peerEid.end(), tpCfg.peerEid.raw); @@ -2094,21 +2119,23 @@ int TileXRCcuDirectRuntime::SelectTpRouteForPeer( return TILEXR_ERROR_NOT_FOUND; } - TileXRCcuHccpTpAttr setAttr {}; - setAttr.sl = mappedSl; - reqHandle = nullptr; - ret = loader_.RaSetTpAttrAsync( - ctxHandle, - tpInfos[tpIndex].tpHandle, - TILEXR_CCU_TP_ATTR_BITMAP_SL, - &setAttr, - &reqHandle); - if (ret != 0 || reqHandle == nullptr) { - return TILEXR_ERROR_MKIRT; - } - ret = WaitRaCtxAsyncRequest(loader_, reqHandle); - if (ret != TILEXR_SUCCESS) { - return ret; + if (tpType == TILEXR_CCU_HCCP_TP_TYPE_RTP) { + TileXRCcuHccpTpAttr setAttr {}; + setAttr.sl = mappedSl; + reqHandle = nullptr; + ret = loader_.RaSetTpAttrAsync( + ctxHandle, + tpInfos[tpIndex].tpHandle, + TILEXR_CCU_TP_ATTR_BITMAP_SL, + &setAttr, + &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + return ret; + } } if (TraceEndpointRoute()) { diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.h b/src/comm/ccu/tilexr_ccu_direct_runtime.h index f664a89e..d016c740 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.h +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.h @@ -137,6 +137,8 @@ struct TileXRCcuPeerEndpointState { void* remoteQpHandle = nullptr; TileXRCcuHccpQpCreateInfo qpInfo {}; TileXRCcuLowerLayerTransportRoute route; + uint32_t jettyTokenValue = 0; + uint32_t tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; uint32_t psn = 0; uint64_t localTpHandle = 0; uint8_t mappedJettyPriority = 0; @@ -176,12 +178,14 @@ class TileXRCcuDirectRuntime { uint32_t peerDevicePhyId, const std::array& localEid, const std::array& peerEid, + uint32_t tpType, uint32_t peerOrdinal, TileXRCcuPeerEndpointState* state); int SelectTpRouteForPeer( void* ctxHandle, const std::array& localEid, const std::array& peerEid, + uint32_t tpType, uint64_t* tpHandle, uint8_t* mappedJettyPriority); int QueryTpHandleForPeer( diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index 39515270..03b8dc8d 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -500,10 +500,7 @@ int TileXRCcuBuildLowerLayerTransportTemplate( remoteNotifyCke.num == 0 || allocation.remoteXn.num == 0) { return Fail(nullptr, report, "missing lower-layer CCU allocated resources"); } - if (remoteCcuBuffers.empty() || remoteCcuBuffers.size() != allocation.remoteXn.num) { - return Fail(nullptr, report, "remote CCU buffer template count does not match remote XN allocation"); - } - if (allocation.channels.num < remoteCcuBuffers.size()) { + if (remoteCcuBuffers.empty() || remoteCcuBuffers.size() != allocation.channels.num) { return Fail(nullptr, report, "channel allocation count does not match lower-layer route count"); } if (remoteCcuBuffers.size() > std::numeric_limits::max()) { diff --git a/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp index f364290d..b160ca62 100644 --- a/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp +++ b/src/comm/ccu/tilexr_ccu_ra_custom_channel_provider.cpp @@ -5,9 +5,17 @@ #include "ccu/tilexr_ccu_ra_custom_channel_provider.h" +#include +#include +#include + namespace TileXR { namespace { +constexpr int TILEXR_CCU_ROCE_EAGAIN = 128101; +constexpr uint32_t TILEXR_CCU_RA_EAGAIN_RETRY_COUNT = 100; +constexpr uint32_t TILEXR_CCU_RA_EAGAIN_RETRY_INTERVAL_MS = 100; + void ResetReport(TileXRCcuRaCustomChannelProviderReport* report) { if (report != nullptr) { @@ -103,10 +111,15 @@ int TileXRCcuRaCustomChannelProvider::AdapterCallback( TileXRCcuRaInfo info {}; info.mode = TILEXR_CCU_NETWORK_OFFLINE; info.phyId = devicePhyId; - return provider->raCustomChannel_( - info, - in, - out); + for (uint32_t retry = 0; retry <= TILEXR_CCU_RA_EAGAIN_RETRY_COUNT; ++retry) { + std::memset(out, 0, sizeof(*out)); + const int ret = provider->raCustomChannel_(info, in, out); + if (ret != TILEXR_CCU_ROCE_EAGAIN || retry == TILEXR_CCU_RA_EAGAIN_RETRY_COUNT) { + return ret; + } + std::this_thread::sleep_for(std::chrono::milliseconds(TILEXR_CCU_RA_EAGAIN_RETRY_INTERVAL_MS)); + } + return TILEXR_CCU_ROCE_EAGAIN; } } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp index 5b4840f7..a797a149 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp @@ -14,6 +14,7 @@ namespace { constexpr const char* TILEXR_CCU_HCOMM_DERIVED_PROVIDER = "tilexr-hcomm-derived-resource-allocator"; constexpr uint32_t TILEXR_CCU_HCOMM_TASK1_PRELUDE_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_HCOMM_TASK1_PRELUDE_RESERVED_XN_COUNT = 1U; +constexpr uint32_t TILEXR_CCU_CHANNEL_XN_STRIDE = 8U; void ResetReport(TileXRCcuResourceAllocatorReport* report) { @@ -212,8 +213,19 @@ int TileXRCcuResourceAllocator::Allocate( } const uint32_t localSqeXnCount = RequiredSqeLoadXnCount(request.sqeArgCount, hcommStyleTask1Prelude); - const uint32_t localXnCount = std::max(localSqeXnCount, request.syncResourceCount); - const uint32_t remoteXnCount = request.syncResourceCount; + const uint32_t localXnCount = std::max( + std::max( + std::max(localSqeXnCount, request.syncResourceCount), + request.bindingsPerSyncResource), + request.minimumLocalXnCount); + const uint32_t remoteXnStride = request.bindingsPerSyncResource > 1U ? + TILEXR_CCU_CHANNEL_XN_STRIDE : 1U; + if (request.syncResourceCount > std::numeric_limits::max() / remoteXnStride) { + return Fail(report, "remote XN resource count overflows"); + } + const uint32_t remoteXnCount = std::max( + request.syncResourceCount * remoteXnStride, + request.minimumRemoteXnCount); const uint32_t localGsaCount = hcommStyleTask1Prelude && spec_.gsaCount != 0 ? 1U : 0U; const uint32_t totalXnCount = localXnCount + remoteXnCount; const uint32_t localWaitCkeCount = request.syncResourceCount; @@ -301,7 +313,8 @@ int TileXRCcuResourceAllocator::Allocate( TileXRCcuSyncResource resource; resource.dieId = spec_.dieId; resource.localXn = static_cast(result.localXn.startId + i); - resource.remoteXn = static_cast(static_cast(result.remoteXn.startId) + i); + resource.remoteXn = static_cast( + static_cast(result.remoteXn.startId) + i * remoteXnStride); resource.notifyCke = static_cast(static_cast(result.remoteNotifyCke.startId) + i); resource.channelId = static_cast(static_cast(result.channels.startId) + i); resource.bindingCount = CheckedU16(request.bindingsPerSyncResource); diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.h b/src/comm/ccu/tilexr_ccu_resource_allocator.h index 959e712d..67e7e040 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.h +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.h @@ -43,6 +43,8 @@ struct TileXRCcuResourceRequest { uint32_t syncResourceCount = 0; uint32_t syncInstructionCount = 0; uint32_t bindingsPerSyncResource = 1; + uint32_t minimumLocalXnCount = 0; + uint32_t minimumRemoteXnCount = 0; uint32_t sourceCkeCount = 1; TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; }; diff --git a/src/comm/ccu/tilexr_ccu_topology.cpp b/src/comm/ccu/tilexr_ccu_topology.cpp index fc20242a..c03e6ee2 100644 --- a/src/comm/ccu/tilexr_ccu_topology.cpp +++ b/src/comm/ccu/tilexr_ccu_topology.cpp @@ -17,6 +17,9 @@ namespace TileXR { namespace { +constexpr const char* TILEXR_CCU_DIRECT_FORCE_TP_TYPE_ENV = + "TILEXR_CCU_DIRECT_FORCE_TP_TYPE"; + struct RootInfo { std::string topoPath; std::unordered_map deviceToLocalId; @@ -29,6 +32,7 @@ struct TopoEdge { uint32_t localB = 0; std::vector localAPorts; std::vector localBPorts; + bool supportsCtp = false; }; std::string ReadTextFile(const std::string& path) @@ -195,6 +199,8 @@ std::vector ParseTopoInfo(const std::string& path) } edge.localAPorts = JsonStringArrayField(edgeObject, "local_a_ports"); edge.localBPorts = JsonStringArrayField(edgeObject, "local_b_ports"); + const auto protocols = JsonStringArrayField(edgeObject, "protocols"); + edge.supportsCtp = std::find(protocols.begin(), protocols.end(), "UB_CTP") != protocols.end(); if (!edge.localAPorts.empty() && !edge.localBPorts.empty()) { edges.push_back(edge); } @@ -206,24 +212,43 @@ bool ResolveLocalPort( const std::vector& edges, uint32_t localId, uint32_t peerLocalId, - std::string* localPort) + std::string* localPort, + bool* supportsCtp) { - if (localPort == nullptr) { + if (localPort == nullptr || supportsCtp == nullptr) { return false; } for (const auto& edge : edges) { if (edge.localA == localId && edge.localB == peerLocalId) { *localPort = edge.localAPorts.front(); + *supportsCtp = edge.supportsCtp; return true; } if (edge.localB == localId && edge.localA == peerLocalId) { *localPort = edge.localBPorts.front(); + *supportsCtp = edge.supportsCtp; return true; } } return false; } +int ForcedTpType() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_FORCE_TP_TYPE_ENV); + if (value == nullptr) { + return -1; + } + const std::string text(value); + if (text == "rtp" || text == "RTP" || text == "0") { + return static_cast(TILEXR_CCU_HCCP_TP_TYPE_RTP); + } + if (text == "ctp" || text == "CTP" || text == "1") { + return static_cast(TILEXR_CCU_HCCP_TP_TYPE_CTP); + } + return -1; +} + } // namespace int TileXRCcuResolvePeerEidRoutes( @@ -262,8 +287,9 @@ int TileXRCcuResolvePeerEidRoutes( for (const uint32_t peerDevicePhyId : peerDevicePhyIds) { const auto peerIdIt = root.deviceToLocalId.find(peerDevicePhyId); std::string localPort; + bool supportsCtp = false; if (peerIdIt == root.deviceToLocalId.end() || - !ResolveLocalPort(edges, localIdIt->second, peerIdIt->second, &localPort)) { + !ResolveLocalPort(edges, localIdIt->second, peerIdIt->second, &localPort, &supportsCtp)) { if (message != nullptr) { *message = "HCCL topology has no device-pair edge"; } @@ -282,6 +308,10 @@ int TileXRCcuResolvePeerEidRoutes( route.peerDevicePhyId = peerDevicePhyId; route.localEid = eidIt->second; route.localPort = localPort; + const int forcedTpType = ForcedTpType(); + route.tpType = forcedTpType >= 0 ? + static_cast(forcedTpType) : + (supportsCtp ? TILEXR_CCU_HCCP_TP_TYPE_CTP : TILEXR_CCU_HCCP_TP_TYPE_RTP); routes->push_back(route); } if (message != nullptr) { diff --git a/src/comm/ccu/tilexr_ccu_topology.h b/src/comm/ccu/tilexr_ccu_topology.h index cbb68d96..de8ac29f 100644 --- a/src/comm/ccu/tilexr_ccu_topology.h +++ b/src/comm/ccu/tilexr_ccu_topology.h @@ -20,6 +20,7 @@ struct TileXRCcuPeerEidRoute { uint32_t peerDevicePhyId = 0; std::array localEid {}; std::string localPort; + uint32_t tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; }; int TileXRCcuResolvePeerEidRoutes( diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 288b04c1..b8a17578 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -630,7 +630,7 @@ uint8_t BuildAllToAllMeshByte( int InitAllToAllMeshState(int rank, int rankSize, AllToAllState* state) { - if (state == nullptr || rank < 0 || rank >= rankSize || rankSize != 4) { + if (state == nullptr || rankSize < 2 || rankSize > 64 || rank < 0 || rank >= rankSize) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } state->chunkBytes = AllToAllBytesFromEnv(); @@ -666,7 +666,8 @@ int InitAllToAllMeshState(int rank, int rankSize, AllToAllState* state) int ResetAllToAllMeshStateForLoop(int rank, int loopIndex, AllToAllState* state) { if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || - state->rankSize != 4 || state->bytes != static_cast(state->rankSize) * state->chunkBytes || + state->rankSize < 2 || state->rankSize > 64 || + state->bytes != static_cast(state->rankSize) * state->chunkBytes || loopIndex < 0) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -716,7 +717,9 @@ int InitAllToAllState(int rank, int peer, AllToAllState* state) return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } state->bytes = AllToAllBytesFromEnv(); - if (state->bytes != 2U * 1024U * 1024U || AllToAllMemSlicePerLoopFromEnv() != 8) { + const bool supportedBytes = state->bytes == 2U * 1024U * 1024U || + state->bytes == 8U * 1024U * 1024U; + if (!supportedBytes || AllToAllMemSlicePerLoopFromEnv() != 8) { state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; return state->initRet; } @@ -1468,7 +1471,9 @@ void PrintCcuResourceState( uint8_t dieId, const TileXRDirectCcuPrepareOptions& options, const char* label, - uint32_t resourceCount = 3U) + uint32_t resourceCount = 3U, + uint32_t extraCkeStartId = 0U, + uint32_t extraCkeCount = 0U) { if (context == nullptr || label == nullptr || resourceCount == 0) { return; @@ -1486,6 +1491,7 @@ void PrintCcuResourceState( std::vector remoteXn(resourceCount, 0); std::vector localWaitCke(resourceCount, 0); std::vector remoteNotifyCke(resourceCount, 0); + std::vector extraCke(extraCkeCount, 0); const uint32_t localXnStartId = options.xnStartId; const uint32_t remoteXnStartId = options.remoteXnStartId; const uint32_t localWaitCkeStartId = options.localWaitCkeStartId; @@ -1498,6 +1504,8 @@ void PrintCcuResourceState( dieId, localWaitCkeStartId, localWaitCke.data(), resourceCount, &report); const int remoteCkeRet = adapter.ReadCkeRange( dieId, remoteNotifyCkeStartId, remoteNotifyCke.data(), resourceCount, &report); + const int extraCkeRet = extraCkeCount == 0U ? TileXR::TILEXR_SUCCESS : + adapter.ReadCkeRange(dieId, extraCkeStartId, extraCke.data(), extraCkeCount, &report); const auto values = [](const std::vector& data) { std::ostringstream out; @@ -1524,6 +1532,9 @@ void PrintCcuResourceState( << " remoteNotifyCkeStartId=" << remoteNotifyCkeStartId << " remoteCkeRet=" << remoteCkeRet << " remoteCke=" << values(remoteNotifyCke) + << " extraCkeStartId=" << extraCkeStartId + << " extraCkeRet=" << extraCkeRet + << " extraCke=" << values(extraCke) << std::endl; } @@ -2091,9 +2102,9 @@ int RunAllToAllMeshLongMissionSmokeForRank( if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (rankSize != 4) { + if (rankSize < 2 || rankSize > 64) { std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize - << " reason=\"direct CCU alltoall mesh requires four ranks\"" << std::endl; + << " reason=\"direct CCU alltoall mesh requires 2..64 ranks\"" << std::endl; return 0; } @@ -2104,10 +2115,17 @@ int RunAllToAllMeshLongMissionSmokeForRank( alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); - options.syncResourceCount = 3U; + options.syncResourceCount = static_cast(rankSize - 1); options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { - options.syncInstructionCount = 131U; + const uint32_t peerCount = static_cast(rankSize - 1); + const uint32_t completionCkeCount = (peerCount + 15U) / 16U; + const uint64_t blockCount = alltoall.chunkBytes / TileXR::TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint64_t copyPerBlock = peerCount * 6ULL + 9ULL + completionCkeCount; + options.syncInstructionCount = static_cast( + 3ULL + peerCount * 3ULL + + blockCount * copyPerBlock + + peerCount * 2ULL + 1ULL); } if (options.gsaStartId == 0) { options.gsaStartId = 1; @@ -2118,7 +2136,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( << " chunkBytes=" << alltoall.chunkBytes << " bytes=" << alltoall.bytes << " loopCount=" << loopCount - << " resourceCount=3" + << " resourceCount=" << (rankSize - 1) << " mesh=1" << " longMission=1" << std::endl; @@ -2146,7 +2164,8 @@ int RunAllToAllMeshLongMissionSmokeForRank( if (prepareRet != TileXR::TILEXR_SUCCESS) { finalRet = 6; } else if (attempt.submitTasks.size() != 1U || attempt.submitTasks.front().argSize != - TILEXR_DIRECT_CCU_SQE_ARGS_LEN || attempt.plan.syncResources.size() != 3U) { + TILEXR_DIRECT_CCU_SQE_ARGS_LEN || + attempt.plan.syncResources.size() != static_cast(rankSize - 1)) { std::cerr << "tilexr_ccu_alltoall invalidMeshPreparedTask" << " rank=" << rank << " taskCount=" << attempt.submitTasks.size() @@ -2216,7 +2235,7 @@ int RunAllToAllMeshLongMissionSmokeForRank( << " rank=" << rank << " loopIndex=" << loopIndex << " ret=" << finalRet - << " resourceCount=3" + << " resourceCount=" << (rankSize - 1) << " selfCopyCompletionCke=" << attempt.plan.syncResources[0].localWaitCke << std::endl; PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); @@ -2225,7 +2244,9 @@ int RunAllToAllMeshLongMissionSmokeForRank( attempt.submitTasks.front().dieId, options, "tilexr_ccu_alltoall", - 3U); + static_cast(rankSize - 1), + attempt.allocation.sourceCke.startId, + attempt.allocation.sourceCke.num); break; } std::cout << "tilexr_ccu_alltoall stableResources=1" @@ -2274,8 +2295,9 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); options.syncResourceCount = 3; options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; + const size_t blockCount = alltoall.bytes / TileXR::TILEXR_CCU_ALLTOALL_BLOCK_BYTES; if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { - options.syncInstructionCount = 5 + 64 * 7; + options.syncInstructionCount = static_cast(5U + blockCount * 7U); } if (options.gsaStartId == 0) { options.gsaStartId = 1; @@ -2287,7 +2309,7 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, << " bytes=" << alltoall.bytes << " loopCount=" << loopCount << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() - << " blockCount=64" + << " blockCount=" << blockCount << " longMission=1" << " preSync=1" << " postSync=0" diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 806594ec..26f471a1 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -234,23 +234,40 @@ apply_alltoall_defaults() export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" if alltoall_mesh_mode_enabled; then + mesh_rank_size="$(parse_int "${TILEXR_CCU_RANK_SIZE:-${TILEXR_CCU_PROBE_RANK_SIZE:-2}}" 2)" + mesh_peer_count=$((mesh_rank_size - 1)) + mesh_completion_cke_count=$(((mesh_peer_count + 15) / 16)) + mesh_chunk_bytes="$(parse_int "${TILEXR_CCU_ALLTOALL_BYTES:-131072}" 131072)" + mesh_block_count=$((mesh_chunk_bytes / 32768)) + mesh_pre_sync_count=$((3 + mesh_peer_count * 3)) + mesh_copy_per_block=$((mesh_peer_count * 6 + 9 + mesh_completion_cke_count)) + if [ "${mesh_peer_count}" -gt 16 ]; then + mesh_remote_xn_count=${mesh_peer_count} + else + mesh_remote_xn_count=16 + fi + mesh_instruction_count=$((mesh_pre_sync_count + mesh_block_count * mesh_copy_per_block + mesh_peer_count * 2 + 1)) + mesh_local_cke_count=$((mesh_peer_count + 1 + mesh_completion_cke_count)) export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE:-hcomm_die}" export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION:-hcomm}" export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" - export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-131}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-${mesh_peer_count}}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-${mesh_instruction_count}}" export TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}" export TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}" - export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-16}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-${mesh_remote_xn_count}}" export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}" - export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-16}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-${mesh_local_cke_count}}" export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}" - export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-16}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-${mesh_peer_count}}" export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX:-3}" elif [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + long_mission_bytes="$(parse_int "${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" 2097152)" + long_mission_block_count=$((long_mission_bytes / 32768)) + long_mission_instruction_count=$((7 + long_mission_block_count * 7)) export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-${long_mission_instruction_count}}" else export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" @@ -288,6 +305,10 @@ if [ "${rank_size}" -lt 1 ]; then echo "ERROR: rank size must be positive: ${rank_size}" >&2 exit 2 fi +if alltoall_mesh_mode_enabled && { [ "${rank_size}" -lt 2 ] || [ "${rank_size}" -gt 64 ]; }; then + echo "ERROR: direct CCU alltoall mesh rank size must be in [2,64]: ${rank_size}" >&2 + exit 2 +fi devices="${TILEXR_CCU_SMOKE_DEVICES:-${TILEXR_TEST_DEVICES:-0,1}}" IFS=',' read -r -a device_list <<< "${devices}" if [ "${#device_list[@]}" -ne "${rank_size}" ]; then @@ -949,7 +970,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then done fi -if alltoall_mode_enabled; then +if alltoall_mode_enabled && [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then loop_count="$(parse_int "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" 1)" expected_results=$((rank_size * loop_count)) actual_results="$(grep -h -c "tilexr_ccu_alltoall result passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index 7aeac4fe..1027808b 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -600,12 +600,12 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) spec.chunkBytes = 2ULL * 1024ULL * 1024ULL; spec.selfSourceGsa = 0x180; spec.selfDestinationGsa = 0x181; - spec.selfSourceXn = 0x280; - spec.selfDestinationXn = 0x281; - spec.selfLengthXn = 0x282; + spec.selfSourceXn = 0x200; + spec.selfDestinationXn = 0x201; + spec.selfLengthXn = 0x202; spec.selfChannelId = 0; spec.selfCompletionCke = 0x480; - spec.remoteCompletionCke = 0x491; + spec.remoteCompletionCkes = {0x491}; spec.peers = {Peer(2, 3, 2), Peer(2, 0, 0), Peer(2, 1, 1)}; std::vector program; @@ -693,9 +693,9 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) return 7; } auto overlappingCke = spec; - overlappingCke.remoteCompletionCke = overlappingCke.peers[0].route.sourceCke; + overlappingCke.remoteCompletionCkes[0] = overlappingCke.peers[0].route.sourceCke; for (auto& peer : overlappingCke.peers) { - peer.route.copyCompletionCke = overlappingCke.remoteCompletionCke; + peer.route.copyCompletionCke = overlappingCke.remoteCompletionCkes[0]; } if (TileXRCcuBuildAllToAllMeshProgram(overlappingCke, &program, &report) != TILEXR_ERROR_PARA_CHECK_FAIL || @@ -733,6 +733,62 @@ def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self) return 9; } } + auto spec8 = spec; + spec8.rankSize = 8; + spec8.localRank = 3; + spec8.peers.clear(); + spec8.remoteCompletionCkes = {0x491}; + uint16_t ordinal8 = 0; + for (uint32_t peerRank = 0; peerRank < spec8.rankSize; ++peerRank) { + if (peerRank != spec8.localRank) { + spec8.peers.push_back(Peer(spec8.localRank, peerRank, ordinal8++)); + } + } + if (TileXRCcuBuildAllToAllMeshProgram(spec8, &program, &report) != TILEXR_SUCCESS || + program.size() != 3367 || report.peerCount != 7 || report.syncResourceCount != 7 || + report.remoteBlockCount != 448 || report.selfBlockCount != 64) { + std::cerr << "unexpected 8-rank 2MB mesh: " << report.message + << " instructions=" << program.size() << "\n"; + return 10; + } + + auto spec2 = spec; + spec2.rankSize = 2; + spec2.localRank = 0; + spec2.peers = {Peer(0, 1, 0)}; + spec2.remoteCompletionCkes = {0x491}; + if (TileXRCcuBuildAllToAllMeshProgram(spec2, &program, &report) != TILEXR_SUCCESS || + program.size() != 1033 || report.peerCount != 1 || report.syncResourceCount != 1 || + report.remoteBlockCount != 64 || report.selfBlockCount != 64) { + std::cerr << "unexpected 2-rank 2MB full mesh: " << report.message + << " instructions=" << program.size() << "\n"; + return 11; + } + + auto spec64 = spec; + spec64.rankSize = 64; + spec64.localRank = 17; + spec64.chunkBytes = 128ULL * 1024ULL; + spec64.peers.clear(); + spec64.remoteCompletionCkes = {0x491, 0x492, 0x493, 0x494}; + uint16_t ordinal64 = 0; + for (uint32_t peerRank = 0; peerRank < spec64.rankSize; ++peerRank) { + if (peerRank == spec64.localRank) { + continue; + } + auto peer = Peer(spec64.localRank, peerRank, ordinal64); + peer.route.bytes = spec64.chunkBytes; + peer.route.copyCompletionCke = spec64.remoteCompletionCkes[ordinal64 / 16U]; + spec64.peers.push_back(peer); + ++ordinal64; + } + if (TileXRCcuBuildAllToAllMeshProgram(spec64, &program, &report) != TILEXR_SUCCESS || + program.size() != 1883 || report.peerCount != 63 || report.syncResourceCount != 63 || + report.remoteBlockCount != 252 || report.selfBlockCount != 4) { + std::cerr << "unexpected 64-rank 128KB mesh: " << report.message + << " instructions=" << program.size() << "\n"; + return 12; + } return 0; } ''' diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 7c55de78..689af500 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -235,6 +235,7 @@ def test_alltoall_overrides_only_copy_route_memory_not_sync_routes(self): ] self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(", prepare_alltoall) + self.assertIn("peerRanks.size() < routedPeerCount", planner) self.assertIn("0U", prepare_alltoall) self.assertIn("uint32_t routeIndex = 0", override_apply) self.assertIn("override.syncRouteIndex != routeIndex", override_apply) @@ -251,7 +252,9 @@ def test_four_rank_mesh_gathers_imports_and_maps_three_routes_per_peer(self): planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt"): planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt") ] - self.assertIn("rankSize != 4", mesh_body) + self.assertIn("rankSize < 2", mesh_body) + self.assertIn("rankSize > 64", mesh_body) + self.assertIn("rankSize - 1", mesh_body) self.assertEqual(1, mesh_body.count("session.AllGather(")) self.assertIn("endpoint.rank != peerRank", mesh_body) self.assertIn("session.ImportRemoteMemoryBuffer", mesh_body) @@ -265,7 +268,7 @@ def test_four_rank_mesh_gathers_imports_and_maps_three_routes_per_peer(self): planner.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): planner.index("void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride") ] - self.assertIn("routesPerPeer = syncRouteCount / peerRouteCount", exchange) + self.assertIn("routesPerPeer = syncRouteCount / routedPeerCount", exchange) self.assertIn("peerBufferIndex = syncIndex / routesPerPeer", exchange) self.assertIn("peerLocalResourceOffset =", exchange) self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", exchange) diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index f30e01bf..62359813 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -506,8 +506,8 @@ def test_direct_install_attempt_becomes_submit_ready_with_remote_xn_peer_exchang lowerLayer.xnClears.push_back({1, 1961, 14}); lowerLayer.ckeClears.push_back({1, 332, 3}); lowerLayer.remoteXnBindings.push_back({1, 2, 1961, 1975, 332, 0, true, 0, true, true, true}); - lowerLayer.remoteXnBindings.push_back({1, 3, 1962, 1976, 333, 0, true, 0, true, true, true}); - lowerLayer.remoteXnBindings.push_back({1, 4, 1963, 1977, 334, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 3, 1962, 1983, 333, 0, true, 0, true, true, true}); + lowerLayer.remoteXnBindings.push_back({1, 4, 1963, 1991, 334, 0, true, 0, true, true, true}); return lowerLayer; } @@ -872,7 +872,7 @@ def test_direct_install_attempt_can_prepare_lower_layer_plan_after_allocation(se { auto* state = static_cast(userData); ++state->callCount; - state->syncResourceCount = allocation.remoteXn.num; + state->syncResourceCount = allocation.channels.num; if (plan == nullptr || report == nullptr) { return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -912,12 +912,12 @@ def test_direct_install_attempt_can_prepare_lower_layer_plan_after_allocation(se } plan->xnClears.push_back({1, allocation.localXn.startId, allocation.localXn.num}); plan->ckeClears.push_back({1, allocation.notifyCke.startId, allocation.notifyCke.num}); - for (uint32_t i = 0; i < allocation.remoteXn.num; ++i) { + for (uint32_t i = 0; i < allocation.channels.num; ++i) { plan->remoteXnBindings.push_back({ 1, static_cast(allocation.channels.startId + i), static_cast(allocation.localXn.startId + i), - static_cast(allocation.remoteXn.startId + i), + static_cast(allocation.remoteXn.startId + i * 8U), static_cast(allocation.notifyCke.startId + i), i, true, @@ -973,7 +973,7 @@ def test_direct_install_attempt_can_prepare_lower_layer_plan_after_allocation(se options.xnStartId = 1961; options.gsaStartId = 510; options.remoteXnStartId = 2361; - options.remoteXnCount = 8; + options.remoteXnCount = 24; options.ckeStartId = 332; options.remoteNotifyCkeStartId = 364; options.remoteNotifyCkeCount = 8; @@ -1133,12 +1133,12 @@ def test_direct_install_attempt_passes_split_cke_ranges_to_lower_layer_callback( } plan->xnClears.push_back({1, allocation.localXn.startId, allocation.localXn.num}); plan->ckeClears.push_back({1, allocation.localWaitCke.startId, allocation.localWaitCke.num}); - for (uint32_t i = 0; i < allocation.remoteXn.num; ++i) { + for (uint32_t i = 0; i < allocation.channels.num; ++i) { plan->remoteXnBindings.push_back({ 1, allocation.channels.startId + i, static_cast(allocation.localXn.startId + i), - static_cast(allocation.remoteXn.startId + i), + static_cast(allocation.remoteXn.startId + i * 8U), static_cast(allocation.remoteNotifyCke.startId + i), i, true, @@ -1756,8 +1756,8 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): ] self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", planner) + self.assertIn("DirectAllToAll2RankInstructionCapacity", source) + self.assertIn("DirectAllToAll2RankInstructionCapacity", planner) self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn", source) self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) @@ -1896,12 +1896,83 @@ def test_direct_four_rank_mesh_builds_one_three_channel_launch_package(self): attempt.package.program.sync.size() != 1811 || attempt.plan.taskWindows[0].instCnt != 1811 || attempt.plan.kernelLocalGsa.num != 2 || attempt.allocation.sourceCke.num != 2 || + attempt.allocation.remoteXn.num != 3 || + attempt.plan.syncResources[1].remoteXn - + attempt.plan.syncResources[0].remoteXn != 1 || + attempt.plan.syncResources[2].remoteXn - + attempt.plan.syncResources[1].remoteXn != 1 || attempt.plan.barrierMode != TileXRCcuBarrierMode::SyncCke) { std::cerr << "unexpected mesh package: " << report.message << " resources=" << attempt.plan.syncResources.size() << " instructions=" << attempt.package.program.sync.size() << "\n"; return 1; } + auto mesh8 = mesh; + mesh8.rankSize = 8; + mesh8.peers.clear(); + for (uint32_t peerRank = 0; peerRank < mesh8.rankSize; ++peerRank) { + if (peerRank == mesh8.localRank) { + continue; + } + TileXRCcuDirectAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + peer.remoteRecvAddr = 0x40000000ULL + peerRank * 0x1000000ULL; + peer.remoteRecvToken = TileXRCcuPackMemoryToken(30 + peerRank, 40 + peerRank, true); + mesh8.peers.push_back(peer); + } + TileXRCcuDirectInstallAttempt attempt8; + TileXRCcuDirectInstallReport report8; + const int ret8 = TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh8, &attempt8, &report8); + (void)ret8; + if (!report8.pipelineBuilt || attempt8.plan.syncResources.size() != 7 || + attempt8.package.program.sync.size() != 3367 || + attempt8.plan.taskWindows[0].instCnt != 3367 || + attempt8.allocation.remoteXn.num != 7 || + attempt8.plan.syncResources[6].remoteXn - + attempt8.plan.syncResources[5].remoteXn != 1 || + attempt8.allocation.sourceCke.num != 2) { + std::cerr << "unexpected 8-rank mesh package: " << report8.message + << " resources=" << attempt8.plan.syncResources.size() + << " instructions=" << attempt8.package.program.sync.size() << "\n"; + return 3; + } + auto mesh2 = mesh; + mesh2.rankSize = 2; + mesh2.localRank = 0; + mesh2.peers.clear(); + TileXRCcuDirectAllToAllMeshPeerSpec peer2; + peer2.peerRank = 1; + peer2.remoteRecvAddr = 0x50000000ULL; + peer2.remoteRecvToken = TileXRCcuPackMemoryToken(50, 60, true); + mesh2.peers.push_back(peer2); + TileXRCcuDirectInstallAttempt attempt2; + TileXRCcuDirectInstallReport report2; + const int ret2 = TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh2, &attempt2, &report2); + (void)ret2; + if (!report2.pipelineBuilt || attempt2.plan.syncResources.size() != 1 || + attempt2.package.program.sync.size() != 1033 || + attempt2.plan.taskWindows[0].instCnt != 1033 || + attempt2.allocation.localXn.num != 3 || + attempt2.allocation.remoteXn.num != 3 || + attempt2.allocation.sourceCke.num != 2) { + std::cerr << "unexpected 2-rank full mesh package: " << report2.message + << " resources=" << attempt2.plan.syncResources.size() + << " instructions=" << attempt2.package.program.sync.size() << "\n"; + return 4; + } + auto invalidMesh = mesh; + invalidMesh.rankSize = 0xffffffffU; + invalidMesh.peers.clear(); + TileXRCcuDirectInstallAttempt invalidAttempt; + TileXRCcuDirectInstallReport invalidReport; + if (TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, invalidMesh, &invalidAttempt, &invalidReport) != + TILEXR_ERROR_PARA_CHECK_FAIL || invalidReport.pipelineBuilt) { + std::cerr << "unexpected invalid rank-size result: " << invalidReport.message << "\n"; + return 5; + } basic.caps.cap0 = (7U << 24) | (11U << 16) | 1599U; TileXRCcuDirectInstallAttempt smallAttempt; TileXRCcuDirectInstallReport smallReport; @@ -1959,7 +2030,7 @@ def test_collective_planner_has_private_alltoall_prepare_path(self): self.assertIn("alltoall.remoteRecvAddr = remoteImportRequest.addr", source) self.assertIn("alltoall.remoteRecvToken", source) self.assertNotIn("alltoall.remoteRecvAddr = peerEndpoint.destinationAddr", source) - self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) + self.assertIn("DirectAllToAll2RankInstructionCapacity(bytes)", source) self.assertIn("tilexr-comm-direct-ccu-alltoall", source) self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) @@ -1990,7 +2061,7 @@ def test_direct_sync_xn_ping_uses_one_mission_route_and_full_4p_transport_resour self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK", planner) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK", planner) self.assertIn("RegisterCcuResourceRmaBuffer", planner) - self.assertIn("next.syncResourceCount = rankSize == 4 ? 3U : 1U", planner) + self.assertIn("next.syncResourceCount = static_cast(rankSize - 1)", planner) self.assertIn("attempt->plan.syncResources.empty()", source) self.assertIn("syncXnPing != nullptr ? options.syncResourceCount", source) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index ae8ad68f..c39186a1 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -738,7 +738,9 @@ def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(s source.index("int RunAllToAllLongMissionSmokeForRank") ] loop = "for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)" - self.assertIn("rankSize != 4", body) + self.assertIn("rankSize < 2", body) + self.assertIn("rankSize > 64", body) + self.assertIn("rankSize - 1", body) self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", body) self.assertIn("aclrtCreateStream", body) self.assertIn(loop, body) @@ -751,7 +753,7 @@ def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(s self.assertNotIn("ReadAndValidatePeerLoopMarker", body) self.assertIn("CheckAllToAllState(&alltoall)", body) self.assertIn("PrintCcuResourceState", body) - self.assertIn("resourceCount=3", body) + self.assertIn('" resourceCount=" << (rankSize - 1)', body) pattern = source[ source.index("uint8_t BuildAllToAllMeshByte"): @@ -837,7 +839,7 @@ def test_smoke_runner_forwards_alltoall_env(self): self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) self.assertIn("TILEXR_CCU_ALLTOALL_LOOP_COUNT", runner) self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) - self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453", runner) + self.assertIn("long_mission_instruction_count=$((7 + long_mission_block_count * 7))", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-452", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-458", runner) diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index 99278d67..e5c83c3c 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -17,7 +17,9 @@ class TileXRCcuDirectSmokeRunnerTest(unittest.TestCase): - def run_fake_mesh_runner(self, devices="4,5,6,7", rank_size="4", loop_count="10"): + def run_fake_mesh_runner( + self, devices="4,5,6,7", rank_size="4", loop_count="10", submit=True + ): temp_dir = tempfile.TemporaryDirectory() temp_path = Path(temp_dir.name) fake_bin = temp_path / "bin" @@ -35,6 +37,7 @@ def run_fake_mesh_runner(self, devices="4,5,6,7", rank_size="4", loop_count="10" "rank_size=${TILEXR_CCU_PROBE_RANK_SIZE}\n" "loops=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}\n" "echo \"tilexr_ccu_alltoall prepare ret=0 installSucceeded=1 submitReady=1\"\n" + "if [ \"${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}\" = 1 ]; then\n" "for ((loop=0; loop(allocation.remoteXn.startId + 8U); remote1.remoteNotifyCke = 0x361; remoteCcuBuffers.push_back(remote1); @@ -1223,7 +1231,7 @@ def test_builds_transport_template_from_basic_info_and_resource_allocation(self) if (snapshot.routes.size() != 2 || snapshot.routes[0].channelId != allocation.channels.startId || snapshot.routes[1].channelId != allocation.channels.startId + 1U || snapshot.routes[0].remoteXnId != allocation.remoteXn.startId || - snapshot.routes[1].remoteXnId != allocation.remoteXn.startId + 1U || + snapshot.routes[1].remoteXnId != allocation.remoteXn.startId + 8U || snapshot.routes[0].remoteNotifyCke != 0x360 || snapshot.routes[1].remoteNotifyCke != 0x361 || snapshot.routes[0].wqeBasicBlockStartId != 0 || @@ -2224,7 +2232,7 @@ def test_plan_builder_surface_is_wired_into_tilexr_comm_without_udma_boundary(se self.assertIn("TileXRCcuBuildPfeCtx", source) self.assertIn("TileXRCcuBuildLocalJettyCtx", source) self.assertIn("TileXRCcuBuildChannelCtxV1", source) - self.assertIn("allocation.channels.num < remoteCcuBuffers.size()", source) + self.assertIn("remoteCcuBuffers.size() != allocation.channels.num", source) self.assertIn("channel allocation count does not match lower-layer route count", source) self.assertNotIn("TILEXR_CCU_DIRECT_SYNC_RESOURCE_MAP", source) self.assertNotIn("UseHcommTraceSyncResourceMap", source) @@ -2515,6 +2523,12 @@ def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): def test_lower_layer_clears_the_complete_allocated_remote_xn_range(self): source = BUILDER_SOURCE.read_text(encoding="utf-8") + self.assertIn( + "remoteCcuBuffers.size() != allocation.channels.num", + source) + self.assertNotIn( + "remoteCcuBuffers.size() != allocation.remoteXn.num", + source) self.assertIn("result.remoteXnStartId = allocation.remoteXn.startId", source) self.assertIn("result.remoteXnCount = allocation.remoteXn.num", source) self.assertIn("snapshot.remoteXnStartId", source) @@ -2551,20 +2565,25 @@ def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): compact_body = " ".join(exchange_body.split()) self.assertIn("const size_t peerRouteCount = static_cast(rankSize - 1)", compact_body) - self.assertIn("const size_t syncRouteCount = allocation.remoteXn.num", compact_body) + self.assertIn("const size_t syncRouteCount = allocation.channels.num", compact_body) self.assertIn("allocation.remoteXn.num < routedPeerCount", compact_body) + self.assertIn("allocation.localWaitCke.num < syncRouteCount", compact_body) + self.assertIn("allocation.remoteNotifyCke.num < syncRouteCount", compact_body) self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize - 1)", compact_body) self.assertIn("peerCcuBuffersByRank", compact_body) self.assertIn("peerCcuBuffer.peerRank", compact_body) self.assertIn("invalid direct CCU peer buffer rank mapping", compact_body) self.assertIn("incomplete direct CCU peer buffer rank mapping", compact_body) self.assertIn("remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{})", compact_body) - self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) + self.assertIn("for (uint32_t syncIndex = 0; syncIndex < syncRouteCount; ++syncIndex)", compact_body) self.assertIn("const size_t peerBufferIndex = syncIndex / routesPerPeer", compact_body) self.assertIn( "(*remoteCcuBuffers)[routeIndex] = *peerCcuBuffersByRank[static_cast(peer)]", compact_body) self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", compact_body) + self.assertIn( + "SelectDirectCcuChannelBoundRemoteXnOffset( peerLocalIndex, routeWithinPeer)", + compact_body) self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) @@ -2592,9 +2611,17 @@ def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self): def test_peer_endpoints_keep_per_peer_resource_and_jetty_tokens(self): source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + self.assertIn("SelectEndpointRouteJettyCtxId(state->eidInfo.funcId, peerOrdinal", source) + self.assertIn( + "TILEXR_CCU_DIRECT_LOOP_JETTY_ID + jettyCtxId", + source) + self.assertIn("SelectEndpointRouteSqVa(localResourceWindow_, jettyCtxId)", source) + self.assertIn("state->resourceWindow.tokenId = localResourceWindow_.tokenId", source) + self.assertIn("state->resourceWindow.tokenValue = localResourceWindow_.tokenValue", source) + self.assertNotIn("mr.in.ub.tokenValue = state->resourceWindow.tokenValue", source) self.assertIn("offer.resourceTokenId = state.resourceWindow.tokenId", source) self.assertIn("offer.resourceTokenValue = state.resourceWindow.tokenValue", source) - self.assertIn("offer.jettyTokenValue = state.resourceWindow.tokenValue", source) + self.assertIn("offer.jettyTokenValue = state.jettyTokenValue", source) self.assertIn("importInfo.in.ub.tokenValue = peerOffer.jettyTokenValue", source) self.assertIn("state.route.memoryTokenValue = peerOffer.resourceTokenValue", source) diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py index baa5791f..28d1b96c 100644 --- a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py @@ -1294,6 +1294,27 @@ def test_direct_runtime_selects_tp_sl_before_creating_peer_qp(self): self.assertIn("qpAttr.ub.priority = state->mappedJettyPriority", create_body) self.assertNotIn("qpAttr.ub.priority = 2", create_body) + def test_direct_runtime_maps_ctp_jetty_priority_without_setting_tp_sl(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + select_start = source.index("int TileXRCcuDirectRuntime::SelectTpRouteForPeer(") + select_body = source[ + select_start: + source.index("int TileXRCcuDirectRuntime::QueryTpHandleForPeer(", select_start) + ] + compact_body = " ".join(select_body.split()) + + self.assertNotIn( + "if (tpType == TILEXR_CCU_HCCP_TP_TYPE_CTP) {", + select_body, + ) + self.assertIn("RaGetTpAttrAsync(", select_body) + self.assertIn("MapQosToTpAndSl(", select_body) + self.assertIn( + "if (tpType == TILEXR_CCU_HCCP_TP_TYPE_RTP) { TileXRCcuHccpTpAttr setAttr", + compact_body, + ) + self.assertIn("*mappedJettyPriority = mappedSl", select_body) + def test_peer_endpoint_route_uses_the_driver_returned_jetty_id(self): source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") prepare_body = source[ diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py index 6052edf4..5b74a0cb 100644 --- a/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_provider.py @@ -227,6 +227,68 @@ def test_provider_accepts_opaque_ra_custom_channel_c_abi_shape(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_provider_retries_transient_roce_eagain(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_ra_custom_channel_provider.h" + + #include + + using namespace TileXR; + + int g_calls = 0; + + int FakeRaCustomChannel( + TileXRCcuRaInfo, + TileXRCcuCustomChannelIn*, + TileXRCcuCustomChannelOut* out) + { + ++g_calls; + if (g_calls < 3) { + out->opRet = 99; + return 128101; + } + out->opRet = 0; + out->data.dataInfo.dataArray[0].baseinfo.msId = 0x55; + out->data.dataInfo.dataArray[0].baseinfo.missionKey = 0xabcdef01U; + out->data.dataInfo.dataArray[0].baseinfo.resourceAddr = 0x500000000ULL; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap0 = (1U << 24) | (2U << 16) | 31U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap1 = (15U << 16) | 7U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap2 = (3U << 16) | 5U; + out->data.dataInfo.dataArray[0].baseinfo.caps.cap3 = (9U << 16) | 1U; + return 0; + } + + int main() + { + TileXRCcuRaCustomChannelProvider provider; + TileXRCcuRaCustomChannelProviderReport providerReport; + if (provider.Init(3, FakeRaCustomChannel, &providerReport) != TILEXR_SUCCESS) { + return 1; + } + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + if (provider.CreateAdapter(&adapter, &adapterReport) != TILEXR_SUCCESS) { + return 2; + } + TileXRCcuBasicInfo basic; + if (adapter.GetBasicInfo(0, &basic, &adapterReport) != TILEXR_SUCCESS) { + std::cerr << adapterReport.message << "\n"; + return 3; + } + if (g_calls != 3 || basic.msId != 0x55 || basic.missionKey != 0xabcdef01U) { + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_provider_is_wired_and_keeps_hcomm_runtime_out_of_ccu_surface(self): cmake = COMM_CMAKE.read_text(encoding="utf-8") header = PROVIDER_HEADER.read_text(encoding="utf-8") diff --git a/tests/ccu/test_tilexr_ccu_resource_allocator.py b/tests/ccu/test_tilexr_ccu_resource_allocator.py index b9f2ca28..f7d60de2 100644 --- a/tests/ccu/test_tilexr_ccu_resource_allocator.py +++ b/tests/ccu/test_tilexr_ccu_resource_allocator.py @@ -143,7 +143,7 @@ def test_allocator_builds_complete_tilexr_owned_producer_plan(self): std::cerr << "first sync resource mismatch\n"; return 7; } - if (plan.syncResources[2].localXn != 1963 || plan.syncResources[2].remoteXn != 1977 || + if (plan.syncResources[2].localXn != 1963 || plan.syncResources[2].remoteXn != 1991 || plan.syncResources[2].notifyCke != 334 || plan.syncResources[2].channelId != 4) { std::cerr << "last sync resource mismatch\n"; return 8; @@ -163,6 +163,7 @@ def test_allocator_builds_complete_tilexr_owned_producer_plan(self): } if (allocation.receiptId == 0 || allocation.packageProvider != "tilexr-hcomm-derived-resource-allocator" || allocation.localXn.startId != 1961 || allocation.remoteXn.startId != 1975 || + allocation.remoteXn.num != 24 || allocation.localGsa.startId != 510 || allocation.localGsa.num != 1 || allocation.notifyCke.startId != 332 || allocation.channels.startId != 2 || allocation.channels.num != 3) { @@ -171,7 +172,7 @@ def test_allocator_builds_complete_tilexr_owned_producer_plan(self): } if (report.missionAllocated != 1 || report.localXnAllocated != 14 || report.localGsaAllocated != 1 || - report.remoteXnAllocated != 3 || report.notifyCkeAllocated != 3 || + report.remoteXnAllocated != 24 || report.notifyCkeAllocated != 3 || report.channelBindingsAllocated != 9 || report.repositoryAllocated != 156 || report.message != "ok") { std::cerr << "report mismatch\n"; @@ -391,6 +392,8 @@ def test_allocator_builds_pure_barrier_plan_without_sqe_load_task(self): request.syncResourceCount = 1; request.syncInstructionCount = 2; request.bindingsPerSyncResource = 1; + request.minimumLocalXnCount = 3; + request.minimumRemoteXnCount = 3; TileXRCcuResourceAllocator allocator; if (allocator.Init(spec) != TILEXR_SUCCESS) { @@ -416,15 +419,15 @@ def test_allocator_builds_pure_barrier_plan_without_sqe_load_task(self): std::cerr << "pure barrier sync task mismatch\n"; return 4; } - if (plan.kernelLocalXn.startId != 1 || plan.kernelLocalXn.num != 1 || - allocation.localXn.startId != 1 || allocation.localXn.num != 1 || - allocation.remoteXn.startId != 2 || allocation.remoteXn.num != 1 || + if (plan.kernelLocalXn.startId != 1 || plan.kernelLocalXn.num != 3 || + allocation.localXn.startId != 1 || allocation.localXn.num != 3 || + allocation.remoteXn.startId != 4 || allocation.remoteXn.num != 3 || allocation.repository.startId != 1 || allocation.repository.num != 2) { std::cerr << "pure barrier allocation mismatch\n"; return 5; } - if (report.localXnAllocated != 1 || - report.remoteXnAllocated != 1 || + if (report.localXnAllocated != 3 || + report.remoteXnAllocated != 3 || report.repositoryAllocated != 2) { std::cerr << "pure barrier report mismatch\n"; return 6; From 04649969101cf5cd166a44d40544066a8bace67e Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Tue, 28 Jul 2026 09:23:29 +0800 Subject: [PATCH 38/41] docs: remove internal CCU notes from PR --- docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md | 185 ----------------- ...26-07-10-ccu-signal-wait-implementation.md | 126 ------------ .../2026-07-20-ccu-alltoall-4p-mesh-design.md | 190 ------------------ 3 files changed, 501 deletions(-) delete mode 100644 docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md delete mode 100644 docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md delete mode 100644 docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md diff --git a/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md b/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md deleted file mode 100644 index 23f499ed..00000000 --- a/docs/CCU_DIRECT_ALLTOALL_CHANGE_RECORD.md +++ /dev/null @@ -1,185 +0,0 @@ -# CCU Direct AllToAll Change Record - -## Scope - -This change records the TileXR-owned Direct CCU alltoall work on the -`direct-ccu-rebased` branch. The implementation keeps HCCL and hcomm as -reference-only inputs: TileXR does not include HCCL/hcomm headers, call their -private CCU interfaces, or link new HCCL/hcomm dependencies. - -The validated target is the 2-rank P2P alltoall smoke path with a 2 MB payload. - -## Code Changes - -### AllToAll program builder - -- Added a TileXR-owned 2-rank Direct CCU alltoall program flow. -- Uses fixed 4 KB memory slices, `memSlicePerLoop <= 8`, 32 KB blocks, and - 64 blocks for the 2 MB smoke scale. -- Encodes the HCCL-style phase structure in TileXR microcode: - - PreSync: remote notification and local wait before data movement. - - Copy: CCU memory copy blocks over the existing P2P copy route. - - PostSync: completion notification and optional peer wait. -- Split synchronization masks for PreSync and PostSync so the phases no longer - reuse one ambiguous CKE bit. -- Added instruction-count reporting for pre-sync, copy, post-sync, and finish - sections. - -### Planner and orchestrator - -- Added planner/orchestrator entry points for preparing a 2-rank alltoall - launch package. -- Added direct resource configuration for alltoall sync resources, channels, - XNs, CKEs, GSA addresses, and remote endpoint routes. -- Kept the production backend guarded: Direct CCU prepare/submit helpers remain - test-only under `TILEXR_CCU_TESTING`. - -### Smoke probe and runner - -- Added alltoall smoke switches: - - `TILEXR_CCU_DIRECT_SMOKE_ALLTOALL` - - `TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION` - - `TILEXR_CCU_ALLTOALL_BYTES` - - `TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP` -- Added `TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING` for a minimal SyncXn route - sanity check. -- Added bounded stream synchronization through - `aclrtSynchronizeStreamWithTimeout` for signal/wait, SyncXn ping, and - alltoall submit paths. -- Fixed multi-phase host submit coordination by making the submit-ready and - submit-done files phase-scoped: - - `rank0.phase0.ready` - - `rank0.phase0.done` - - `rank1.phase1.ready` - - `rank1.phase1.done` -- Extended the P2P copy direction parser to accept `local_to_remote`, - `LocalToRemote`, and `1`. - -### Tests - -- Added and updated unit tests for: - - alltoall program instruction layout and resource usage, - - planner/orchestrator alltoall prepare paths, - - smoke runner environment forwarding, - - phase-scoped collective submit gates, - - bounded synchronization in smoke modes, - - lower-layer resource planning expectations. - -## Validated Commands - -Local structural tests: - -```bash -python -m unittest \ - tests.ccu.test_tilexr_ccu_alltoall_program \ - tests.ccu.test_tilexr_ccu_direct_orchestrator \ - tests.ccu.test_tilexr_ccu_direct_smoke_probe \ - tests.ccu.test_tilexr_ccu_lower_layer_plan_builder -``` - -Result: - -```text -Ran 105 tests in 3.728s -OK (skipped=52) -``` - -Remote build and dependency guard on `141.62.24.62`: - -```bash -cd /home/tileXR -source scripts/common_env.sh -timeout 180s cmake -S . -B build_ccu_direct \ - -DTILEXR_CCU_TESTING=1 \ - -DCMAKE_INSTALL_PREFIX=/home/tileXR/install -timeout 240s cmake --build build_ccu_direct --target tile-comm -j2 -bash tests/ccu/check_tile_comm_no_hcomm_deps.sh \ - build_ccu_direct/src/comm/libtile-comm.so -``` - -Result: build succeeded and the dependency guard reported no hcomm/HCCL private -CCU dependency or symbol reference. - -Remote P2P positive-control smoke on devices `6,7`: - -```bash -env ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 \ - TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ - TILEXR_TILE_COMM_LIB=/home/tileXR/build_ccu_direct/src/comm/libtile-comm.so \ - TILEXR_CCU_DIRECT_TRACE=1 \ - TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=1 \ - TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 \ - TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 \ - TILEXR_CCU_SMOKE_DEVICES=6,7 \ - TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY=1 \ - TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=remote_to_local \ - TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ - TILEXR_CCU_SMOKE_WORK_DIR=build_ccu_direct/diag62_p2p_r2l_67_current \ - TILEXR_CCU_SMOKE_TIMEOUT=120 \ - TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS=30000 \ - TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ - TILEXR_CCU_DIRECT_READBACK_INSTRUCTIONS=1 \ - timeout 150s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh -``` - -Result: rank0 and rank1 exited with status 0. - -Remote 2-rank 2 MB alltoall smoke on devices `6,7`: - -```bash -env ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 \ - TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ - TILEXR_TILE_COMM_LIB=/home/tileXR/build_ccu_direct/src/comm/libtile-comm.so \ - TILEXR_CCU_DIRECT_TRACE=1 \ - TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE=1 \ - TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK0=3 \ - TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_RANK1=3 \ - TILEXR_CCU_SMOKE_DEVICES=6,7 \ - TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ - TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ - TILEXR_CCU_ALLTOALL_BYTES=2097152 \ - TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 \ - TILEXR_CCU_SMOKE_WORK_DIR=build_ccu_direct/diag62_alltoall_host_phased_67_current \ - TILEXR_CCU_SMOKE_TIMEOUT=180 \ - TILEXR_CCU_DIRECT_SMOKE_READY_TIMEOUT_MS=30000 \ - TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ - TILEXR_CCU_DIRECT_READBACK_INSTRUCTIONS=1 \ - timeout 220s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh -``` - -Key log evidence: - -```text -tilexr_ccu_alltoall config rank=0 peer=1 bytes=2097152 memSlicePerLoop=8 blockCount=64 hostPhases=2 -tilexr_ccu_direct_smoke collectiveSubmitReady rank=0 phase=0 localReady=1 allRanksReady=1 -tilexr_ccu_direct_smoke collectiveSubmitDone rank=0 phase=0 localResult=0 allRanksDone=1 allRanksSucceeded=1 -tilexr_ccu_direct_smoke collectiveSubmitReady rank=0 phase=1 localReady=1 allRanksReady=1 -tilexr_ccu_direct_smoke collectiveSubmitDone rank=0 phase=1 localResult=0 allRanksDone=1 allRanksSucceeded=1 -tilexr_ccu_alltoall result passed=1 rank=0 ret=0 readRet=0 mismatches=0 -tilexr_ccu_alltoall config rank=1 peer=0 bytes=2097152 memSlicePerLoop=8 blockCount=64 hostPhases=2 -tilexr_ccu_direct_smoke collectiveSubmitReady rank=1 phase=0 localReady=1 allRanksReady=1 -tilexr_ccu_direct_smoke collectiveSubmitDone rank=1 phase=0 localResult=0 allRanksDone=1 allRanksSucceeded=1 -tilexr_ccu_direct_smoke collectiveSubmitReady rank=1 phase=1 localReady=1 allRanksReady=1 -tilexr_ccu_direct_smoke collectiveSubmitDone rank=1 phase=1 localResult=0 allRanksDone=1 allRanksSucceeded=1 -tilexr_ccu_alltoall result passed=1 rank=1 ret=0 readRet=0 mismatches=0 -``` - -## Environment Notes - -- File transfer to the server used mutagen, as required by the project - instructions. -- On `141.62.24.62`, the default system CANN `libra.so` does not export - `RaCustomChannel`. Hardware CCU smoke tests must set: - -```bash -ASCEND_HOME_PATH=/home/Hccl_QQTest/Ascend/cann-9.1.0 -``` - -- Hardware test commands use an outer `timeout` and also pass - `TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT` for stream-level timeout control. - -## Current Limitations - -- The validated alltoall path is 2-rank P2P with host-phased submission. -- The formal LoopGroup version is not claimed as validated by this record. -- The validated route uses devices `6,7` on `141.62.24.62`. diff --git a/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md b/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md deleted file mode 100644 index 81f8b54a..00000000 --- a/docs/superpowers/plans/2026-07-10-ccu-signal-wait-implementation.md +++ /dev/null @@ -1,126 +0,0 @@ -# CCU Signal/Wait Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Build internal two-rank Direct CCU signal/wait synchronization in `TileXRCcuBackend`, then compose a two-rank barrier from the same mechanism. - -**Architecture:** Add internal request/plan types to the CCU backend boundary and route preparation through the existing Direct CCU install pipeline. Generate role-specific CCU synchronization programs, reuse lower-layer resource proof and repository/mission install, and validate with unit tests plus the two-rank smoke probe. - -**Tech Stack:** C++14, CANN ACL/runtime, TileXR Direct CCU modules under `src/comm/ccu`, Python `unittest`, shell smoke runner. - -## Global Constraints - -- Do not expose public C API in `src/include/tilexr_api.h`. -- Do not link or include hcomm/HCCL private CCU producer APIs. -- Reuse existing Direct CCU runtime lifecycle, resource-window registration, peer allgather, lower-layer install, repository install, mission install, and `rtCCULaunch`. -- Implement single-direction signal/wait first, then two-direction barrier. -- Hardware validation must prove device-side synchronization with delayed signal timing, not host marker success. - ---- - -### Task 1: Internal Backend Types and Submission Surface - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_backend.h` -- Modify: `src/comm/ccu/tilexr_ccu_backend.cpp` -- Test: `tests/ccu/test_tilexr_ccu_backend_boundary.py` - -**Interfaces:** -- Produces: `enum class TileXRCcuSignalWaitRole { Signal, Wait, SignalAndWait };` -- Produces: `struct TileXRCcuSignalWaitRequest` -- Produces: `struct TileXRCcuSignalWaitPlan` -- Produces: `int TileXRCcuBackend::PrepareSignalWait(const TileXRCcuSignalWaitRequest&, TileXRCcuSignalWaitPlan*)` -- Produces: `int TileXRCcuBackend::SubmitSignalWait(const TileXRCcuSignalWaitPlan&, aclrtStream, TileXRCcuDirectSubmitReport*)` - -- [ ] Add failing boundary tests that assert internal signal/wait types exist in `tilexr_ccu_backend.h` and do not appear in `src/include/tilexr_api.h`. -- [ ] Add backend declarations and simple forwarding methods. -- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_backend_boundary`. - -### Task 2: Signal/Wait Program Builder - -**Files:** -- Create: `src/comm/ccu/tilexr_ccu_signal_wait_program.h` -- Create: `src/comm/ccu/tilexr_ccu_signal_wait_program.cpp` -- Modify: `src/comm/CMakeLists.txt` -- Test: `tests/ccu/test_tilexr_ccu_signal_wait_program.py` - -**Interfaces:** -- Consumes: `TileXRCcuSignalWaitRole` -- Produces: `struct TileXRCcuSignalWaitProgramSpec` -- Produces: `int TileXRCcuBuildSignalWaitProgram(const TileXRCcuSignalWaitProgramSpec&, std::vector*, TileXRCcuBarrierProgramReport*)` - -- [ ] Add tests for signal-only, wait-only, signal-and-wait, and invalid resource cases. -- [ ] Implement the builder using existing microcode encoders, not duplicate opcode packing. -- [ ] Add the new source file to the `tile-comm` target. -- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_signal_wait_program`. - -### Task 3: Planner Prepare Path - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.h` -- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` -- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.h` -- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` -- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` - -**Interfaces:** -- Consumes: `TileXRCcuSignalWaitRequest` -- Produces: `int TileXRCcuCollectivePlanner::PrepareSignalWait(...)` -- Produces: `int TileXRCcuRunDirectSignalWaitInstallAttempt(...)` - -- [ ] Add orchestrator tests that build signal/wait launch packages and verify submit tasks are produced when install evidence is satisfied. -- [ ] Implement direct signal/wait install attempt by reusing `RunDirectInstallAttemptImpl` structure and selecting the signal/wait program builder. -- [ ] Add planner method that validates rank size 2, peer rank, role, runtime availability, and basic info. -- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator`. - -### Task 4: Backend Wiring - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_backend.cpp` -- Test: `tests/ccu/test_tilexr_ccu_backend_boundary.py` - -**Interfaces:** -- Consumes: `TileXRCcuCollectivePlanner::PrepareSignalWait` -- Produces: working `TileXRCcuBackend::PrepareSignalWait` and `SubmitSignalWait` - -- [ ] Add tests for null plan, unavailable runtime, not-ready submit, and null stream. -- [ ] Wire backend preparation through planner and submission through `TileXRCcuSubmitPreparedTasks`. -- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_backend_boundary`. - -### Task 5: Smoke Probe and Runner - -**Files:** -- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` -- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` -- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` -- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` - -**Interfaces:** -- Consumes: `TileXRCcuBackend::PrepareSignalWait` -- Produces env-gated smoke modes: - - `TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=1` - - `TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=` - - `TILEXR_CCU_DIRECT_SMOKE_BARRIER=1` - -- [ ] Add source tests for signal/wait mode selection and inactive/active timing behavior. -- [ ] Add runner env propagation and log assertions for signal/wait and barrier. -- [ ] Implement smoke probe path using internal backend methods. -- [ ] Run `python3 -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner`. - -### Task 6: Local and Remote Validation - -**Files:** -- No new source files; run validation commands. - -**Interfaces:** -- Consumes: all previous tasks. -- Produces: verified Direct CCU signal/wait and barrier behavior. - -- [ ] Run focused CCU unit tests: - `python3 -m unittest tests.ccu.test_tilexr_ccu_signal_wait_program tests.ccu.test_tilexr_ccu_direct_orchestrator tests.ccu.test_tilexr_ccu_backend_boundary tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner` -- [ ] Build `tile-comm` on the NPU server. -- [ ] Run no-hcomm dependency guard. -- [ ] Run two-card `rank0 -> rank1` signal/wait smoke with delayed signal rank. -- [ ] Run two-card `rank1 -> rank0` signal/wait smoke with delayed signal rank. -- [ ] Run two-card barrier smoke. - diff --git a/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md b/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md deleted file mode 100644 index d27fdcca..00000000 --- a/docs/superpowers/specs/2026-07-20-ccu-alltoall-4p-mesh-design.md +++ /dev/null @@ -1,190 +0,0 @@ -# CCU AlltoAll 4P Mesh Design - -## Goal - -Extend the validated direct CCU AlltoAll path from two ranks to a true four-rank -Mesh1D collective. Each rank uses one communicator, one installed mission, and -one stable set of QP, jetty, CKE, XN, channel, and registered-memory resources. -Validation first runs one submission and then ten consecutive submissions with -the same prepared resources. - -The hardware target is `141.61.50.31` on NPU devices `4,5,6,7`. Each rank owns -an 8 MiB send buffer and an 8 MiB receive buffer. Each peer chunk is 2 MiB. - -## Collective Semantics - -Buffers use rank-major chunk layout: - -```text -send[targetRank][2 MiB] -recv[sourceRank][2 MiB] -``` - -For local rank `r`, the collective must produce: - -```text -recv_r[s] == send_s[r] -``` - -for every source rank `s` in `0..3`. The test pattern encodes source rank, -target rank, loop index, and byte offset. This detects incorrect source, -destination, generation, chunk placement, and partial-copy behavior. - -## Architecture - -Add a general four-rank Mesh spec and program instead of changing the meaning -of `TileXRCcuDirectAllToAll2RankSpec`. The existing 2P interfaces remain as a -regression baseline. Shared block-copy, marker, and diagnostic helpers may be -factored only where this does not change 2P behavior. - -Each rank has three peer descriptors sorted by peer rank. A descriptor contains -the imported remote destination address and token, peer rank, loop-marker XNs, -and the copy, PreSync, and token route resources. - -The fixed sync-resource layout is: - -```text -peer[0]: copy=0, pre=1, token=2 -peer[1]: copy=3, pre=4, token=5 -peer[2]: copy=6, pre=7, token=8 -``` - -The planner performs one endpoint AllGather, validates all four endpoints, -imports the other three destination buffers, and installs route-specific -memory overrides for all nine sync routes. The lower-layer override state must -therefore become a collection indexed by sync-route index instead of a single -optional override. - -The program uses one mission. It publishes marker, output address, and token to -all three peers before waiting on any peer, then waits for every peer's PreSync -mask `0x7`. This all-post-before-wait ordering prevents rank-dependent peer -iteration from creating a wait cycle. - -After PreSync, the mission copies the three remote chunks to -`peer.recv[localRank]` and copies the self chunk from `send[localRank]` to -`recv[localRank]`. It waits for all local and remote completion signals before -returning. - -## CCU Self Copy - -The self chunk is copied by CCU, not by ACL. HCCL/hcomm reference code maps -`GroupCopy` through `CcuRepLocCpy` to the hardware -`TransLocMemToLocMemInstr` instruction. TileXR will add an independent -`TileXRCcuEncodeTransLocMemToLocMem` encoder using the verified instruction -field layout without including, linking, or calling private HCCL/hcomm code. - -The AlltoAll program builder will load self source and destination addresses, -their tokens, and length, issue local-to-local transfers, and wait for local -completion. Unit tests must decode every relevant instruction field and verify -the self offsets for local ranks zero through three. - -## Instruction And Resource Capacity - -Each 2 MiB remote chunk retains the validated 64 blocks by 7 instructions copy -shape. Three remote chunks plus PreSync are expected to require about 1365 -instructions, with additional instructions for CCU self copy and final waits. - -Prepare must calculate the exact program size before installation and compare -the requested repository range with device basic-info capacity. Insufficient -instruction, channel, XN, CKE, GSA, or route resources must fail with requested -and available counts. The implementation must not truncate a program or reuse -one peer's resource IDs for another peer. - -## Repeated Submission Protocol - -Allocation, endpoint exchange, remote import, route construction, -registration, mission installation, prepared-task creation, and stream creation -all occur once outside the loop. - -For each loop index: - -1. Fill all four send chunks with source/target/loop-specific patterns. -2. Reset the complete 8 MiB receive buffer. -3. Update prepared-task argument zero with a rank-and-loop marker. -4. Enter the four-rank `ready.phaseN` gate. -5. Submit the same prepared task and synchronize the same stream. -6. Read and validate the current marker from all three peers. -7. Read and compare the complete 8 MiB receive buffer. -8. Enter the four-rank `done.phaseN` gate with the local validation result. - -The marker contains a fixed magic prefix, sender rank, and loop index. CKE bits -remain presence flags and are cleared by their waits; marker XNs carry the -generation identity needed to reject stale synchronization. - -If any rank reports an error, all ranks stop before the next loop. Diagnostics -must include local rank, loop index, peer rank, route and channel IDs, marker -XN/CKE values, mission current instruction, local-copy completion, and the first -mismatch's source rank, chunk offset, global offset, expected byte, and observed -byte. - -## Runner - -The smoke runner must accept rank size four and a four-device list. It will -construct rank-specific environment arrays for ranks zero through three, -launch four processes, track four PIDs and statuses, and validate four logs. -Hard-coded rank0/rank1 loops and result thresholds must become rank-size-driven. - -Rank-specific endpoint and resource-window fields, including EID index, must be -forwarded for all four ranks. Four-rank AlltoAll defaults use EID index 3 for -each rank unless explicitly overridden. - -The runner must require the exact number of successful results and marker -checks for the requested loop count. Seeing one successful line is not -sufficient. - -## Testing - -Automated coverage includes: - -- microcode encoding and decoding for `TransLocMemToLocMem`; -- rank 0..3 program generation, peer ordering, all-post-before-wait ordering, - self offsets, remote offsets, completion waits, and instruction counts; -- planner endpoint exchange, three remote imports, nine route mappings, and - resource-exhaustion diagnostics; -- smoke pattern/reset, three peer-marker checks, full 8 MiB mismatch reporting, - and prepare-before-loop ordering; -- dynamic four-process runner launch, environment forwarding, status handling, - exact result counts, and dry-run output; -- all existing 2P program, planner, smoke, runner, and loop-reuse regression - tests. - -## Hardware Validation - -Before every hardware run, query `npu-smi` and apply the repository busy guard -to devices `4,5,6,7`. If any selected device is busy, poll every 30 seconds and -do not terminate or bypass existing jobs. - -Run these stages in order: - -1. Build `tile-comm` and the four-rank smoke probe. -2. Run 4P with loop count one in a fresh work directory. -3. Run 4P with loop count ten in another fresh work directory. -4. Re-run the existing 2P loop-count-ten test on its validated device pair. - -The 4P loop-one run requires four successful rank results, twelve matching peer -markers, and zero mismatches over every 8 MiB receive buffer. - -The 4P loop-ten run requires forty successful rank results and 120 matching -peer markers. Each rank must retain the same mission ID, key, instruction start, -instruction count, task count, and installed resource IDs across all ten loops. -Every loop must report zero mismatches. - -## Deployment And Cleanup - -Use Mutagen as the preferred source synchronization mechanism and reuse the -existing passwordless SSH configuration for `141.61.50.31`. Do not install a -new key or modify the remote `authorized_keys` file. Confirm the remote account -and repository path before creating a sync session or transferring files. If -the installed Mutagen version cannot use the existing SSH configuration, record -that incompatibility and use `scp` for only the changed files. - -Do not modify unrelated untracked files, SSH configuration, or remote -authentication state, and do not stop unrelated remote workloads. - -## Non-Goals - -- Supporting arbitrary rank sizes beyond four in this validation cycle. -- Adding AlltoAllV variable counts or displacements. -- Matching multi-jetty performance tuning from HCCL. -- Replacing TileXR's runtime with HCCL/hcomm private APIs. -- Claiming network-copy concurrency or performance until profiling proves it. From ce9951b5ca9d66855cb4fac6df25de8cef80d026 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Tue, 28 Jul 2026 09:29:17 +0800 Subject: [PATCH 39/41] docs: remove CCU implementation plans from PR --- .../plans/2026-07-20-ccu-alltoall-4p-mesh.md | 626 ------------------ .../2026-07-20-ccu-alltoall-loop-reuse.md | 112 ---- .../2026-07-10-ccu-signal-wait-design.md | 217 ------ ...26-07-20-ccu-alltoall-loop-reuse-design.md | 42 -- 4 files changed, 997 deletions(-) delete mode 100644 docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md delete mode 100644 docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md delete mode 100644 docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md delete mode 100644 docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md diff --git a/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md b/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md deleted file mode 100644 index d1e0decc..00000000 --- a/docs/superpowers/plans/2026-07-20-ccu-alltoall-4p-mesh.md +++ /dev/null @@ -1,626 +0,0 @@ -# CCU AlltoAll 4P Mesh Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add and validate one true four-rank direct-CCU Mesh1D AlltoAll mission that moves four 2 MiB chunks per rank, including the self chunk, and supports ten submissions with unchanged communicator, mission, and transport resources. - -**Architecture:** Keep the validated two-rank API unchanged and add a four-rank Mesh program, orchestrator entry point, and planner entry point. Each rank owns three peer descriptors sorted by peer rank and nine sync routes (`copy`, `pre`, `token` per peer); the mission publishes all three peer handshakes before any wait, then performs three remote CCU copies plus one CCU local copy and waits for all completions. Prepare/import/install happens once, while each loop rewrites data and the SQE marker and uses phase-specific four-rank ready/done gates. - -**Tech Stack:** C++14, TileXR direct CCU runtime and microcode encoders, Python `unittest` source/compile probes, Bash hardware runner, CMake, ACL runtime, Mutagen, SSH. - ---- - -## File Map - -- `src/comm/ccu/tilexr_ccu_microcode.{h,cpp}`: encode the hardware local-memory-to-local-memory transfer instruction. -- `src/comm/ccu/tilexr_ccu_alltoall_program.{h,cpp}`: describe and build one rank's four-rank Mesh mission. -- `src/comm/ccu/tilexr_ccu_direct_orchestrator.{h,cpp}`: allocate nine resources, build the Mesh program, check capacity, and package one prepared mission. -- `src/comm/ccu/tilexr_ccu_collective_planner.{h,cpp}`: AllGather four endpoints, import three destinations, map nine routes, and invoke the Mesh orchestrator. -- `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp`: prepare once, submit repeatedly, verify three peer markers and all 8 MiB of receive data. -- `tests/ccu/run_tilexr_ccu_direct_smoke.sh`: launch and evaluate a rank-size-driven process set. -- `tests/ccu/test_tilexr_ccu_*.py`: unit, source-contract, compile-probe, and runner tests for every layer. - -### Task 1: Encode CCU Local-To-Local Transfer - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_microcode.h` -- Modify: `src/comm/ccu/tilexr_ccu_microcode.cpp` -- Test: `tests/ccu/test_tilexr_ccu_microcode.py` - -- [ ] **Step 1: Write the failing encoder tests** - -Add a compile-and-run probe that calls the new API with distinct field values and decodes all four words: - -```cpp -TileXR::TileXRCcuMemTransferSpec spec; -spec.localGsa = 0x101; -spec.localXn = 0x102; -spec.remoteGsa = 0x201; // destination GSA for the local-copy opcode -spec.remoteXn = 0x202; // destination XN for the local-copy opcode -spec.lengthXn = 0x103; -spec.channelId = 0x104; -spec.setCkeId = 0x105; -spec.setCkeMask = 0x7; -spec.waitCkeId = 0x106; -spec.waitCkeMask = 0x8; -TileXR::TileXRCcuInstr instr; -assert(TileXR::TileXRCcuEncodeTransLocMemToLocMem(spec, &instr) == TileXR::TILEXR_SUCCESS); -assert(slot(instr.words[0], 0) == 0x100a); -assert(slot(instr.words[0], 1) == spec.remoteGsa); -assert(slot(instr.words[0], 2) == spec.remoteXn); -assert(slot(instr.words[0], 3) == spec.localGsa); -assert(slot(instr.words[1], 0) == spec.localXn); -assert(slot(instr.words[1], 1) == spec.lengthXn); -assert(slot(instr.words[1], 2) == spec.channelId); -assert(slot(instr.words[3], 0) == spec.setCkeId); -assert(slot(instr.words[3], 1) == spec.setCkeMask); -assert(slot(instr.words[3], 2) == spec.waitCkeId); -assert(slot(instr.words[3], 3) == spec.waitCkeMask); -``` - -Also assert null output, zero GSA/XN/length/channel, half-specified CKE pairs, and invalid reduce fields return `TILEXR_ERROR_PARA_CHECK_FAIL`. - -- [ ] **Step 2: Run the test to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_microcode -v` - -Expected: FAIL because `TileXRCcuEncodeTransLocMemToLocMem` is undeclared/undefined. - -- [ ] **Step 3: Add the public declaration and minimal encoder** - -Add to the header: - -```cpp -int TileXRCcuEncodeTransLocMemToLocMem( - const TileXRCcuMemTransferSpec& spec, - TileXRCcuInstr* instr); -``` - -Add opcode `0x100a` beside the two existing transfer opcodes and encode using the same validation and flag packing: - -```cpp -int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) -{ - if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - instr->words[0] = PackSlots(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER, - spec.remoteGsa, spec.remoteXn, spec.localGsa); - instr->words[1] = PackSlots(spec.localXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); - instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); - instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); - return TILEXR_SUCCESS; -} -``` - -- [ ] **Step 4: Run the focused and full microcode tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_microcode -v` - -Expected: PASS, including the existing remote/local transfer encoders. - -- [ ] **Step 5: Commit** - -```bash -git add src/comm/ccu/tilexr_ccu_microcode.h src/comm/ccu/tilexr_ccu_microcode.cpp tests/ccu/test_tilexr_ccu_microcode.py -git commit -m "feat(ccu): encode local memory transfers" -``` - -### Task 2: Build The Four-Rank Mesh Mission - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.h` -- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.cpp` -- Test: `tests/ccu/test_tilexr_ccu_alltoall_program.py` - -- [ ] **Step 1: Add failing rank-parameterized Mesh tests** - -Define a fixture for each `localRank` in `0..3` with peers in intentionally unsorted input order. Decode the generated instructions and assert: - -```python -self.assertEqual([p for p in range(4) if p != local_rank], report.peerRanks) -self.assertEqual(3, report.peerCount) -self.assertEqual(9, report.syncResourceCount) -self.assertEqual(64, report.remoteBlockCountPerPeer) -self.assertEqual(64, report.selfBlockCount) -self.assertLess(last_peer_post_index, first_peer_wait_index) -self.assertEqual(local_rank * 2 * 1024 * 1024, decoded_self_source_offset) -self.assertEqual(local_rank * 2 * 1024 * 1024, decoded_self_destination_offset) -``` - -For every peer `p`, assert remote source offset is `localRank * chunkBytes`, remote destination offset is `localRank * chunkBytes`, the peer route IDs are `3*ordinal+{0,1,2}`, and the PreSync wait mask is `0x7`. Add failures for rank size other than four, duplicate/missing/self peer, zero token/address, non-2-MiB-aligned size, duplicate resource IDs, and insufficient output pointer/report. - -- [ ] **Step 2: Run the test to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_alltoall_program -v` - -Expected: FAIL because the Mesh types and builder do not exist. - -- [ ] **Step 3: Add explicit Mesh types** - -Add these interfaces without changing `TileXRCcuAllToAll2RankProgramSpec`: - -```cpp -struct TileXRCcuAllToAllMeshPeerSpec { - uint32_t peerRank = 0; - uint64_t remoteRecvAddr = 0; - uint64_t remoteRecvToken = 0; - uint16_t copyResourceIndex = 0; - uint16_t preSyncResourceIndex = 0; - uint16_t tokenResourceIndex = 0; -}; - -struct TileXRCcuAllToAllMeshProgramSpec { - uint32_t rankSize = 4; - uint32_t localRank = 0; - uint64_t localSendAddr = 0; - uint64_t localSendToken = 0; - uint64_t localRecvAddr = 0; - uint64_t localRecvToken = 0; - uint64_t chunkBytes = 2ULL * 1024ULL * 1024ULL; - uint16_t markerArgIndex = 0; - uint16_t localGsa = 0; - uint16_t selfDestinationGsa = 0; - uint16_t localXn = 0; - uint16_t selfDestinationXn = 0; - uint16_t lengthXn = 0; - uint16_t selfChannelId = 0; - uint16_t selfCompletionCke = 0; - std::vector peers; -}; - -int TileXRCcuBuildAllToAllMeshProgram( - const TileXRCcuAllToAllMeshProgramSpec& spec, - std::vector* program, - TileXRCcuAllToAllProgramReport* report); -``` - -Extend the report with peer ranks, peer/resource counts, self/remote block counts, and local/remote completion counts. - -- [ ] **Step 4: Implement deterministic validation and emission** - -Implement this exact high-level ordering using the existing `Append*` helpers and the Task 1 local-copy encoder: - -```cpp -ValidateFourRankShapeAndUniqueResources(spec); -auto peers = spec.peers; -std::sort(peers.begin(), peers.end(), ByPeerRank); -LoadMarkerFromSqeArg(spec.markerArgIndex); -for (const auto& peer : peers) { - PostMarker(peer); - PostRemoteDestinationAddress(peer); - PostRemoteDestinationToken(peer); -} -for (const auto& peer : peers) { - WaitForPeerPreSync(peer, 0x7); -} -for (const auto& peer : peers) { - EmitRemoteBlocks(peer, spec.localRank * spec.chunkBytes, spec.chunkBytes); -} -EmitLocalBlocks(spec.localRank * spec.chunkBytes, spec.chunkBytes); -for (const auto& peer : peers) { - WaitForRemoteCopyCompletion(peer); -} -WaitForSelfCopyCompletion(spec.selfCompletionCke); -EmitFinish(); -``` - -Do not reuse one peer's channel/XN/CKE IDs, do not emit a wait inside the publish loop, and derive `report.totalInstructionCount` from `program->size()`. - -- [ ] **Step 5: Run Mesh and 2P regression tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_alltoall_program -v` - -Expected: PASS for ranks 0..3 and all existing 2P cases. - -- [ ] **Step 6: Commit** - -```bash -git add src/comm/ccu/tilexr_ccu_alltoall_program.h src/comm/ccu/tilexr_ccu_alltoall_program.cpp tests/ccu/test_tilexr_ccu_alltoall_program.py -git commit -m "feat(ccu): build four-rank all-to-all mesh mission" -``` - -### Task 3: Allocate And Package Mesh Resources - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.h` -- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` -- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` - -- [ ] **Step 1: Write failing resource/package tests** - -Add offline tests that provide a four-peer endpoint spec and assert one mission, one task, nine sync resources, peer-ordinal route mapping `0/1/2`, `3/4/5`, `6/7/8`, and exact program-sized repository installation. Mutate each basic-info capacity (`mission`, `instruction`, `channel`, `xn`, `cke`, `gsa`) below the requested count and require the report message to contain both `requested=` and `available=`. - -- [ ] **Step 2: Run the test to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator -v` - -Expected: FAIL because `TileXRCcuRunDirectAllToAllMeshInstallAttempt` is missing. - -- [ ] **Step 3: Add the orchestrator API** - -```cpp -struct TileXRCcuDirectAllToAllMeshPeerSpec { - uint32_t peerRank = 0; - uint64_t remoteRecvAddr = 0; - uint64_t remoteRecvToken = 0; -}; - -struct TileXRCcuDirectAllToAllMeshSpec { - uint32_t rankSize = 4; - uint32_t localRank = 0; - uint64_t localSendAddr = 0; - uint64_t localSendToken = 0; - uint64_t localRecvAddr = 0; - uint64_t localRecvToken = 0; - uint64_t chunkBytes = 2ULL * 1024ULL * 1024ULL; - std::vector peers; -}; - -int TileXRCcuRunDirectAllToAllMeshInstallAttempt( - const TileXRCcuDirectInstallOptions& options, - const TileXRCcuDirectAllToAllMeshSpec& alltoall, - TileXRCcuDirectInstallAttempt* attempt, - TileXRCcuDirectInstallReport* report); -``` - -- [ ] **Step 4: Build once to size, validate capacity, then install once** - -Create a resource request with `syncResourceCount=9`, allocate distinct copy/pre/token resources per peer, reserve self-copy GSA/XN/channel/CKE, build the Mesh program into a temporary vector, and use its exact size before repository installation: - -```cpp -const uint32_t requestedInstructions = static_cast(program.size()); -const uint32_t availableInstructions = RangeAvailableCount( - attempt->specInfo.instructionNum, options.instructionStartId); -if (requestedInstructions > availableInstructions) { - return CapacityError("instruction", requestedInstructions, - availableInstructions, report); -} -attempt->package.missions.resize(1); -attempt->submitTasks.resize(1); -attempt->submitTasks[0].argSize = std::max(1, options.sqeArgCount); -``` - -Preserve the existing 2P path byte-for-byte except for shared pure helpers extracted to avoid duplication. - -- [ ] **Step 5: Run orchestrator and AlltoAll tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_orchestrator tests.ccu.test_tilexr_ccu_alltoall_program -v` - -Expected: PASS; invalid capacities identify the exhausted resource and exact counts. - -- [ ] **Step 6: Commit** - -```bash -git add src/comm/ccu/tilexr_ccu_direct_orchestrator.h src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp tests/ccu/test_tilexr_ccu_direct_orchestrator.py -git commit -m "feat(ccu): package four-rank all-to-all resources" -``` - -### Task 4: Gather Four Endpoints And Import Three Destinations - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.h` -- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` -- Test: `tests/ccu/test_tilexr_ccu_collective_planner.py` -- Test: `tests/ccu/test_tilexr_ccu_direct_backend.py` - -- [ ] **Step 1: Write failing planner tests** - -Use the existing fake session/backend to return four gathered `DirectCcuMemoryCopyEndpoint` values. Assert exactly one AllGather, exactly three imports in peer-rank order, nine lower-layer routes, three distinct imported `targetSegVa` values repeated only across that peer's three routes, and no import for the local endpoint. Add invalid endpoint tests for wrong rank size, duplicate endpoint rank, `valid=0`, wrong byte count, missing token, and failed import. - -- [ ] **Step 2: Run planner tests to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_collective_planner tests.ccu.test_tilexr_ccu_direct_backend -v` - -Expected: FAIL because the Mesh planner API and multi-override storage are absent. - -- [ ] **Step 3: Add the Mesh planner entry point** - -```cpp -int PrepareDirectCcuAllToAllMeshInstallAttempt( - TileXRCcuRuntimeSession& session, - const TileXRCcuDirectInstallOptions& options, - uint64_t localSourceAddr, - uint64_t localDestinationAddr, - uint64_t chunkBytes, - TileXRCcuDirectInstallAttempt* attempt, - TileXRCcuDirectInstallReport* report); -``` - -Require `session.RankSize() == 4` and register `4 * chunkBytes` for both local buffers while retaining `chunkBytes` as the per-peer transfer size. - -- [ ] **Step 4: Replace the single testing override with route-indexed overrides** - -```cpp -struct DirectCcuRemoteRouteMemoryOverride { - uint32_t syncRouteIndex = 0; - TileXRCcuRemoteCcuBufferInfo buffer; -}; -std::vector directCcuRemoteRouteMemoryOverrides_; -``` - -`SetDirectCcuRemoteRouteMemoryOverride` must keep its old all-routes behavior for 2P tests. `SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute` updates/inserts one indexed entry, `Apply...` overlays every matching route, and `Clear...` clears the vector. - -- [ ] **Step 5: Implement one gather and three imports** - -```cpp -std::vector endpoints(4); -TILEXR_RETURN_IF_ERROR(session.AllGather(&localEndpoint, sizeof(localEndpoint), endpoints.data())); -for (uint32_t peer = 0, ordinal = 0; peer < 4; ++peer) { - if (peer == static_cast(session.Rank())) continue; - ValidateEndpoint(endpoints[peer], peer, 4 * chunkBytes); - auto imported = ImportDestination(session, endpoints[peer]); - mesh.peers.push_back({peer, endpoints[peer].destinationRemoteImport.addr, - TileXRCcuPackMemoryToken(endpoints[peer].destinationRemoteImport.tokenId, - endpoints[peer].destinationRemoteImport.tokenValue, true)}); - for (uint32_t route = 0; route < 3; ++route) { - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(3 * ordinal + route, peer, - imported.targetSegVa, endpoints[peer].destinationRemoteImport.tokenId, - endpoints[peer].destinationRemoteImport.rawTokenId, - endpoints[peer].destinationRemoteImport.tokenValue); - } - ++ordinal; -} -``` - -Set `syncResourceCount=9`, derive `syncInstructionCount` from the built Mesh program rather than a stale constant, invoke the Mesh orchestrator, and clear overrides on every return path with a small RAII guard. - -- [ ] **Step 6: Run planner, backend, and 2P regression tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_collective_planner tests.ccu.test_tilexr_ccu_direct_backend tests.ccu.test_tilexr_ccu_direct_orchestrator -v` - -Expected: PASS; the existing 2P single-route behavior remains unchanged. - -- [ ] **Step 7: Commit** - -```bash -git add src/comm/ccu/tilexr_ccu_collective_planner.h src/comm/ccu/tilexr_ccu_collective_planner.cpp tests/ccu/test_tilexr_ccu_collective_planner.py tests/ccu/test_tilexr_ccu_direct_backend.py -git commit -m "feat(ccu): plan four-rank all-to-all endpoints" -``` - -### Task 5: Reuse One Prepared Mission For Ten Mesh Submissions - -**Files:** -- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` -- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` - -- [ ] **Step 1: Write failing source-contract and helper tests** - -Add tests requiring Mesh mode to allocate `rankSize * chunkBytes`, call `PrepareDirectCcuAllToAllMeshInstallAttempt` before the loop, create the stream before the loop, and call neither prepare nor stream creation inside it. Require a phase path containing `ready.phase` and `done.phase`, three marker validations per loop, full-buffer validation, and failure output fields `rank`, `loopIndex`, `peerRank`, `route`, `channel`, `xn`, `cke`, `currentInstruction`, `selfCopyCompletion`, `sourceRank`, `chunkOffset`, and `globalOffset`. - -- [ ] **Step 2: Run smoke-probe tests to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` - -Expected: FAIL because the probe remains two-rank-specific. - -- [ ] **Step 3: Add generation-specific data and marker helpers** - -```cpp -uint8_t ExpectedAllToAllByte(uint32_t source, uint32_t target, uint32_t loop, uint64_t offset) -{ - return static_cast((source * 67U + target * 29U + loop * 17U + - static_cast(offset * 13U)) & 0xffU); -} - -uint64_t MakeLoopMarker(uint32_t rank, uint32_t loop) -{ - return 0x5458524100000000ULL | (static_cast(rank & 0xffU) << 8U) | - static_cast(loop & 0xffU); -} -``` - -Fill `send[target][offset]` from these fields, reset all 8 MiB of receive memory each loop, and validate `recv[source][offset]` with source/local-rank/loop. - -- [ ] **Step 4: Prepare once and execute phase-isolated loops** - -```cpp -PrepareMeshOnce(&attempt, &report); -CreateStreamOnce(&stream); -const auto stable = CaptureMissionAndResourceIdentity(attempt); -for (uint32_t loopIndex = 0; loopIndex < loopCount; ++loopIndex) { - FillFourSendChunks(rank, loopIndex); - ResetCompleteReceiveBuffer(); - attempt.submitTasks[0].args[0] = MakeLoopMarker(rank, loopIndex); - FourRankGate(workDir / ("ready.phase" + std::to_string(loopIndex)), rank, 0); - SubmitAndSynchronizeSameTask(attempt.submitTasks[0], stream); - ValidateThreePeerMarkers(loopIndex); - const auto localResult = ValidateCompleteReceiveBuffer(rank, loopIndex); - FourRankGate(workDir / ("done.phase" + std::to_string(loopIndex)), rank, localResult); - StopAllRanksIfAnyDoneResultFailed(loopIndex); - AssertMissionAndResourceIdentity(stable, attempt, loopIndex); -} -``` - -The ready/done payload must include the loop marker, not a reused boolean. Any failure prints diagnostics and exits before beginning the next phase. - -- [ ] **Step 5: Run probe tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` - -Expected: PASS for Mesh contracts and existing 2P loop-reuse contracts. - -- [ ] **Step 6: Commit** - -```bash -git add tests/ccu/ccu_tilexr_direct_smoke_probe.cpp tests/ccu/test_tilexr_ccu_direct_smoke_probe.py -git commit -m "test(ccu): exercise repeated four-rank all-to-all" -``` - -### Task 6: Generalize The Smoke Runner To Four Processes - -**Files:** -- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` -- Modify: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` - -- [ ] **Step 1: Add failing four-rank runner tests** - -Create fake probe logs for ranks `0..3`. Verify device list `4,5,6,7`, per-rank endpoint/EID/resource-window forwarding, four PIDs/statuses/logs, and exact result rules: - -```python -self.assertIn("expectedResults=40", result.stdout) -self.assertIn("expectedMarkerMatches=120", result.stdout) -self.assertIn("rank=3 device=7", result.stdout) -self.assertNotIn("resultCount>=1", runner_source) -``` - -Add negative cases for one missing rank log, one nonzero status, 39/40 results, 119/120 markers, invalid device count, and duplicate devices. - -- [ ] **Step 2: Run runner tests to verify RED** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_runner -v` - -Expected: FAIL because launch/status/log handling is hard-coded to two ranks. - -- [ ] **Step 3: Replace rank0/rank1 variables with indexed arrays** - -```bash -rank_size="${TILEXR_CCU_RANK_SIZE:-2}" -IFS=',' read -r -a devices <<< "${TILEXR_CCU_DEVICE_LIST:-0,1}" -[ "${#devices[@]}" -eq "${rank_size}" ] || fail "device count does not match rank size" -declare -a pids statuses logs -for ((rank=0; rank"${work_dir}/ccu_rank${rank}.log" 2>&1 & - pids[$rank]=$! - logs[$rank]="${work_dir}/ccu_rank${rank}.log" -done -``` - -Wait every PID even after one fails, record every status, and never kill unrelated processes. `export_rank_environment` must forward rank-specific EID, endpoint, token, XN, CKE, and resource-window fields for ranks `0..3`; default Mesh EID index is `3`. - -- [ ] **Step 4: Enforce exact aggregate counts** - -```bash -expected_results=$((rank_size * loop_count)) -expected_markers=$((rank_size * (rank_size - 1) * loop_count)) -[ "${result_count}" -eq "${expected_results}" ] || fail "result count mismatch" -[ "${marker_count}" -eq "${expected_markers}" ] || fail "marker count mismatch" -``` - -Keep rank size two as the default so current callers remain compatible. - -- [ ] **Step 5: Run runner and probe tests** - -Run: `python -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_runner tests.ccu.test_tilexr_ccu_direct_smoke_probe -v` - -Expected: PASS for 2P and 4P dry-run/fake-process cases. - -- [ ] **Step 6: Commit** - -```bash -git add tests/ccu/run_tilexr_ccu_direct_smoke.sh tests/ccu/test_tilexr_ccu_direct_smoke_runner.py -git commit -m "test(ccu): launch four-rank direct smoke" -``` - -### Task 7: Local Regression And Remote Hardware Validation - -**Files:** -- Modify only if a test exposes a defect in files from Tasks 1-6. -- Verify: all affected CCU suites and `build_ccu_direct`. - -- [ ] **Step 1: Run all affected local tests** - -```bash -python -m unittest \ - tests.ccu.test_tilexr_ccu_microcode \ - tests.ccu.test_tilexr_ccu_alltoall_program \ - tests.ccu.test_tilexr_ccu_direct_orchestrator \ - tests.ccu.test_tilexr_ccu_collective_planner \ - tests.ccu.test_tilexr_ccu_direct_backend \ - tests.ccu.test_tilexr_ccu_direct_smoke_probe \ - tests.ccu.test_tilexr_ccu_direct_smoke_runner -v -``` - -Expected: PASS with zero failures/errors. - -- [ ] **Step 2: Build the library and smoke probe on Linux** - -```bash -source scripts/common_env.sh -cmake --build build_ccu_direct --target tile-comm -j2 -cmake --build build_ccu_direct --target ccu_tilexr_direct_smoke_probe -j2 -``` - -Expected: both targets complete successfully and link against the real runtime libraries, not the `devlib` HAL stub. - -- [ ] **Step 3: Confirm passwordless access and the existing remote path** - -Run from Windows: - -```powershell -ssh -o BatchMode=yes root@141.61.50.31 'pwd; test -d /root/TileXR && echo TILEXR_REMOTE_OK' -``` - -Expected: passwordless login succeeds and prints the confirmed repository path. Do not modify `authorized_keys`. - -- [ ] **Step 4: Reuse or create the confirmed Mutagen sync** - -Inspect first: - -```powershell -& 'C:\Users\l00654177\AppData\Local\Programs\Mutagen\mutagen.exe' sync list -``` - -If no session exactly matches local `C:\Users\l00654177\Desktop\TileXR` and confirmed `root@141.61.50.31:/root/TileXR`, create one with ignores for `.git`, build/cache directories, credentials, and the unrelated untracked home-directory artifacts: - -```powershell -& 'C:\Users\l00654177\AppData\Local\Programs\Mutagen\mutagen.exe' sync create ` - --name tilexr-141-61-50-31 ` - --ignore-vcs ` - --ignore 'build*' --ignore '.anaconda' --ignore '.conda' --ignore '.mutagen*' --ignore '.ssh' ` - 'C:\Users\l00654177\Desktop\TileXR' 'root@141.61.50.31:/root/TileXR' -``` - -Then run `mutagen sync monitor tilexr-141-61-50-31` until status is `Watching for changes`. If Mutagen cannot use the existing SSH configuration, record the error and transfer only Task 1-6 changed files with `scp`. - -- [ ] **Step 5: Poll devices 4,5,6,7 until idle** - -On the server, run the repository busy guard or `npu-smi info`; if any selected device is occupied, query again every 30 seconds. Do not stop, signal, or reconfigure another workload. - -- [ ] **Step 6: Run 4P loop=1 in a fresh directory** - -```bash -TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ -TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ -TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH=1 \ -TILEXR_CCU_RANK_SIZE=4 \ -TILEXR_CCU_DEVICE_LIST=4,5,6,7 \ -TILEXR_CCU_ALLTOALL_BYTES=2097152 \ -TILEXR_CCU_ALLTOALL_LOOP_COUNT=1 \ -TILEXR_CCU_SMOKE_WORK_DIR=/tmp/tilexr-ccu-4p-loop1-$(date +%s) \ -bash tests/ccu/run_tilexr_ccu_direct_smoke.sh -``` - -Expected: exit 0, exactly 4 successful loop results, exactly 12 peer marker matches, and zero mismatches over four 8 MiB receive buffers. - -- [ ] **Step 7: Run 4P loop=10 in another fresh directory** - -Use the Step 6 command with `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10` and a `loop10` work directory. - -Expected: exit 0, exactly 40 successful loop results, exactly 120 peer marker matches, zero mismatches, phase files `0..9`, and stable mission ID/key/instruction range/task count/QP/jetty/CKE/XN/channel IDs across all ten loops. - -- [ ] **Step 8: Re-run the 2P loop=10 regression** - -After polling the selected validated pair idle, run the existing 2P command with `TILEXR_CCU_RANK_SIZE=2`, its two-device list, and `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10`. - -Expected: exit 0, exactly 20 successful loop results, exactly 20 peer marker matches, and unchanged 2P mission/resource identity. - -- [ ] **Step 9: Inspect evidence and repository hygiene** - -```bash -git diff --check -git status --short -``` - -Expected: no whitespace errors; only intended Task 1-6 files are modified. Preserve unrelated untracked files. - -- [ ] **Step 10: Commit the verified implementation** - -Stage only the explicit Task 1-6 paths and commit: - -```bash -git commit -m "feat(ccu): validate four-rank all-to-all mesh" -``` - -Record in the final verification report the three hardware commands, work directories, exact result/marker counts, mission/resource identity evidence, and any temporary Mutagen session name. diff --git a/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md b/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md deleted file mode 100644 index 358a8d8e..00000000 --- a/docs/superpowers/plans/2026-07-20-ccu-alltoall-loop-reuse.md +++ /dev/null @@ -1,112 +0,0 @@ -# CCU AlltoAll Loop Reuse Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Validate ten consecutive two-rank CCU AlltoAll submissions using one prepared mission and loop-specific device synchronization markers. - -**Architecture:** Extend PreSync with a marker loaded from SQE argument zero and synchronized through a dedicated XN with mask `0x1`; address and token retain masks `0x2` and `0x4`, and the receiver waits for `0x7`. The smoke probe prepares once, updates buffers and task arg zero per loop, submits repeatedly with phase-specific host gates, then validates both the peer marker and full destination data. - -**Tech Stack:** C++14, CCU microcode encoders, Python `unittest`, Bash hardware runner, ACL runtime. - ---- - -### Task 1: Encode The Loop Marker In PreSync - -**Files:** -- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.h` -- Modify: `src/comm/ccu/tilexr_ccu_alltoall_program.cpp` -- Modify: `src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp` -- Modify: `src/comm/ccu/tilexr_ccu_collective_planner.cpp` -- Test: `tests/ccu/test_tilexr_ccu_alltoall_program.py` -- Test: `tests/ccu/test_tilexr_ccu_direct_orchestrator.py` - -- [ ] **Step 1: Write failing generator tests** - -Assert that marker-enabled PreSync emits `LoadSqeArgsToX(markerLocalXn, 0)`, then `SyncXn(markerRemoteXn, markerLocalXn, preChannel, notifyCke, 0x1)`, followed by the existing address/token notifications and a `SetCke` wait mask of `0x7`. - -- [ ] **Step 2: Run the focused tests and verify RED** - -Run: - -```bash -python3 -m unittest tests.ccu.test_tilexr_ccu_alltoall_program tests.ccu.test_tilexr_ccu_direct_orchestrator -``` - -Expected: failure because the program spec has no marker fields and the instruction count remains `5 + 64 * 7`. - -- [ ] **Step 3: Implement marker-enabled PreSync** - -Add `preSyncLocalMarkerXn`, `preSyncRemoteMarkerXn`, `preSyncMarkerArgIndex`, and `preSyncMarkerEnabled` to the program spec. Encode the marker load with `TileXRCcuEncodeLoadSqeArgsToX`, notify mask `0x1`, and wait mask `0x7`. Increase the AlltoAll instruction capacity to `7 + 64 * 7` and map the copy resource XNs to the marker pair. - -- [ ] **Step 4: Run focused tests and verify GREEN** - -Run the command from Step 2. Expected: all tests pass. - -### Task 2: Repeat One Prepared Mission With Per-Loop Data - -**Files:** -- Modify: `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` -- Modify: `tests/ccu/run_tilexr_ccu_direct_smoke.sh` -- Test: `tests/ccu/test_tilexr_ccu_direct_smoke_probe.py` -- Test: `tests/ccu/test_tilexr_ccu_direct_smoke_runner.py` - -- [ ] **Step 1: Write failing loop-control tests** - -Require `TILEXR_CCU_ALLTOALL_LOOP_COUNT`, a default of one, preparation outside the loop, task argument zero updated per loop, `phase=loopIndex` ready/done gates, source/destination refresh, peer marker readback, and loop-indexed result output. - -- [ ] **Step 2: Run focused smoke tests and verify RED** - -Run: - -```bash -python3 -m unittest tests.ccu.test_tilexr_ccu_direct_smoke_probe tests.ccu.test_tilexr_ccu_direct_smoke_runner -``` - -Expected: failure because no loop-count environment variable or repeated submission exists. - -- [ ] **Step 3: Implement repeated submission** - -Add a validated loop count, rank-and-loop pattern generation, destination reset, marker encoding, prepared task arg mutation, phase-specific gates, marker XN readback, and per-loop result reporting. Keep allocation, registration, plan preparation, and installation outside the loop. Forward the environment variable from the runner. - -- [ ] **Step 4: Run focused smoke tests and verify GREEN** - -Run the command from Step 2. Expected: all tests pass. - -### Task 3: Regression And Hardware Verification - -**Files:** -- Verify only; no planned production edits. - -- [ ] **Step 1: Run affected CCU tests** - -```bash -python3 -m unittest \ - tests.ccu.test_tilexr_ccu_alltoall_program \ - tests.ccu.test_tilexr_ccu_direct_orchestrator \ - tests.ccu.test_tilexr_ccu_direct_smoke_probe \ - tests.ccu.test_tilexr_ccu_direct_smoke_runner \ - tests.ccu.test_tilexr_ccu_lower_layer_plan_builder -``` - -Expected: zero failures. - -- [ ] **Step 2: Build `tile-comm` and the smoke probe** - -```bash -source scripts/common_env.sh -cmake --build build_ccu_direct --target tile-comm ccu_tilexr_direct_smoke_probe -j2 -``` - -Expected: both targets build successfully. - -- [ ] **Step 3: Wait for NPU 6 and 7 to become idle** - -Poll `npu-smi info` through `tests/ccu/ccu_npu_smi_busy_guard.py --devices 6,7` every 30 seconds. Do not terminate unknown jobs. - -- [ ] **Step 4: Run loop-count ten hardware validation** - -Set `TILEXR_CCU_ALLTOALL_LOOP_COUNT=10` with the established two-rank long-mission configuration. Expected: each rank prints ten successful loop results, every peer marker matches its loop, and every loop reports `mismatches=0`. - -- [ ] **Step 5: Inspect the final diff** - -Run `git diff --check` and confirm unrelated untracked files remain untouched. diff --git a/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md b/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md deleted file mode 100644 index 9d47d48c..00000000 --- a/docs/superpowers/specs/2026-07-10-ccu-signal-wait-design.md +++ /dev/null @@ -1,217 +0,0 @@ -# CCU Signal/Wait and Barrier Design - -## Goal - -Implement a two-rank synchronization capability over TileXR Direct CCU and wire it into the internal `TileXRComm` CCU backend. The work is intentionally internal first: no public C API is added until the runtime semantics, resource ownership, and hardware smoke are stable. - -The implementation order is: - -1. Single-direction signal/wait between two ranks. -2. Two-direction barrier built from the same signal/wait machinery. - -## Scope - -In scope: - -- Add internal `TileXRCcuBackend` capability for two-rank signal/wait. -- Reuse the existing Direct CCU runtime lifecycle, resource-window registration, peer allgather, lower-layer install, repository install, mission install, and `rtCCULaunch` submit path. -- Add smoke coverage for `rank0 -> rank1`, `rank1 -> rank0`, and two-rank barrier. -- Keep the feature behind internal/test-only entry points until validated. - -Out of scope: - -- Public C API such as `TileXRCommSignal`, `TileXRCommWait`, or `TileXRCommBarrier`. -- N-rank barrier. -- Alltoall or general collective backend dispatch. -- Host marker based success criteria. - -## Existing Building Blocks - -The design reuses these current modules: - -- `TileXRCcuBackend`: internal backend owned by `TileXRComm`. -- `TileXRCcuRuntimeSession`: rank/device state, Direct CCU runtime availability, socket/thread allgather. -- `TileXRCcuDirectRuntime`: HCCP/RA/runtime loading, RA ctx resource window, endpoint route collection, peer buffer export. -- `TileXRCcuCollectivePlanner`: lower-layer template generation and direct install attempt preparation. -- `TileXRCcuResourceAllocator`: mission, repository, XN, GSA, CKE, and channel allocation. -- `TileXRCcuBuildBarrierProgram`: existing CCU microcode builder for post/wait style synchronization. -- `TileXRCcuSubmitPreparedTasks`: prepared task submission through `rtCCULaunch`. - -## Internal API - -Add internal request and plan types under `src/comm/ccu`: - -```cpp -enum class TileXRCcuSignalWaitRole { - Signal, - Wait, - SignalAndWait, -}; - -struct TileXRCcuSignalWaitRequest { - int peerRank = -1; - TileXRCcuSignalWaitRole role = TileXRCcuSignalWaitRole::Signal; - uint32_t timeout = 0; -}; - -struct TileXRCcuSignalWaitPlan { - bool ready = false; - TileXRCcuDirectInstallAttempt attempt; - std::vector submitTasks; -}; -``` - -Add internal backend methods: - -```cpp -int PrepareSignalWait( - const TileXRCcuSignalWaitRequest& request, - TileXRCcuSignalWaitPlan* plan); - -int SubmitSignalWait( - const TileXRCcuSignalWaitPlan& plan, - aclrtStream stream, - TileXRCcuDirectSubmitReport* report); -``` - -These methods are C++ internal only. They are not declared in `src/include/tilexr_api.h`. - -## Signal/Wait Semantics - -For two ranks, one rank is the signaler and the other rank is the waiter. - -Signal rank: - -- Installs a CCU task that posts to the peer rank's notify/wait CKE through the lower-layer channel. -- The task should be post-only and should not wait for the peer. -- `aclrtSynchronizeStream()` returning on the signal rank only proves the signal task has been submitted and completed locally. - -Wait rank: - -- Installs a CCU task that waits on its local wait CKE. -- The task should not complete until the peer signal arrives. -- `aclrtSynchronizeStream()` returning on the wait rank is the synchronization proof. - -The smoke test must verify this by delaying the signal rank and checking that the wait rank's stream synchronize time exceeds the configured threshold. - -## Barrier Semantics - -The two-rank barrier is built from two opposing signal/wait operations: - -- rank0 signals rank1 and waits for rank1. -- rank1 signals rank0 and waits for rank0. - -The first implementation should prefer one CCU task per rank containing both post and wait instructions. This avoids a host scheduling gap between separate signal and wait submissions and keeps the barrier semantics close to the device timeline. - -If the one-task form exposes hardware ordering issues, the fallback is two prepared tasks per rank: post first, wait second. The fallback must remain internal and be selected only for diagnostics or if hardware behavior requires it. - -## Resource Flow - -The resource flow matches the current Direct CCU install path: - -1. Refresh Direct CCU basic info for the selected die. -2. Decode resource spec from basic info. -3. Allocate mission, repository instruction, XN, CKE, and channel resources. -4. Register the local CCU resource window through RA ctx. -5. Export local resource-window token and endpoint route. -6. Allgather peer resource-window and endpoint data. -7. Exchange peer XN/CKE/channel ownership proof. -8. Build lower-layer transport snapshot and install plan. -9. Build a CCU synchronization program from the selected role. -10. Build repository image and launch package. -11. Install repository, lower-layer resources, and mission/key. -12. Generate submit tasks only after install evidence matches the launch package. - -No env override should mutate prepared task fields, peer binding proof, mission key, or instruction ranges. - -## Microcode Plan - -Signal uses a post-only synchronization instruction. The preferred instruction mode is `SyncCkePostOnly` when the lower-layer proof contains the CKE resources needed for peer notification. `SyncXnPostOnly` can be retained as a diagnostic fallback only if the CKE path is not viable on hardware. - -Wait uses the existing local CKE wait encoding. If current `TileXRCcuBuildBarrierProgram` cannot express a pure remote-triggered wait cleanly, add a small dedicated builder such as: - -```cpp -int TileXRCcuBuildSignalWaitProgram( - const TileXRCcuSignalWaitProgramSpec& spec, - std::vector* program, - TileXRCcuBarrierProgramReport* report); -``` - -That builder should still call the existing low-level microcode encoders and should avoid duplicating instruction encoding logic. - -## Error Handling - -Preparation returns explicit errors for: - -- Direct CCU runtime unavailable. -- Rank size not equal to 2. -- Invalid peer rank. -- Missing Direct CCU basic info. -- Resource allocation failure. -- Resource-window registration failure. -- Peer route/token exchange failure. -- Lower-layer install-plan failure. -- Repository/mission install failure. -- Install evidence mismatch. - -Submission returns explicit errors for: - -- Empty or not-ready plan. -- Null stream. -- Runtime launch failure. -- Mid-batch submit failure. - -Reports should include enough task detail for diagnosis: mission id, key, instruction range, argument size, submitted task count, and runtime return code when available. - -## Testing - -Unit tests: - -- Signal/wait request validation. -- Signal-only program generation. -- Wait-only program generation. -- Signal-and-wait program generation. -- Resource allocation shape for one peer route. -- Submit path rejects null stream and empty task list. -- Backend boundary tests verify the feature remains internal and does not appear in public headers. - -Smoke tests: - -- `rank0 signal -> rank1 wait`. -- `rank1 signal -> rank0 wait`. -- Two-rank barrier. -- Delayed signal rank proves wait rank blocks on CCU completion. -- Timeout wrapping remains enabled around the whole runner. - -Expected smoke evidence: - -```text -tilexr_ccu_signal_wait prepare ret=0 ... installSucceeded=1 ... submitReady=1 -tilexr_ccu_signal_wait submit ret=0 ... submitted=1 -tilexr_ccu_signal_wait timing rank= syncMs= -tilexr_ccu_signal_wait result passed=1 -``` - -For barrier: - -```text -tilexr_ccu_barrier prepare ret=0 ... installSucceeded=1 ... submitReady=1 -tilexr_ccu_barrier submit ret=0 ... submitted=1 -tilexr_ccu_barrier result passed=1 -``` - -## Validation Gates - -Before considering the feature complete: - -1. CCU unit tests pass. -2. `tile-comm` builds on the NPU server. -3. No hcomm/HCCL private CCU dependency is introduced. -4. Two-card signal/wait smoke passes in both directions. -5. Two-card barrier smoke passes. -6. Delayed waiter/signal timing proves device-side synchronization rather than host-side gating. - -## Open Decisions - -No open product decisions remain for the first implementation. Public API shape and N-rank barrier are intentionally deferred until after the internal backend path is validated. - diff --git a/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md b/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md deleted file mode 100644 index 88742de3..00000000 --- a/docs/superpowers/specs/2026-07-20-ccu-alltoall-loop-reuse-design.md +++ /dev/null @@ -1,42 +0,0 @@ -# CCU AlltoAll Loop Reuse Design - -## Goal - -Validate ten consecutive two-rank bidirectional CCU AlltoAll submissions while reusing one communicator, one installed mission, and the same QP, jetty, CKE, XN, and registered-memory resources. - -## Round Protocol - -The smoke probe prepares the long AlltoAll mission once. For each loop index from zero through nine, both ranks: - -1. Fill the source buffer with a rank-and-loop-specific pattern. -2. Fill the destination buffer with a sentinel value. -3. Set SQE argument zero to a rank-and-loop-specific 64-bit marker. -4. Enter a loop-specific host ready gate using `phase=loopIndex`. -5. Submit the same prepared task and synchronize its stream. -6. Read the peer marker XN and require the exact marker for the current peer and loop. -7. Read the complete destination buffer and compare it with the current peer pattern. -8. Enter a loop-specific done gate with the local validation result. - -Any failure terminates the loop and reports the loop index, mission context, XN/CKE state, marker value, and mismatch details. - -## Device Marker - -PreSync carries three variables on the same channel: - -- peer receive address with CKE mask `0x2`; -- peer receive token with CKE mask `0x4`; -- current SQE loop marker with CKE mask `0x1`. - -The receiver waits for mask `0x7`. The marker is loaded with `LoadSqeArgsToX`, so the installed instruction sequence remains unchanged while each submission supplies a distinct value. The marker uses a fixed magic prefix plus rank and loop index, allowing host readback to reject a signal from another loop. - -The CKE bits remain presence flags and are cleared by the existing wait instruction. The loop-specific XN marker supplies the generation identity that the existing CKE protocol lacks. - -## Configuration - -`TILEXR_CCU_ALLTOALL_LOOP_COUNT` controls repeated submissions and defaults to one. Values outside `1..1024` fail parameter validation. The runner forwards the variable to both ranks. - -## Verification - -- Unit tests verify the marker instruction ordering, mask `0x7`, instruction capacity, default loop count, runner forwarding, and loop-specific phase gates. -- The remote hardware test uses NPU 6 and 7 with loop count ten. -- Success requires ten loop results per rank (twenty total), exact peer markers, and zero mismatches in every loop. From 0c385b4d4fa40c08cf07b3fdb7eb2673a59c2a54 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Tue, 28 Jul 2026 09:46:56 +0800 Subject: [PATCH 40/41] docs(ccu): update all-to-all runtime design --- .../2026-07-14-ccu-alltoall-mvp-design.md | 620 ++++++++++++------ 1 file changed, 421 insertions(+), 199 deletions(-) diff --git a/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md b/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md index b8ed5de1..54d91812 100644 --- a/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md +++ b/docs/superpowers/specs/2026-07-14-ccu-alltoall-mvp-design.md @@ -1,281 +1,503 @@ -# TileXR Direct CCU 2-Rank AllToAll MVP Design +# TileXR Direct CCU AllToAll, Loop Reuse, and Signal/Wait Design + +## Status + +This document describes the direct CCU implementation in the current TileXR +tree. It replaces the original fixed two-rank MVP proposal. + +The implemented test surface provides: + +- a two-rank LocalToRemote AllToAll long mission; +- a runtime-sized AllToAll mesh for 2 through 64 ranks; +- repeated submission of one installed mission; +- internal two-rank signal, wait, and signal-and-wait plans; +- an opt-in hardware smoke runner with data and synchronization checks. + +The implementation is TileXR-owned. Active code does not include, call, or +link HCCL or hcomm private CCU interfaces. HCCL topology files and captured +microcode may be used as behavior and topology references only. + +## Scope and Boundaries + +The direct CCU path is currently an internal and test-oriented capability. +AllToAll preparation entry points used by the smoke probe are compiled under +`TILEXR_CCU_TESTING`. Signal/wait is exposed through the internal +`TileXRCcuBackend` C++ interface. No public collective or signal/wait C API +is added to `src/include/tilexr_api.h`. + +The current implementation uses host-expanded 32 KB copy blocks. Loop reuse +means repeated submission of one prepared CCU mission; it does not mean that a +CCU LoopEngine or LoopGroup traverses the payload. + +Hardware validation is narrower than the accepted software shape: + +- the mesh builder and planner accept 2 through 64 ranks; +- the smoke probe accepts 128 KB or 2 MB per mesh destination chunk; +- the two-rank smoke accepts 2 MB or 8 MB payloads; +- the maintained validation in this change covers 2-rank and 4-rank 2 MB + runs on Ascend950PR. + +## Components + +The feature reuses the direct CCU stack: + +- `TileXRCcuRuntimeSession`: rank, device, socket exchange, runtime and RA + lifecycle, resource-window registration, and endpoint allgather. +- `TileXRCcuCollectivePlanner`: endpoint import, peer resource exchange, + topology-aware route construction, and install-attempt preparation. +- `TileXRCcuResourceAllocator`: mission, repository, XN, GSA, CKE, channel, + and source-CKE allocation. +- `tilexr_ccu_alltoall_program`: two-rank and mesh microcode builders and + mesh binding validation. +- `tilexr_ccu_signal_wait_program`: Signal, Wait, and SignalAndWait program + generation. +- `TileXRCcuDirectOrchestrator`: launch package, repository image, lower + layer install, mission install, and prepared task generation. +- `run_tilexr_ccu_direct_smoke.sh` and + `ccu_tilexr_direct_smoke_probe.cpp`: opt-in hardware execution and result + validation. + +## Common Prepare and Submit Lifecycle + +All modes use the same ownership sequence: + +1. Initialize the direct CCU runtime for the selected logical and physical + device. +2. Refresh and decode CCU basic resource information for the install die. +3. Register the local CCU resource window through RA ctx. +4. Register or import operation buffers and exchange endpoint metadata. +5. Resolve peer EIDs and ports from `/etc/hccl_rootinfo.json` and its + referenced topology file. +6. Allocate mission, repository, XN, GSA, CKE, source CKE, and channel ranges. +7. Exchange peer-local XN/CKE ownership and build verified channel routes. +8. Install lower-layer PFE, channel, and jetty contexts. +9. Build and install the instruction repository, mission, key, and task + windows. +10. Expose prepared tasks only when required install surfaces are verified. +11. Submit through `rtCCULaunch` and wait through + `aclrtSynchronizeStreamWithTimeout`. + +One verified endpoint jetty may be shared by multiple logical channels. A +channel must not reference an invented jetty context. Peer notification uses +the peer's exchanged local wait CKE and channel-bound remote XN. + +Topology resolution selects CTP when the topology edge advertises `UB_CTP`; +otherwise it selects RTP. `TILEXR_CCU_DIRECT_FORCE_TP_TYPE=ctp|rtp` is a +diagnostic override. + +## Data Layout + +### Two-Rank Long Mission + +Each rank owns one send buffer and one receive buffer: -## Scope +```text +rank 0 send[bytes] -> rank 1 receive[bytes] +rank 1 send[bytes] -> rank 0 receive[bytes] +``` -Build a TileXR-owned direct CCU 2-rank AllToAll smoke path for a fixed P2P -payload of 2 MB per rank. The MVP follows the HCCL CCU AllToAll algorithm -shape, but it must not include or call any HCCL or hcomm interface. +The program requires a nonzero 4 KB-aligned payload. The program builder +accepts `memSlicePerBlock` in `[1, 8]`; the maintained smoke path requires +eight 4 KB memory slices per block: -The MVP validates: +```text +blockBytes = 4096 * 8 = 32768 +blockCount = bytes / 32768 +``` -- Rank 0 sends 2 MB to rank 1's receive buffer. -- Rank 1 sends 2 MB to rank 0's receive buffer. -- The CCU mission performs PreSync, data movement, copy completion wait, and - PostSync. -- Host-side validation reads local receive buffers and checks peer-specific - data patterns. +The current long-mission smoke defaults to 2 MB, so it emits 64 copy blocks. -This is a hardware data-plane and synchronization smoke. It is not yet the -LoopEngine validation target. +### Runtime-Sized Mesh -## Non-Goals +For rank count `N` and per-destination chunk size `C`, each rank allocates +`N * C` bytes for both source and destination. -- Do not include HCCL/hcomm headers. -- Do not call HCCL/hcomm launch, channel, resource, or CCU wrapper APIs. -- Do not link new HCCL/hcomm libraries. -- Do not claim LoopEngine is validated by the MVP. -- Do not add a broad public collective API before the smoke path proves the - direct CCU sequence. +```text +source[targetRank][chunkOffset] +destination[sourceRank][chunkOffset] +``` -## Reference Model +For every remote peer, rank `r` writes: -The local HCCL source under `.tmp/hccl-gitcode` uses this AllToAll shape: +```text +local source[peerRank] -> peer destination[r] +``` -1. `PreSync`: publish output address and token to peers and wait for peer - readiness. -2. `DoAlltoAll`: write local source slices into peer output through CCU - channels and wait for copy events. -3. `PostSync`: notify and wait for peers after data movement. +The self chunk uses local memory-to-memory transfer through local MS: -TileXR will use the same algorithm shape and size model, but express every -operation through TileXR-owned direct CCU program builders, resources, and -submit tasks. +```text +local source[r] -> local destination[r] +``` -## Data Model +The mesh requires `2 <= N <= 64`, exactly `N - 1` unique peer routes, and +a chunk size divisible by 32 KB. -The MVP is fixed-size and two-rank: +## Two-Rank Long-Mission Microcode -```text -rankSize = 2 -totalBytesPerRank = 2 * 1024 * 1024 -memorySliceBytes = 4096 -memSlicePerLoop = 8 -bytesPerBlock = memorySliceBytes * memSlicePerLoop = 32768 -blockCount = totalBytesPerRank / bytesPerBlock = 64 -``` +The two-rank launch package allocates three synchronization resources: + +1. copy and loop-marker route; +2. PreSync address/token route; +3. reserved post route. + +The installed smoke program intentionally enables PreSync and disables a +separate PostSync and finish instruction. Completion of every copy block is +already consumed in-order before stream completion. + +### PreSync and Loop Marker -`memSlicePerLoop` is capped at 8. The default is 8. The MVP should reject -non-4KB-aligned sizes and any block configuration that would produce a partial -MemorySlice. +PreSync publishes three values: -The data direction is LocalToRemote: +- SQE argument zero loop marker with mask `0x1`; +- local receive address with mask `0x2`; +- local receive token with mask `0x4`. + +The receiver waits for mask `0x7`. The marker instruction sequence is: ```text -rank0: rank0.send -> rank1.recv -rank1: rank1.send -> rank0.recv +LoadSqeArgsToX(localMarkerXn, arg0) +SyncXn(remoteMarkerXn, localMarkerXn, channel, notifyCke, 0x1) ``` -This matches the HCCL-style AllToAll write model and keeps the operation -semantically close to send-to-peer communication. +Address and token use `LoadImdToXn` plus `SyncXn`. The final PreSync +`SetCke` wait consumes all three presence bits. This prevents an old CKE +arrival from being accepted as the current loop. + +### Copy Blocks -## Mission Flow +Each 32 KB block emits a seven-instruction LocalToRemote memory-copy program. +The completion CKE is consumed before the next block reuses it. For `B` +blocks, the current long mission contains: -Each rank submits one CCU mission. The mission contains four phases. +```text +instructions = 7 + 7 * B +``` -### 1. PreSync +The leading seven instructions are marker/address/token PreSync and its wait. +For 2 MB, `B = 64` and the mission contains 455 instructions. -The rank confirms that peer output address/token resources are ready before -remote writes begin. +## N-Rank Mesh Microcode -The MVP can implement this with TileXR-owned CKE/checklist operations: +Let: ```text -SetCke(local source CKE) -SyncCke(peer pre-sync wait CKE, local source CKE) -ClearCke(local pre-sync wait CKE) +P = N - 1 # remote peer count +G = ceil(P / 16) # grouped completion CKE count +B = chunkBytes / 32768 # blocks per destination chunk ``` -PreSync is intentionally separate from final PostSync. This makes readiness -and completion failures distinguishable in logs and mission traces. +Each peer has one logical synchronization resource and one channel. Shared +local address, token, and length XNs are reused across peer channels. Remote +completion bits are grouped in sets of at most 16 because one CKE mask is +16 bits. -### 2. Data Move +### Mesh PreSync -The MVP expands the 2 MB transfer into 64 fixed 32 KB blocks at host program -build time: +The program loads the local receive address and token once, initializes the +source CKE, then posts both values to every peer. It waits once per peer. ```text -for block in 0..63: - localAddr = sendBase + block * 32768 - remoteAddr = peerRecvBase + block * 32768 - length = 32768 - TransLocMemToRmtMem(localAddr, remoteAddr, length) - ClearCke(copy completion CKE) +preSyncInstructions = 3 + 3 * P ``` -This represents the HCCL-style `memSlicePerLoop=8` work unit without relying -on LoopEngine support. The block size is deliberately the same as eight 4 KB -MemorySlices. +Mesh PreSync does not use the two-rank SQE loop marker. -### 3. Copy Completion Wait +### Mesh Copy -Each block waits for its own transfer completion using the existing memory -copy completion CKE semantics. The same completion CKE can be reused because -the blocks are emitted serially and each block consumes completion before the -next block begins. +For each block: + +- every remote peer contributes six transfer instructions; +- the self chunk contributes nine local-MS instructions; +- one completion wait is emitted for each group of at most 16 remote peers. + +```text +copyInstructionsPerBlock = 6 * P + 9 + G +``` -### 4. PostSync +Remote peer ordinal `i` sets bit `i % 16` in completion CKE group +`i / 16`. The builder validates that completion CKEs are unique and do not +overlap the source CKE. -After all 64 blocks complete, the rank performs a final two-way completion -barrier: +### Mesh PostSync and Total Size + +After all blocks, every peer receives one `SyncCke` completion notification +and every local peer wait CKE is consumed. One finish instruction closes the +program. ```text -SetCke(local source CKE) -SyncCke(peer post-sync wait CKE, local source CKE) -ClearCke(local post-sync wait CKE) +postSyncInstructions = 2 * P +totalInstructions = + (3 + 3 * P) + + B * (6 * P + 9 + G) + + 2 * P + + 1 ``` -This is the TileXR-owned `SignalAndWait` completion barrier. +The resource request uses: + +- `P` synchronization resources and channels; +- `P` peer local-wait CKEs; +- `P` remote-notify CKEs; +- one source CKE plus `G` grouped completion CKEs; +- at least three local and three remote XNs; +- two local GSAs for the self copy. + +The builder performs a second binding validation pass over the encoded +program before repository installation. + +## Loop Reuse + +`TILEXR_CCU_ALLTOALL_LOOP_COUNT` controls repeated submission and defaults +to one. The probe accepts values from 1 through 1024. + +Preparation, buffer registration, lower-layer installation, repository +installation, and mission installation occur once. Each loop: + +1. fills source data with a rank-and-loop-specific pattern; +2. resets destination data to a loop-specific sentinel; +3. enters a host ready gate using `phase=loopIndex`; +4. submits the same prepared task and synchronizes the same stream; +5. validates the complete destination buffer; +6. enters a host done gate with the local result. -## Program Builder +For the two-rank long mission, each loop also: -Add a new builder under `src/comm/ccu`: +- writes a rank-and-loop marker to prepared task argument zero; +- reads the peer marker from the remote XN after stream completion; +- requires the exact expected marker before accepting the data result. + +The marker format is: ```text -tilexr_ccu_alltoall_program.h -tilexr_ccu_alltoall_program.cpp +0x4343554c00000000 | (rank << 16) | loopIndex ``` -Main types: +For the mesh path, the current program has no SQE marker. Loop identity is +validated by source/destination data patterns and loop-specific host gates. +The probe additionally requires mission id, key, instruction range, task +shape, XN, CKE, and channel resources to remain identical after every loop +and prints `stableResources=1`. + +This is mission reuse, not LoopEngine execution. + +## Signal/Wait and Barrier + +Signal/wait is an internal two-rank backend feature: ```cpp -struct TileXRCcuAllToAll2RankProgramSpec { - uint64_t localSendAddr; - uint64_t localSendToken; - uint64_t remoteRecvAddr; - uint64_t remoteRecvToken; - uint64_t bytes; - uint32_t memorySliceBytes; - uint32_t memSlicePerLoop; - uint16_t localGsa; - uint16_t remoteGsa; - uint16_t localXn; - uint16_t remoteXn; - uint16_t lengthXn; - uint16_t channelId; - uint16_t copyCompletionCke; - uint16_t preSyncLocalWaitCke; - uint16_t preSyncRemoteNotifyCke; - uint16_t postSyncLocalWaitCke; - uint16_t postSyncRemoteNotifyCke; - uint16_t sourceCke; -}; +TileXRCcuBackend::PrepareSignalWait(...) +TileXRCcuBackend::SubmitSignalWait(...) ``` -The builder validates: +`PrepareSignalWait` rejects any rank size other than two and requires the +other rank as `peerRank`. It allocates one synchronization resource and +uses the same direct runtime, endpoint exchange, lower-layer install, +repository, mission, and submit lifecycle as AllToAll. -- `bytes == 2 MB` for the initial smoke. -- `memorySliceBytes == 4096`. -- `1 <= memSlicePerLoop <= 8`. -- `bytes % (memorySliceBytes * memSlicePerLoop) == 0`. -- all XN/GSA/CKE/channel/token/address resources are non-zero. +### Roles -The builder emits one instruction stream: +`TileXRCcuSignalWaitProgramRole` has three roles: -```text -PreSync instructions -64 * 32KB LocalToRemote copy block instructions -PostSync instructions -Finish instruction -``` +- `Signal`: reserve prelude, source CKE initialization, peer `SyncCke`, + and finish. The emitted program has five instructions. +- `Wait`: one local CKE wait instruction. The allocated repository window + may be larger, but the installed task window is reduced to the actual + emitted program. +- `SignalAndWait`: Signal followed by a local wait and finish. The emitted + program has six instructions. + +The Signal role proves local post completion. The Wait role proves that the +peer signal reached its local wait CKE. SignalAndWait forms a two-rank +barrier when both ranks run it. + +### Smoke Role Selection -## Planner and Orchestrator +With `TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=1`, +`TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK` selects the Signal rank (default 0); +the other rank is Wait. -Add a testing/private direct CCU path first: +With `TILEXR_CCU_DIRECT_SMOKE_BARRIER=1`, both ranks use SignalAndWait. The +barrier flag also enables the generic barrier-program override, so callers +should set `TILEXR_CCU_DIRECT_BARRIER_MODE` explicitly. The maintained CKE +form uses `sync_cke`. + +`TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK` and +`TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS` can delay one rank before +submit. Delaying the signaler provides timing evidence that the waiter's +stream blocks on the device-side event rather than a host marker. + +## Hardware Runner Safety + +`tests/ccu/run_tilexr_ccu_direct_smoke.sh` is safe by default. It exits +without touching ACL or NPU runtime unless: ```text -TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt -TileXRCcuRunDirectAllToAll2RankInstallAttempt -BuildDirectAllToAll2RankLaunchPackage +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 ``` -The planner reuses the existing direct CCU lower-layer path: +When enabled, the runner: -- direct runtime session -- driver adapter -- endpoint route/channel install -- repository install -- mission/task build -- prepared task submit +- checks `npu-smi info`; +- rejects selected busy or unhealthy devices by default; +- compiles the private smoke probe against `libtile-comm.so`; +- launches one rank process per selected device; +- installs CCU resources; +- submits real CCU tasks only when + `TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1`; +- enforces per-process and stream timeouts; +- validates result counts before reporting success. -It should allocate enough CKE resources for separate pre-sync, copy-completion, -post-sync, and source CKE use. If the first implementation must reuse the -existing one-resource allocator shape, the resource mapping must be explicit in -trace output and unit tests. +`TILEXR_CCU_SMOKE_ALLOW_UNHEALTHY_NPU=1` permits Warning/Alarm devices but +still rejects busy devices. It must be used only with explicit authorization. -## Smoke Test +## Usage -Extend `tests/ccu/ccu_tilexr_direct_smoke_probe.cpp` with an opt-in mode: +The examples assume `tile-comm` is already built and use the b110 CANN +package on the validation server. -```text -TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 -TILEXR_CCU_ALLTOALL_BYTES=2097152 -TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 +### Two-Rank AllToAll With Loop Reuse + +```bash +cd /home/TileXR +source /home/pkg/b110/cann-9.1.0/set_env.sh + +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ +TILEXR_TILE_COMM_LIB=/home/TileXR/build_ccu_merge_abe4bd38/src/comm/libtile-comm.so \ +TILEXR_CCU_SMOKE_DEVICES=6,7 \ +TILEXR_CCU_RANK_SIZE=2 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=1 \ +TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ +TILEXR_CCU_ALLTOALL_BYTES=2097152 \ +TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 \ +TILEXR_CCU_ALLTOALL_LOOP_COUNT=10 \ +TILEXR_CCU_SMOKE_TIMEOUT=180 \ +TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ +timeout 220s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh ``` -The runner should: +Success requires 20 passing loop results, 20 matched peer markers, and zero +data mismatches. + +### Four-Rank Mesh -1. allocate local send and receive buffers on each rank; -2. fill send with a rank-specific byte pattern; -3. fill receive with a sentinel; -4. prepare the alltoall mission; -5. wait until both ranks are submit-ready; -6. submit the prepared task and synchronize the stream; -7. wait for both ranks to report done; -8. copy local receive buffer back to host; -9. verify it equals the peer rank pattern. +```bash +cd /home/TileXR +source /home/pkg/b110/cann-9.1.0/set_env.sh + +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ +TILEXR_TILE_COMM_LIB=/home/TileXR/build_ccu_merge_abe4bd38/src/comm/libtile-comm.so \ +TILEXR_CCU_SMOKE_DEVICES=0,1,2,3 \ +TILEXR_CCU_RANK_SIZE=4 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=1 \ +TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH=1 \ +TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ +TILEXR_CCU_ALLTOALL_BYTES=2097152 \ +TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=8 \ +TILEXR_CCU_ALLTOALL_LOOP_COUNT=1 \ +TILEXR_CCU_SMOKE_TIMEOUT=240 \ +TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ +timeout 280s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` -Expected passing evidence: +For loop count one, success requires four passing results, zero data +mismatches, and `stableResources=1` on every rank. -```text -tilexr_ccu_alltoall config rank= bytes=2097152 memSlicePerLoop=8 blockCount=64 -tilexr_ccu_alltoall submit ... submitRet=0 syncRet=0 -tilexr_ccu_alltoall result passed=1 rank= mismatches=0 +### One-Way Signal/Wait + +```bash +cd /home/TileXR +source /home/pkg/b110/cann-9.1.0/set_env.sh + +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ +TILEXR_TILE_COMM_LIB=/home/TileXR/build_ccu_merge_abe4bd38/src/comm/libtile-comm.so \ +TILEXR_CCU_SMOKE_DEVICES=6,7 \ +TILEXR_CCU_RANK_SIZE=2 \ +TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT=1 \ +TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=0 \ +TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK=0 \ +TILEXR_CCU_DIRECT_SMOKE_PRE_SUBMIT_DELAY_MS=1000 \ +TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ +TILEXR_CCU_SMOKE_TIMEOUT=120 \ +TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ +timeout 150s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh ``` -All hardware runs must use an outer `timeout`. +Rank 0 is Signal and rank 1 is Wait. Reverse the direction by setting +`TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK=1`. -## Unit Tests +### Two-Rank Barrier -Add focused tests before hardware validation: +```bash +cd /home/TileXR +source /home/pkg/b110/cann-9.1.0/set_env.sh + +TILEXR_RUN_CCU_DIRECT_SMOKE_PROBE=1 \ +TILEXR_TILE_COMM_LIB=/home/TileXR/build_ccu_merge_abe4bd38/src/comm/libtile-comm.so \ +TILEXR_CCU_SMOKE_DEVICES=6,7 \ +TILEXR_CCU_RANK_SIZE=2 \ +TILEXR_CCU_DIRECT_SMOKE_BARRIER=1 \ +TILEXR_CCU_DIRECT_BARRIER_MODE=sync_cke \ +TILEXR_CCU_DIRECT_SMOKE_SUBMIT=1 \ +TILEXR_CCU_SMOKE_TIMEOUT=120 \ +TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT=12000 \ +timeout 150s bash tests/ccu/run_tilexr_ccu_direct_smoke.sh +``` -- program builder rejects invalid `memSlicePerLoop > 8`; -- program builder rejects non-4KB-aligned size; -- 2 MB generates exactly 64 copy blocks; -- PreSync instructions appear before the first copy block; -- PostSync instructions appear after the last copy block; -- no HCCL/hcomm symbols are introduced into `src/comm`. +## Validation and Evidence -The dependency guard remains mandatory: +Source-level coverage checks: ```bash -bash tests/ccu/check_tile_comm_no_hcomm_deps.sh build/src/comm/libtile-comm.so +python3 -m unittest \ + tests.ccu.test_tilexr_ccu_alltoall_program \ + tests.ccu.test_tilexr_ccu_barrier_program \ + tests.ccu.test_tilexr_ccu_direct_orchestrator \ + tests.ccu.test_tilexr_ccu_direct_smoke_probe \ + tests.ccu.test_tilexr_ccu_direct_smoke_runner \ + tests.ccu.test_tilexr_ccu_lower_layer_plan_builder \ + tests.ccu.test_tilexr_ccu_resource_allocator \ + tests.ccu.test_tilexr_ccu_signal_wait_program + +bash tests/ccu/check_tile_comm_no_hcomm_deps.sh \ + build_ccu_direct/src/comm/libtile-comm.so ``` -## LoopGroup Follow-Up - -After the MVP passes, replace host-expanded 64-block emission with a TileXR-owned -LoopGroup implementation: - -- add TileXR LoopCtx/LoopGroup encoding; -- add LoopEngine allocation and lifecycle tracking; -- add GoSize calculation equivalent to the HCCL model: - - `addrOffset` - - `loopParam` - - `parallelParam` - - `residual` -- keep the smoke API and validation unchanged; -- add evidence that LoopEngine, not host expansion, performed the 2 MB - MemorySlice traversal. - -The follow-up must be a separate change so the MVP result cannot be mistaken -for LoopEngine validation. - -## Open Risks - -- TileXR currently lacks a first-class LoopGroup/LoopEngine encoder. -- Current direct memory-copy builder is single-copy oriented; the alltoall - builder must avoid accidental resource overlap across 64 copy blocks. -- PreSync and PostSync require enough independent CKE/checklist resources to - keep readiness and completion distinguishable. -- LocalToRemote validation writes peer memory, so host-side checks must read - each rank's local receive buffer after both ranks complete. +Maintained hardware evidence for the current change: + +- 2-rank, devices 6 and 7, 2 MB long mission: both ranks returned + `syncRet=0`, peer markers matched, and `mismatches=0`. +- 4-rank, devices 0 through 3, 2 MB mesh: all ranks returned `syncRet=0`, + `mismatches=0`, and `stableResources=1`. + +Signal/wait and barrier are implemented and covered by source-level tests. +They require their own hardware run before making a validation claim for a +new server or CANN/driver combination. + +## Failure Diagnostics + +On a timeout, the probe reports: + +- rank and loop index; +- submit and stream synchronization return codes; +- mission id, key, start/end/current instruction; +- local and remote XN values; +- local wait, remote notify, source, and completion CKE values; +- first and last data mismatch offsets and affected 32 KB blocks. + +All ranks stopping on the same wait instruction indicates a route, remote XN, +or CKE synchronization problem. A zero mismatch count before stream +completion does not prove success because destination validation has not run. + +## Remaining Limitations + +- AllToAll remains an internal/test-only direct CCU path. +- The mesh software contract reaches 64 ranks, but hardware validation in + this change covers four ranks. +- The smoke probe supports only selected payload sizes even though the + builders accept aligned sizes. +- Mesh loop reuse currently has no device generation marker. +- LoopEngine and LoopGroup are not implemented. +- Signal/wait and barrier remain two-rank only. +- Cleanup after timeout may be skipped by the smoke probe to isolate runtime + cleanup hangs; the outer timeout remains mandatory. From 2c4edec87e77c135531092c7c615a232a954e832 Mon Sep 17 00:00:00 2001 From: GoodTimeLZ Date: Tue, 28 Jul 2026 10:28:47 +0800 Subject: [PATCH 41/41] test(ccu): isolate long-mission loop timing --- tests/ccu/ccu_tilexr_direct_smoke_probe.cpp | 78 ++++--------------- tests/ccu/run_tilexr_ccu_direct_smoke.sh | 11 +++ .../ccu/test_tilexr_ccu_direct_smoke_probe.py | 13 +++- 3 files changed, 39 insertions(+), 63 deletions(-) diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index b8a17578..a486fb03 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -606,17 +606,6 @@ uint64_t BuildAllToAllLoopMarker(int rank, int loopIndex) static_cast(loopIndex & 0xffff); } -std::vector BuildAllToAllLoopPattern(int rank, int loopIndex, size_t bytes) -{ - std::vector pattern(bytes); - for (size_t i = 0; i < bytes; ++i) { - pattern[i] = static_cast( - (static_cast(rank + 1) * 17U + - static_cast(loopIndex + 1) * 29U + i * 13U) & 0xffU); - } - return pattern; -} - uint8_t BuildAllToAllMeshByte( uint32_t sourceRank, uint32_t targetRank, @@ -718,7 +707,8 @@ int InitAllToAllState(int rank, int peer, AllToAllState* state) } state->bytes = AllToAllBytesFromEnv(); const bool supportedBytes = state->bytes == 2U * 1024U * 1024U || - state->bytes == 8U * 1024U * 1024U; + state->bytes == 8U * 1024U * 1024U || + state->bytes == 16U * 1024U * 1024U; if (!supportedBytes || AllToAllMemSlicePerLoopFromEnv() != 8) { state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; return state->initRet; @@ -758,46 +748,6 @@ int InitAllToAllState(int rank, int peer, AllToAllState* state) return ret; } -int ResetAllToAllStateForLoop(int rank, int peer, int loopIndex, AllToAllState* state) -{ - if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || - state->bytes == 0 || loopIndex < 0) { - return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; - } - state->expected = BuildAllToAllLoopPattern(peer, loopIndex, state->bytes); - state->observed.assign(state->bytes, 0); - state->readRet = ACL_SUCCESS; - state->mismatchCount = 0; - state->firstMismatchOffset = 0; - state->lastMismatchOffset = 0; - state->firstMismatchObserved = 0; - state->firstMismatchExpected = 0; - state->mismatchedBlockCount = 0; - state->firstMismatchedBlock = 0; - state->lastMismatchedBlock = 0; - state->passed = false; - - const std::vector source = BuildAllToAllLoopPattern(rank, loopIndex, state->bytes); - const std::vector destination( - state->bytes, - static_cast(0xa5U ^ static_cast(loopIndex))); - int ret = aclrtMemcpy( - state->source.ptr, - state->bytes, - source.data(), - source.size(), - ACL_MEMCPY_HOST_TO_DEVICE); - if (ret != ACL_SUCCESS) { - return ret; - } - return aclrtMemcpy( - state->destination.ptr, - state->bytes, - destination.data(), - destination.size(), - ACL_MEMCPY_HOST_TO_DEVICE); -} - int CheckAllToAllState(AllToAllState* state) { if (state == nullptr || state->destination.ptr == nullptr || state->bytes == 0) { @@ -2357,19 +2307,18 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, finalRet = 7; } else { bool skipStreamDestroy = false; + int lastLoopIndex = -1; const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { - const int resetRet = ResetAllToAllStateForLoop(rank, peer, loopIndex, &alltoall); + lastLoopIndex = loopIndex; const uint64_t localLoopMarker = BuildAllToAllLoopMarker(rank, loopIndex); attempt.submitTasks.front().args[0] = localLoopMarker; const bool collectiveSubmitReady = WaitForCollectiveSubmitReadiness( rank, rankSize, - resetRet == ACL_SUCCESS && installReport.submitReady, + installReport.submitReady, loopIndex); - if (resetRet != ACL_SUCCESS) { - finalRet = 14; - } else if (!collectiveSubmitReady) { + if (!collectiveSubmitReady) { finalRet = 13; } @@ -2425,13 +2374,16 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, finalRet = 15; } } - if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { - finalRet = 14; - } if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex) && finalRet == 0) { finalRet = 13; } - PrintAllToAllResult(rank, loopIndex, finalRet, alltoall); + std::cout << "tilexr_ccu_alltoall loopResult" + << " passed=" << (finalRet == 0 ? 1 : 0) + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet + << " dataCheckDeferred=1" + << std::endl; if (finalRet != 0) { std::cerr << "tilexr_ccu_alltoall loopFailure" << " rank=" << rank @@ -2446,6 +2398,10 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, break; } } + if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { + finalRet = 14; + } + PrintAllToAllResult(rank, lastLoopIndex, finalRet, alltoall); if (skipStreamDestroy) { std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" << " rank=" << rank diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index 26f471a1..1ba2d8ab 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -973,6 +973,17 @@ fi if alltoall_mode_enabled && [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then loop_count="$(parse_int "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" 1)" expected_results=$((rank_size * loop_count)) + if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ] && + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH:-0}" != "1" ]; then + expected_loop_results=$((rank_size * loop_count)) + actual_loop_results="$(grep -h -c "tilexr_ccu_alltoall loopResult passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" + echo "tilexr_ccu_direct_smoke_runner alltoallLoopCounts expectedResults=${expected_loop_results} actualResults=${actual_loop_results}" + if [ "${actual_loop_results}" -ne "${expected_loop_results}" ]; then + echo "ERROR: direct CCU alltoall loop result count mismatch expected=${expected_loop_results} actual=${actual_loop_results}" >&2 + exit 9 + fi + expected_results="${rank_size}" + fi actual_results="$(grep -h -c "tilexr_ccu_alltoall result passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" echo "tilexr_ccu_direct_smoke_runner alltoallCounts expectedResults=${expected_results} actualResults=${actual_results}" if [ "${actual_results}" -ne "${expected_results}" ]; then diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index c39186a1..0e5a8b06 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -660,6 +660,7 @@ def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL", source) self.assertIn('kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"', source) self.assertIn('kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"', source) + self.assertIn("state->bytes == 16U * 1024U * 1024U", source) self.assertIn("struct AllToAllState", source) self.assertIn("AllToAllSmokeEnabled", source) self.assertIn("AllToAllLongMissionEnabled", source) @@ -693,8 +694,8 @@ def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): self.assertIn("std::strtol", loop_count_body) self.assertIn("parsed < 1 || parsed > 1024", loop_count_body) self.assertIn("BuildAllToAllLoopMarker", source) - self.assertIn("BuildAllToAllLoopPattern", source) - self.assertIn("ResetAllToAllStateForLoop", source) + self.assertNotIn("BuildAllToAllLoopPattern", source) + self.assertNotIn("ResetAllToAllStateForLoop", source) self.assertIn("ReadAndValidatePeerLoopMarker", source) self.assertIn("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)", body) self.assertIn("attempt.submitTasks.front().args[0] =", body) @@ -703,6 +704,13 @@ def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex)", body) self.assertIn("adapter.ReadXnRange", source) self.assertIn("peerLoopMarker", source) + loop_body = body[body.index("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)"):] + self.assertNotIn( + "CheckAllToAllState(&alltoall)", + loop_body[:loop_body.index("lastLoopIndex = loopIndex")], + ) + self.assertIn("CheckAllToAllState(&alltoall)", body) + self.assertIn("dataCheckDeferred=1", body) self.assertIn("attempt.plan.syncResources[0].remoteXn", body) self.assertNotIn("attempt.plan.syncResources[0].localXn,", body) self.assertIn("loopIndex=", body) @@ -838,6 +846,7 @@ def test_smoke_runner_forwards_alltoall_env(self): self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) self.assertIn("TILEXR_CCU_ALLTOALL_LOOP_COUNT", runner) + self.assertIn("tilexr_ccu_direct_smoke_runner alltoallLoopCounts", runner) self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) self.assertIn("long_mission_instruction_count=$((7 + long_mission_block_count * 7))", runner) self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner)