diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index 52aa2d98..b6749cd8 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -106,6 +106,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp tilexr_internal.h tilexr_log.h ccu/tilexr_ccu_abi_constants.h + ccu/tilexr_ccu_alltoall_program.h + ccu/tilexr_ccu_alltoall_program.cpp ccu/tilexr_ccu_barrier_program.h ccu/tilexr_ccu_barrier_program.cpp ccu/tilexr_ccu_backend.h diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp new file mode 100644 index 00000000..16a23e26 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -0,0 +1,471 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_alltoall_program.h" + +namespace TileXR { +namespace { + +uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID); +} + +uint16_t PreSyncTokenMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + (void)spec; + return static_cast(1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID); +} + +uint16_t PostSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) +{ + return spec.ckeMask; +} + +void ResetReport(TileXRCcuAllToAllProgramReport* report) +{ + if (report != nullptr) { + *report = TileXRCcuAllToAllProgramReport{}; + } +} + +int Fail( + std::vector* program, + TileXRCcuAllToAllProgramReport* report, + const std::string& message) +{ + if (program != nullptr) { + program->clear(); + } + if (report != nullptr) { + report->message = message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +int ValidateSpec( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (program == nullptr) { + return Fail(program, report, "missing output direct CCU alltoall program"); + } + if (spec.localRank > 1U) { + return Fail(program, report, "direct CCU alltoall localRank must be 0 or 1"); + } + if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || spec.remoteRecvAddr == 0) { + return Fail(program, report, "missing direct CCU alltoall address"); + } + if (spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.remoteRecvToken == 0) { + return Fail(program, report, "missing direct CCU alltoall token"); + } + if (spec.bytes == 0 || spec.bytes % TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES != 0) { + return Fail(program, report, "direct CCU alltoall bytes must be nonzero and 4KB aligned"); + } + if (spec.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES) { + return Fail(program, report, "direct CCU alltoall memorySliceBytes must be 4096"); + } + if (spec.memSlicePerBlock == 0 || spec.memSlicePerBlock > TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK) { + return Fail(program, report, "direct CCU alltoall memSlicePerBlock must be in [1, 8]"); + } + const uint64_t bytesPerBlock = static_cast(spec.memorySliceBytes) * spec.memSlicePerBlock; + if (spec.bytes % bytesPerBlock != 0) { + return Fail(program, report, "direct CCU alltoall bytes must align to memSlicePerBlock"); + } + if (spec.localGsa == 0 || spec.remoteGsa == 0 || spec.localXn == 0 || spec.remoteXn == 0 || + spec.lengthXn == 0) { + return Fail(program, report, "missing direct CCU alltoall GSA/XN resource"); + } + const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; + const uint16_t copyChannelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; + if (preSyncChannelId == 0 || copyChannelId == 0 || postSyncChannelId == 0 || + spec.copyCompletionCke == 0 || spec.preSyncLocalWaitCke == 0 || + spec.preSyncRemoteNotifyCke == 0 || spec.postSyncLocalWaitCke == 0 || + spec.postSyncRemoteNotifyCke == 0 || + (spec.postSyncNotify && spec.sourceCke == 0) || spec.ckeMask == 0) { + return Fail(program, report, "missing direct CCU alltoall CKE/channel resource"); + } + if (spec.preSyncMarkerEnabled && + (spec.preSyncLocalMarkerXn == 0 || spec.preSyncRemoteMarkerXn == 0 || + spec.preSyncMarkerArgIndex >= TILEXR_CCU_SQE_ARGS_LEN)) { + return Fail(program, report, "missing direct CCU alltoall loop marker resource"); + } + return TILEXR_SUCCESS; +} + +int AppendSetSourceCke( + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t mask, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuCkeSpec set; + set.ckeId = spec.sourceCke; + set.mask = mask; + set.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSetCke(set, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall source CKE set"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendNotifyWait( + uint16_t localWaitCke, + uint16_t mask, + const char* phase, + bool clearCkeWait, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuCkeSpec wait; + wait.waitCkeId = localWaitCke; + wait.waitMask = mask; + wait.clearWait = true; + + TileXRCcuInstr instr; + const int ret = clearCkeWait ? + TileXRCcuEncodeClearCke(wait, &instr) : + TileXRCcuEncodeSetCke(wait, &instr); + if (ret != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " NotifyWait"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendSyncPair( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint16_t localMask, + uint16_t waitMask, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuSyncCkeSpec post; + post.remoteCke = remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = localMask; + post.channelId = channelId; + post.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall SyncCke"); + } + program->push_back(instr); + + return AppendNotifyWait(localWaitCke, waitMask, "PostSync", true, program, report); +} + +int AppendRemoteNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + uint16_t localXn, + uint16_t remoteXn, + uint64_t value, + uint16_t secFlag, + uint16_t mask, + const char* phase, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(localXn, value, secFlag, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " variable load"); + } + program->push_back(instr); + + TileXRCcuSyncXnSpec notify; + notify.remoteXn = remoteXn; + notify.localXn = localXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = mask; + notify.clearWait = true; + + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + " SyncXn notify"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendRemoteMarkerNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadSqeArgsToX( + spec.preSyncLocalMarkerXn, + spec.preSyncMarkerArgIndex, + &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PreSync loop marker load"); + } + program->push_back(instr); + + TileXRCcuSyncXnSpec notify; + notify.remoteXn = spec.preSyncRemoteMarkerXn; + notify.localXn = spec.preSyncLocalMarkerXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK; + notify.clearWait = true; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PreSync loop marker notify"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendPreSyncPhase( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t outputChannelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + const uint16_t outputMask = PreSyncSignalMask(spec); + const uint16_t tokenMask = PreSyncTokenMask(spec); + const uint16_t markerMask = spec.preSyncMarkerEnabled ? TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK : 0U; + const uint16_t waitMask = static_cast(markerMask | outputMask | tokenMask); + const uint16_t localOutputXn = + spec.preSyncLocalAddrXn == 0 ? spec.localXn : spec.preSyncLocalAddrXn; + const uint16_t localTokenXn = + spec.preSyncLocalTokenXn == 0 ? spec.lengthXn : spec.preSyncLocalTokenXn; + const uint16_t tokenChannelId = + spec.preSyncTokenChannelId == 0 ? outputChannelId : spec.preSyncTokenChannelId; + const uint16_t tokenNotifyCke = + spec.preSyncRemoteTokenNotifyCke == 0 ? remoteNotifyCke : spec.preSyncRemoteTokenNotifyCke; + if (spec.preSyncMarkerEnabled && + AppendRemoteMarkerNotify( + remoteNotifyCke, + outputChannelId, + spec, + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AppendRemoteNotify( + remoteNotifyCke, + outputChannelId, + localOutputXn, + spec.preSyncRemoteAddrXn, + spec.localRecvAddr, + 0, + outputMask, + "PreSync output", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AppendRemoteNotify( + tokenNotifyCke, + tokenChannelId, + localTokenXn, + spec.preSyncRemoteTokenXn, + spec.localRecvToken, + 1, + tokenMask, + "PreSync token", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (!spec.preSyncWait) { + return TILEXR_SUCCESS; + } + + return AppendNotifyWait(localWaitCke, waitMask, "PreSync output", false, program, report); +} + +int AppendPostSyncPhase( + uint16_t remoteNotifyCke, + uint16_t localWaitCke, + uint16_t channelId, + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + const uint16_t notifyMask = PostSyncSignalMask(spec); + const uint16_t waitMask = PostSyncSignalMask(spec); + if (AppendSetSourceCke(spec, notifyMask, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (!spec.postSyncWait) { + TileXRCcuSyncCkeSpec post; + post.remoteCke = remoteNotifyCke; + post.localCke = spec.sourceCke; + post.localCkeMask = notifyMask; + post.channelId = channelId; + post.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall PostSync notify-only SyncCke"); + } + program->push_back(instr); + return TILEXR_SUCCESS; + } + return AppendSyncPair( + remoteNotifyCke, + localWaitCke, + channelId, + spec, + notifyMask, + waitMask, + program, + report); +} + +int AppendCopyBlock( + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuMemoryCopySpec copy; + copy.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; + copy.localGsa = spec.localGsa; + copy.localXn = spec.localXn; + copy.remoteGsa = spec.remoteGsa; + copy.remoteXn = spec.remoteXn; + copy.lengthXn = spec.lengthXn; + copy.localAddr = spec.localSendAddr + offset; + copy.localToken = spec.localSendToken; + copy.remoteAddr = spec.remoteRecvAddr + offset; + copy.remoteToken = spec.remoteRecvToken; + copy.lengthBytes = bytesPerBlock; + copy.channelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + copy.completionCke = spec.copyCompletionCke; + copy.completionMask = spec.ckeMask; + + std::vector block; + TileXRCcuMemoryProgramReport memoryReport; + if (TileXRCcuBuildMemoryCopyProgram(copy, &block, &memoryReport) != TILEXR_SUCCESS) { + return Fail(program, report, memoryReport.message); + } + program->insert(program->end(), block.begin(), block.end()); + return TILEXR_SUCCESS; +} + +int AppendFinish( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(spec.localXn, 0, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall finish instruction"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +void FillReport( + const TileXRCcuAllToAll2RankProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + if (report == nullptr) { + return; + } + const uint32_t bytesPerBlock = spec.memorySliceBytes * spec.memSlicePerBlock; + const uint32_t markerInstructionCount = spec.preSyncMarkerEnabled ? 2U : 0U; + report->preSyncInstructionCount = + spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) + markerInstructionCount : 0U; + report->blockCount = static_cast(spec.bytes / bytesPerBlock); + report->bytesPerBlock = bytesPerBlock; + report->copyInstructionCount = report->blockCount * 7U; + report->postSyncInstructionCount = !spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U); + report->finishInstructionCount = spec.emitFinish ? 1U : 0U; + report->totalInstructionCount = static_cast(program.size()); + report->message = "ok"; +} + +} // namespace + +int TileXRCcuBuildAllToAll2RankProgram( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + ResetReport(report); + if (program != nullptr) { + program->clear(); + } + int ret = ValidateSpec(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + const uint64_t bytesPerBlock = static_cast(spec.memorySliceBytes) * spec.memSlicePerBlock; + const uint32_t blockCount = static_cast(spec.bytes / bytesPerBlock); + const uint16_t preSyncChannelId = spec.preSyncChannelId == 0 ? spec.channelId : spec.preSyncChannelId; + const uint16_t postSyncChannelId = spec.postSyncChannelId == 0 ? spec.channelId : spec.postSyncChannelId; + const uint32_t markerInstructionCount = spec.preSyncMarkerEnabled ? 2U : 0U; + program->reserve( + (spec.preSyncNotify ? (spec.preSyncWait ? 5U : 4U) + markerInstructionCount : 0U) + blockCount * 7U + + (!spec.postSyncNotify ? 0U : (spec.postSyncWait ? 3U : 2U)) + + (spec.emitFinish ? 1U : 0U)); + + if (spec.preSyncNotify) { + ret = AppendPreSyncPhase( + spec.preSyncRemoteNotifyCke, + spec.preSyncLocalWaitCke, + preSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + for (uint32_t block = 0; block < blockCount; ++block) { + const uint64_t offset = static_cast(block) * bytesPerBlock; + ret = AppendCopyBlock(spec, offset, bytesPerBlock, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + if (spec.postSyncNotify) { + ret = AppendPostSyncPhase( + spec.postSyncRemoteNotifyCke, + spec.postSyncLocalWaitCke, + postSyncChannelId, + spec, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + if (spec.emitFinish) { + ret = AppendFinish(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + FillReport(spec, *program, report); + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h new file mode 100644 index 00000000..484ff30a --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_ALLTOALL_PROGRAM_H +#define TILEXR_CCU_ALLTOALL_PROGRAM_H + +#include "ccu/tilexr_ccu_memory_program.h" + +#include +#include +#include + +namespace TileXR { + +constexpr uint32_t TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES = 4096U; +constexpr uint32_t TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK = 8U; +constexpr uint32_t TILEXR_CCU_ALLTOALL_BLOCK_BYTES = + TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES * TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; +constexpr uint16_t TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_TOKEN_XN_ID = 2U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_ID = 3U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_SIGNAL_MASK = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK = 1U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK = 2U; + +struct TileXRCcuAllToAll2RankProgramSpec { + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t remoteSendAddr = 0; + uint64_t remoteSendToken = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; + uint64_t bytes = 0; + uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + uint16_t localGsa = 0; + uint16_t remoteGsa = 0; + uint16_t localXn = 0; + uint16_t remoteXn = 0; + uint16_t lengthXn = 0; + uint16_t preSyncLocalAddrXn = 0; + uint16_t preSyncLocalTokenXn = 0; + uint16_t preSyncLocalMarkerXn = 0; + uint16_t preSyncRemoteMarkerXn = 0; + uint16_t preSyncMarkerArgIndex = 0; + uint16_t channelId = 0; + uint16_t preSyncChannelId = 0; + uint16_t preSyncTokenChannelId = 0; + uint16_t copyChannelId = 0; + uint16_t postSyncChannelId = 0; + uint16_t copyCompletionCke = 0; + uint16_t preSyncRemoteAddrXn = 0; + uint16_t preSyncRemoteTokenXn = 0; + uint16_t preSyncLocalWaitCke = 0; + uint16_t preSyncRemoteNotifyCke = 0; + uint16_t preSyncTokenLocalWaitCke = 0; + uint16_t preSyncRemoteTokenNotifyCke = 0; + uint16_t postSyncLocalWaitCke = 0; + uint16_t postSyncRemoteNotifyCke = 0; + uint16_t sourceCke = 0; + uint16_t ckeMask = 1; + bool preSyncNotify = true; + bool preSyncWait = true; + bool preSyncMarkerEnabled = false; + bool postSyncNotify = true; + bool postSyncWait = true; + bool emitFinish = true; +}; + +struct TileXRCcuAllToAllProgramReport { + uint32_t preSyncInstructionCount = 0; + uint32_t blockCount = 0; + uint32_t bytesPerBlock = 0; + uint32_t copyInstructionCount = 0; + uint32_t postSyncInstructionCount = 0; + uint32_t finishInstructionCount = 0; + uint32_t totalInstructionCount = 0; + std::string message; +}; + +int TileXRCcuBuildAllToAll2RankProgram( + const TileXRCcuAllToAll2RankProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report); + +} // namespace TileXR + +#endif // TILEXR_CCU_ALLTOALL_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_backend.h b/src/comm/ccu/tilexr_ccu_backend.h index 65ebfd82..8ede81c9 100644 --- a/src/comm/ccu/tilexr_ccu_backend.h +++ b/src/comm/ccu/tilexr_ccu_backend.h @@ -53,6 +53,8 @@ enum class TileXRCcuSignalWaitRole { struct TileXRCcuSignalWaitRequest { int peerRank = -1; TileXRCcuSignalWaitRole role = TileXRCcuSignalWaitRole::Signal; + bool overrideBarrierMode = false; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncCke; uint32_t syncInstructionCount = 0; uint16_t missionStartId = 0; uint16_t instructionStartId = 0; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index 7991522b..f4659250 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -25,6 +25,9 @@ namespace TileXR { #ifdef TILEXR_CCU_TESTING constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = + 7U + 64U * 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; @@ -124,6 +127,20 @@ bool UseCcuResourceWindowForMemoryCopy() return value != nullptr && value[0] != '\0' && value[0] != '0'; } +uint16_t SelectSyncXnPingMask(const char *envName) +{ + const char *text = std::getenv(envName); + if (text == nullptr || text[0] == '\0') { + return 0; + } + char *end = nullptr; + const unsigned long parsed = std::strtoul(text, &end, 0); + if (end == text || *end != '\0' || parsed == 0 || parsed > 0xffffUL) { + return 0; + } + return static_cast(parsed); +} + void TraceDirectCcuProcessMemoryToken( uint64_t addr, uint64_t bytes, @@ -613,11 +630,13 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( if (peerResources.localXnCount == 0 || peerResources.remoteXnCount == 0 || peerResources.localWaitCkeCount == 0 || + peerResources.remoteNotifyCkeCount == 0 || peerResources.channelCount == 0 || peerLocalXnOffset >= peerResources.localXnCount || selectedRemoteXnOffset >= peerResources.remoteXnCount || peerLocalIndex >= peerResources.channelCount || - peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount) { + peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount || + peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount) { if (report != nullptr) { report->message = "peer direct CCU local XN/CKE resources are incomplete"; } @@ -643,7 +662,8 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( const bool notifyCkeOwnerVerified = static_cast(remoteNotifyCke) >= peerResources.localWaitCkeStartId && static_cast(remoteNotifyCke) < - static_cast(peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount; + static_cast(peerResources.localWaitCkeStartId) + + peerResources.localWaitCkeCount; const bool localChannelOwnerVerified = allocation.channels.num != 0 && peerLocalXnOwnerVerified && @@ -672,6 +692,24 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( uint32_t memoryTokenId, uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) +{ + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 0, + peerRank, + remoteCcuVa, + memoryTokenId, + rawMemoryTokenId, + memoryTokenValue); + directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; +} + +void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + uint32_t syncRouteIndex, + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue) { directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; directCcuRemoteRouteMemoryOverride_.peerRank = peerRank; @@ -681,12 +719,16 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( directCcuRemoteRouteMemoryOverride_.memoryTokenValue = memoryTokenValue; directCcuRemoteRouteMemoryOverrideValid_ = remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; + directCcuRemoteRouteMemoryOverrideAllRoutes_ = false; + directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = syncRouteIndex; } void TileXRCcuCollectivePlanner::ClearDirectCcuRemoteRouteMemoryOverride() { directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; directCcuRemoteRouteMemoryOverrideValid_ = false; + directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; } void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( @@ -695,14 +737,19 @@ void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( if (!directCcuRemoteRouteMemoryOverrideValid_ || remoteCcuBuffers == nullptr) { return; } + uint32_t routeIndex = 0; for (auto &remoteCcuBuffer : *remoteCcuBuffers) { - if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank) { + if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank || + (!directCcuRemoteRouteMemoryOverrideAllRoutes_ && + routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_)) { + ++routeIndex; continue; } remoteCcuBuffer.remoteCcuVa = directCcuRemoteRouteMemoryOverride_.remoteCcuVa; remoteCcuBuffer.memoryTokenId = directCcuRemoteRouteMemoryOverride_.memoryTokenId; remoteCcuBuffer.rawMemoryTokenId = directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId; remoteCcuBuffer.memoryTokenValue = directCcuRemoteRouteMemoryOverride_.memoryTokenValue; + ++routeIndex; } } #endif @@ -899,6 +946,8 @@ int TileXRCcuCollectivePlanner::PrepareSignalWait( TileXRCcuDirectSignalWaitSpec signalWait; signalWait.role = ToDirectSignalWaitProgramRole(request.role); + signalWait.overrideBarrierMode = request.overrideBarrierMode; + signalWait.barrierMode = request.barrierMode; ret = TileXRCcuRunDirectSignalWaitInstallAttempt(options, signalWait, &plan->attempt, report); if (ret != TILEXR_SUCCESS) { *plan = TileXRCcuSignalWaitPlan {}; @@ -1143,6 +1192,358 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuMemoryCopyInstallAttempt( ClearDirectCcuRemoteRouteMemoryOverride(); return ret; } + +int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU alltoall install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for alltoall install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before alltoall buffer registration"; + } + return ret; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU alltoall local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU alltoall peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU alltoall remote destination buffer"; + } + return ret; + } + + TileXRCcuDirectAllToAll2RankSpec alltoall; + alltoall.localRank = static_cast(rank); + alltoall.localSendAddr = localEndpoint.sourceAddr; + alltoall.localSendToken = localEndpoint.sourceToken; + alltoall.localRecvAddr = localEndpoint.destinationAddr; + alltoall.localRecvToken = localEndpoint.destinationToken; + alltoall.remoteSendAddr = peerEndpoint.sourceAddr; + alltoall.remoteSendToken = peerEndpoint.sourceToken; + alltoall.remoteRecvAddr = remoteImportRequest.addr; + alltoall.remoteRecvToken = + TileXRCcuPackMemoryToken(remoteImportRequest.tokenId, remoteImportRequest.tokenValue, true); + alltoall.bytes = bytes; + alltoall.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + alltoall.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + + if (TraceDirectCcuMemoryCopy()) { + std::cerr << "TileXRDirectCcuTrace alltoallEndpoint" + << " rank=" << rank + << " peerRank=" << peerRank + << " localSendAddr=0x" << std::hex << alltoall.localSendAddr + << " localSendToken=0x" << alltoall.localSendToken + << " localRecvAddr=0x" << alltoall.localRecvAddr + << " localRecvToken=0x" << alltoall.localRecvToken + << " remoteSendAddr=0x" << alltoall.remoteSendAddr + << " remoteSendToken=0x" << alltoall.remoteSendToken + << " remoteRecvAddr=0x" << alltoall.remoteRecvAddr + << " remoteRecvToken=0x" << alltoall.remoteRecvToken + << " bytes=0x" << alltoall.bytes + << std::dec << std::endl; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 3; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-alltoall"; + } + + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 0U, + peerRank, + importedRemoteBuffer.targetSegVa, + remoteImportRequest.tokenId, + remoteImportRequest.rawTokenId, + remoteImportRequest.tokenValue); + ret = TileXRCcuRunDirectAllToAll2RankInstallAttempt(next, alltoall, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} + +int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU SyncXn ping install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU SyncXn ping endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int ret = session.RefreshDirectCcuBasicInfo(installDieId); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return ret; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for SyncXn ping install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before SyncXn ping"; + } + return ret; + } + + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU SyncXn ping local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather( + &localEndpoint, + sizeof(localEndpoint), + allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU SyncXn ping peer endpoints"; + } + return ret; + } + const DirectCcuMemoryCopyEndpoint &peerEndpoint = allEndpoints[peerRank]; + if (peerEndpoint.valid == 0 || peerEndpoint.bytes != bytes) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU SyncXn ping peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; + TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; + ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU SyncXn ping remote endpoint buffer"; + } + return ret; + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 1; + next.syncInstructionCount = std::max( + next.syncInstructionCount, + TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-sync-xn-ping"; + } + + TileXRCcuDirectSyncXnPingSpec syncXnPing; + syncXnPing.localRank = static_cast(rank); + syncXnPing.peerRank = peerRank; + syncXnPing.payload = 0x54585253594e0000ULL | static_cast(rank & 0xffff); + syncXnPing.remoteNotifyMask = + SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK"); + syncXnPing.localWaitMask = + SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK"); + + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + 0U, + peerRank, + importedRemoteBuffer.targetSegVa, + remoteImportRequest.tokenId, + remoteImportRequest.rawTokenId, + remoteImportRequest.tokenValue); + ret = TileXRCcuRunDirectSyncXnPingInstallAttempt(next, syncXnPing, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} #endif int TileXRCcuCollectivePlanner::RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session) diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index 9c512711..f09489ca 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -63,6 +63,24 @@ class TileXRCcuCollectivePlanner { TileXRCcuMemoryCopyDirection direction, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuAllToAll2RankInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuSyncXnPingInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t bytes, + uint32_t peerRank, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); #endif int RefreshDirectCcuLowerLayerPlan(TileXRCcuRuntimeSession &session); bool HasDirectCcuLowerLayerPlan() const; @@ -94,6 +112,13 @@ class TileXRCcuCollectivePlanner { uint32_t memoryTokenId, uint32_t rawMemoryTokenId, uint32_t memoryTokenValue); + void SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( + uint32_t syncRouteIndex, + uint32_t peerRank, + uint64_t remoteCcuVa, + uint32_t memoryTokenId, + uint32_t rawMemoryTokenId, + uint32_t memoryTokenValue); void ClearDirectCcuRemoteRouteMemoryOverride(); void ApplyDirectCcuRemoteRouteMemoryOverride( std::vector *remoteCcuBuffers) const; @@ -117,6 +142,8 @@ class TileXRCcuCollectivePlanner { #ifdef TILEXR_CCU_TESTING TileXRCcuRemoteCcuBufferInfo directCcuRemoteRouteMemoryOverride_ = {}; bool directCcuRemoteRouteMemoryOverrideValid_ = false; + bool directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + uint32_t directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; #endif }; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index 3f0fc29c..d13fc152 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -5,6 +5,7 @@ #include "ccu/tilexr_ccu_direct_orchestrator.h" +#include "ccu/tilexr_ccu_alltoall_program.h" #include "ccu/tilexr_ccu_runtime.h" #include @@ -35,9 +36,13 @@ constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALUE_MASK = 0xffffffffULL; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = + 7U + 64U * 7U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; void ResetReport(TileXRCcuDirectInstallReport* report) { @@ -753,6 +758,52 @@ int ConfigureDirectMemoryCopyResources( return TILEXR_SUCCESS; } +int ConfigureDirectAllToAll2RankResources( + const TileXRCcuDirectInstallOptions& options, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT || + attempt->plan.taskWindows.size() != 1) { + if (report != nullptr) { + report->message = "alltoall direct CCU plan requires three sync resources and one task"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (options.gsaStartId == 0 || attempt->resourceSpec.gsaCount < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "alltoall direct CCU requires a kernel-local GSA resource window"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const uint8_t dieId = attempt->specInfo.dieId; + const uint16_t localXnStart = attempt->allocation.localXn.startId; + if (!ContainsRange( + attempt->resourceSpec.xnStartId, + attempt->resourceSpec.xnCount, + localXnStart, + TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)) { + if (report != nullptr) { + report->message = "alltoall direct CCU local XN window is too small"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->allocation.localXn = + MakeRange(dieId, localXnStart, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT)); + attempt->allocation.localGsa = + MakeRange(dieId, options.gsaStartId, static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT)); + attempt->plan.kernelLocalXn = attempt->allocation.localXn; + attempt->plan.kernelLocalGsa = attempt->allocation.localGsa; + attempt->plan.taskWindows[0].instCnt = + static_cast(std::max( + attempt->plan.taskWindows[0].instCnt, + TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT)); + return TILEXR_SUCCESS; +} + int BuildDirectMemoryCopyLaunchPackage( const TileXRCcuDirectMemoryCopySpec& memoryCopy, TileXRCcuDirectInstallAttempt* attempt, @@ -853,6 +904,173 @@ int BuildDirectMemoryCopyLaunchPackage( return TILEXR_SUCCESS; } +int BuildDirectAllToAll2RankLaunchPackage( + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT || + attempt->plan.kernelLocalXn.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_XN_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall GSA/XN resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]; + const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]; + const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]; + const bool preSyncOnCopyRoute = + std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE") != nullptr; + const bool preSyncPeerLocalXn = + std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_PEER_LOCAL_XN") != nullptr; + const uint16_t preSyncRemoteAddrXn = + preSyncPeerLocalXn ? preResource.localXn : preResource.remoteXn; + + TileXRCcuAllToAll2RankProgramSpec alltoallSpec; + alltoallSpec.localRank = alltoall.localRank; + alltoallSpec.localSendAddr = alltoall.localSendAddr; + alltoallSpec.localSendToken = alltoall.localSendToken; + alltoallSpec.localRecvAddr = alltoall.localRecvAddr; + alltoallSpec.localRecvToken = alltoall.localRecvToken; + alltoallSpec.remoteSendAddr = alltoall.remoteSendAddr; + alltoallSpec.remoteSendToken = alltoall.remoteSendToken; + alltoallSpec.remoteRecvAddr = alltoall.remoteRecvAddr; + alltoallSpec.remoteRecvToken = alltoall.remoteRecvToken; + alltoallSpec.bytes = alltoall.bytes; + alltoallSpec.memorySliceBytes = alltoall.memorySliceBytes; + alltoallSpec.memSlicePerBlock = alltoall.memSlicePerBlock; + alltoallSpec.localGsa = attempt->plan.kernelLocalGsa.startId; + alltoallSpec.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + alltoallSpec.localXn = attempt->plan.kernelLocalXn.startId; + alltoallSpec.remoteXn = static_cast(attempt->plan.kernelLocalXn.startId + 1U); + alltoallSpec.lengthXn = static_cast(attempt->plan.kernelLocalXn.startId + 2U); + alltoallSpec.preSyncLocalAddrXn = + preSyncOnCopyRoute ? copyResource.localXn : preResource.localXn; + alltoallSpec.preSyncLocalTokenXn = postResource.localXn; + alltoallSpec.preSyncLocalMarkerXn = copyResource.localXn; + alltoallSpec.preSyncRemoteMarkerXn = copyResource.remoteXn; + alltoallSpec.preSyncMarkerArgIndex = 0; + alltoallSpec.preSyncMarkerEnabled = true; + alltoallSpec.channelId = copyResource.channelId; + alltoallSpec.preSyncChannelId = + preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId; + alltoallSpec.preSyncTokenChannelId = preResource.channelId; + alltoallSpec.copyChannelId = copyResource.channelId; + alltoallSpec.postSyncChannelId = postResource.channelId; + alltoallSpec.preSyncRemoteAddrXn = + preSyncOnCopyRoute ? copyResource.localXn : preSyncRemoteAddrXn; + alltoallSpec.preSyncRemoteTokenXn = + preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn; + alltoallSpec.preSyncRemoteNotifyCke = + preSyncOnCopyRoute ? attempt->allocation.remoteNotifyCke.startId : preResource.notifyCke; + alltoallSpec.preSyncLocalWaitCke = + preSyncOnCopyRoute + ? (copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke) + : (preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke); + alltoallSpec.preSyncRemoteTokenNotifyCke = preResource.notifyCke; + alltoallSpec.preSyncTokenLocalWaitCke = + preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke; + alltoallSpec.copyCompletionCke = + copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke; + alltoallSpec.postSyncRemoteNotifyCke = postResource.notifyCke; + alltoallSpec.postSyncLocalWaitCke = + postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke; + alltoallSpec.sourceCke = preResource.sourceCke; + alltoallSpec.ckeMask = preResource.remoteNotifyMask == 0 ? 1U : preResource.remoteNotifyMask; + alltoallSpec.preSyncNotify = std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC") == nullptr; + alltoallSpec.preSyncWait = std::getenv("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT") == nullptr; + alltoallSpec.postSyncNotify = false; + alltoallSpec.postSyncWait = false; + alltoallSpec.emitFinish = false; + + if (DirectTraceEnabled()) { + std::cerr << "TileXRDirectCcuTrace alltoallSpec" + << " direction=LocalToRemote" + << " localRank=" << alltoallSpec.localRank + << " localGsa=" << alltoallSpec.localGsa + << " localXn=" << alltoallSpec.localXn + << " remoteGsa=" << alltoallSpec.remoteGsa + << " remoteXn=" << alltoallSpec.remoteXn + << " lengthXn=" << alltoallSpec.lengthXn + << " preLocalAddrXn=" << alltoallSpec.preSyncLocalAddrXn + << " preLocalTokenXn=" << alltoallSpec.preSyncLocalTokenXn + << " preChannelId=" << alltoallSpec.preSyncChannelId + << " preTokenChannelId=" << alltoallSpec.preSyncTokenChannelId + << " copyChannelId=" << alltoallSpec.copyChannelId + << " postChannelId=" << alltoallSpec.postSyncChannelId + << " preNotifyCke=" << alltoallSpec.preSyncRemoteNotifyCke + << " preTokenNotifyCke=" << alltoallSpec.preSyncRemoteTokenNotifyCke + << " preTokenWaitCke=" << alltoallSpec.preSyncTokenLocalWaitCke + << " preRemoteAddrXn=" << alltoallSpec.preSyncRemoteAddrXn + << " preRemoteTokenXn=" << alltoallSpec.preSyncRemoteTokenXn + << " copyCompletionCke=" << alltoallSpec.copyCompletionCke + << " postNotifyCke=" << alltoallSpec.postSyncRemoteNotifyCke + << " preSyncNotify=" << (alltoallSpec.preSyncNotify ? 1 : 0) + << " preSyncWait=" << (alltoallSpec.preSyncWait ? 1 : 0) + << " preSyncOnCopyRoute=" << (preSyncOnCopyRoute ? 1 : 0) + << " preSyncPeerLocalXn=" << (preSyncPeerLocalXn ? 1 : 0) + << " postSyncNotify=" << (alltoallSpec.postSyncNotify ? 1 : 0) + << " postSyncWait=" << (alltoallSpec.postSyncWait ? 1 : 0) + << " emitFinish=" << (alltoallSpec.emitFinish ? 1 : 0) + << " localSendAddr=0x" << std::hex << alltoallSpec.localSendAddr + << " localRecvAddr=0x" << alltoallSpec.localRecvAddr + << " remoteSendAddr=0x" << alltoallSpec.remoteSendAddr + << " remoteRecvAddr=0x" << alltoallSpec.remoteRecvAddr + << " bytes=0x" << alltoallSpec.bytes + << std::dec << std::endl; + } + + TileXRCcuProgram program; + TileXRCcuAllToAllProgramReport alltoallReport; + if (TileXRCcuBuildAllToAll2RankProgram(alltoallSpec, &program.sync, &alltoallReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = alltoallReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (program.sync.empty() || program.sync.size() > std::numeric_limits::max()) { + if (report != nullptr) { + report->message = "invalid direct CCU alltoall instruction count"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + uint32_t SignalWaitInstructionCount(TileXRCcuSignalWaitProgramRole role) { if (role == TileXRCcuSignalWaitProgramRole::Wait) { @@ -874,6 +1092,39 @@ TileXRCcuBarrierMode SignalWaitBarrierMode(TileXRCcuSignalWaitProgramRole role) TileXRCcuBarrierMode::SyncCke; } +TileXRCcuBarrierMode EffectiveSignalWaitBarrierMode(const TileXRCcuDirectSignalWaitSpec& signalWait) +{ + return signalWait.overrideBarrierMode ? signalWait.barrierMode : SignalWaitBarrierMode(signalWait.role); +} + +uint32_t BarrierInstructionCount(TileXRCcuBarrierMode mode) +{ + switch (mode) { + case TileXRCcuBarrierMode::SyncCke: + case TileXRCcuBarrierMode::SyncCkeSetWait: + return 3U; + case TileXRCcuBarrierMode::SyncXnLoadPostOnly: + return 2U; + case TileXRCcuBarrierMode::SyncXn: + case TileXRCcuBarrierMode::LocalCke: + return 2U; + case TileXRCcuBarrierMode::SyncXnPostOnly: + case TileXRCcuBarrierMode::SyncCkePostOnly: + case TileXRCcuBarrierMode::LocalCkePostOnly: + return 1U; + default: + return 2U; + } +} + +uint32_t SignalWaitInstructionCount(const TileXRCcuDirectSignalWaitSpec& signalWait) +{ + if (signalWait.overrideBarrierMode) { + return BarrierInstructionCount(signalWait.barrierMode); + } + return SignalWaitInstructionCount(signalWait.role); +} + int BuildDirectSignalWaitLaunchPackage( const TileXRCcuDirectSignalWaitSpec& signalWait, TileXRCcuDirectInstallAttempt* attempt, @@ -902,7 +1153,31 @@ int BuildDirectSignalWaitLaunchPackage( TileXRCcuProgram program; TileXRCcuBarrierProgramReport signalWaitReport; - if (TileXRCcuBuildSignalWaitProgram(spec, &program.sync, &signalWaitReport) != TILEXR_SUCCESS) { + if (signalWait.overrideBarrierMode) { + TileXRCcuBarrierSyncSpec barrier; + barrier.channelId = spec.channelId; + barrier.remoteXn = spec.remoteXn; + barrier.localXn = spec.localXn; + barrier.remoteNotifyCke = spec.remoteNotifyCke; + barrier.remoteNotifyMask = spec.remoteNotifyMask; + barrier.localWaitCke = spec.localWaitCke; + barrier.localWaitMask = spec.localWaitMask; + barrier.sourceCke = spec.sourceCke; + barrier.sourceCkeMask = spec.sourceCkeMask; + barrier.clearLocalWait = spec.clearLocalWait; + std::vector barriers; + barriers.push_back(barrier); + if (TileXRCcuBuildBarrierProgram( + barriers, + &program.sync, + &signalWaitReport, + signalWait.barrierMode) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = signalWaitReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } else if (TileXRCcuBuildSignalWaitProgram(spec, &program.sync, &signalWaitReport) != TILEXR_SUCCESS) { if (report != nullptr) { report->message = signalWaitReport.message; } @@ -943,6 +1218,132 @@ int BuildDirectSignalWaitLaunchPackage( return TILEXR_SUCCESS; } +int BuildDirectSyncXnPingLaunchPackage( + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || attempt->plan.syncResources.size() != 1 || attempt->plan.taskWindows.size() != 1) { + if (report != nullptr) { + report->message = "missing direct CCU SyncXn ping producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (syncXnPing.localRank > 1U || syncXnPing.peerRank > 1U || syncXnPing.localRank == syncXnPing.peerRank) { + if (report != nullptr) { + report->message = "direct CCU SyncXn ping requires two distinct rank ids"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; + const uint16_t localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; + const uint16_t defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank); + const uint16_t defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank); + const uint16_t remoteNotifyMask = + syncXnPing.remoteNotifyMask == 0 ? defaultRemoteNotifyMask : syncXnPing.remoteNotifyMask; + const uint16_t localWaitMask = + syncXnPing.localWaitMask == 0 ? defaultLocalWaitMask : syncXnPing.localWaitMask; + if (resource.localXn == 0 || resource.remoteXn == 0 || resource.channelId == 0 || + resource.notifyCke == 0 || localWaitCke == 0 || resource.sourceCke == 0) { + if (report != nullptr) { + report->message = "missing direct CCU SyncXn ping XN/CKE/channel resource"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuProgram program; + program.sync.reserve(TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(resource.localXn, syncXnPing.payload, 0, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping payload load"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuSyncXnSpec notify; + notify.remoteXn = resource.remoteXn; + notify.localXn = resource.localXn; + notify.channelId = resource.channelId; + notify.notifyCke = resource.notifyCke; + notify.notifyMask = remoteNotifyMask; + notify.clearWait = true; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping notify"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuCkeSpec source; + source.ckeId = resource.sourceCke; + source.mask = remoteNotifyMask; + source.clearWait = true; + if (TileXRCcuEncodeSetCke(source, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping source CKE set"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuSyncCkeSpec syncCke; + syncCke.remoteCke = resource.notifyCke; + syncCke.localCke = resource.sourceCke; + syncCke.localCkeMask = remoteNotifyMask; + syncCke.channelId = resource.channelId; + syncCke.clearWait = true; + if (TileXRCcuEncodeSyncCke(syncCke, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping SyncCke notify"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + + TileXRCcuCkeSpec wait; + wait.waitCkeId = localWaitCke; + wait.waitMask = localWaitMask; + wait.clearWait = true; + if (TileXRCcuEncodeSetCke(wait, &instr) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = "failed to encode direct CCU SyncXn ping wait"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + program.sync.push_back(instr); + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + void FillReportFromAttempt(const TileXRCcuDirectInstallAttempt& attempt, TileXRCcuDirectInstallReport* report) { if (report == nullptr) { @@ -1118,7 +1519,9 @@ int TileXRCcuSubmitPreparedTasks( int RunDirectInstallAttemptImpl( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectMemoryCopySpec* memoryCopy, + const TileXRCcuDirectAllToAll2RankSpec* alltoall, const TileXRCcuDirectSignalWaitSpec* signalWait, + const TileXRCcuDirectSyncXnPingSpec* syncXnPing, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { @@ -1163,18 +1566,27 @@ int RunDirectInstallAttemptImpl( ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); - attempt->resourceRequest.sqeArgCount = memoryCopy == nullptr && signalWait == nullptr ? options.sqeArgCount : 0U; + const bool customProgram = + memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr || syncXnPing != nullptr; + attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = - memoryCopy == nullptr && signalWait == nullptr ? options.syncResourceCount : 1U; + alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : + customProgram ? 1U : options.syncResourceCount; attempt->resourceRequest.syncInstructionCount = memoryCopy != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : + alltoall != nullptr ? + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : signalWait != nullptr ? - std::max(options.syncInstructionCount, SignalWaitInstructionCount(signalWait->role)) : + std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : + syncXnPing != nullptr ? + std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT) : options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.barrierMode = - signalWait == nullptr ? options.barrierMode : SignalWaitBarrierMode(signalWait->role); + attempt->resourceRequest.barrierMode = + alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); TileXRCcuResourceAllocator allocator; if (allocator.Init(attempt->resourceSpec) != TILEXR_SUCCESS) { @@ -1201,9 +1613,21 @@ int RunDirectInstallAttemptImpl( "failed to configure direct CCU memory copy resources" : report->message); } + } else if (alltoall != nullptr) { + ret = ConfigureDirectAllToAll2RankResources(options, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to configure direct CCU alltoall resources" : + report->message); + } } attempt->plan.barrierMode = - signalWait == nullptr ? attempt->plan.barrierMode : SignalWaitBarrierMode(signalWait->role); + alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); if (ret != TILEXR_SUCCESS) { @@ -1224,7 +1648,7 @@ int RunDirectInstallAttemptImpl( "failed to reconcile direct CCU lower-layer peer resources" : report->message); } - if (memoryCopy == nullptr) { + if (!customProgram) { ret = PopulateHcommStyleSqeTaskArgs(attempt, report); if (ret != TILEXR_SUCCESS) { return Fail(attempt, report, "failed to populate direct CCU SQE task arguments"); @@ -1234,17 +1658,25 @@ int RunDirectInstallAttemptImpl( TileXRCcuLaunchPackageReport packageReport; ret = memoryCopy != nullptr ? BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report) : + alltoall != nullptr ? + BuildDirectAllToAll2RankLaunchPackage(*alltoall, attempt, report) : signalWait != nullptr ? BuildDirectSignalWaitLaunchPackage(*signalWait, attempt, report) : + syncXnPing != nullptr ? + BuildDirectSyncXnPingLaunchPackage(*syncXnPing, attempt, report) : TileXRCcuBuildLaunchPackage(attempt->plan, &attempt->package, &packageReport); if (ret != TILEXR_SUCCESS) { return Fail( attempt, report, - memoryCopy == nullptr && signalWait == nullptr ? packageReport.message : + !customProgram ? packageReport.message : (report == nullptr || report->message.empty() ? (memoryCopy != nullptr ? "failed to build direct CCU memory copy launch package" : + alltoall != nullptr ? + "failed to build direct CCU alltoall launch package" : + syncXnPing != nullptr ? + "failed to build direct CCU SyncXn ping launch package" : "failed to build direct CCU signal/wait launch package") : report->message)); } @@ -1308,7 +1740,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1327,7 +1759,31 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, attempt, report); +} + +int TileXRCcuRunDirectAllToAll2RankInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (alltoall.localRank > 1U || + alltoall.localSendAddr == 0 || alltoall.localSendToken == 0 || + alltoall.localRecvAddr == 0 || alltoall.localRecvToken == 0 || + alltoall.remoteSendAddr == 0 || alltoall.remoteSendToken == 0 || + alltoall.remoteRecvAddr == 0 || alltoall.remoteRecvToken == 0 || + alltoall.bytes == 0 || alltoall.memorySliceBytes != TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES || + alltoall.memSlicePerBlock == 0 || + alltoall.memSlicePerBlock > TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK) { + ResetReport(report); + ClearAttempt(attempt); + if (report != nullptr) { + report->message = "invalid direct CCU alltoall address/token/slice inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectSignalWaitInstallAttempt( @@ -1336,7 +1792,16 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, &signalWait, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, nullptr, attempt, report); +} + +int TileXRCcuRunDirectSyncXnPingInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index 7a7271cf..dd5a2ee5 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -8,6 +8,7 @@ #include "ccu/tilexr_ccu_install_provider.h" #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" +#include "ccu/tilexr_ccu_alltoall_program.h" #include "ccu/tilexr_ccu_memory_program.h" #include "ccu/tilexr_ccu_signal_wait_program.h" #include "ccu/tilexr_ccu_specs.h" @@ -69,8 +70,33 @@ struct TileXRCcuDirectMemoryCopySpec { uint64_t lengthBytes = 0; }; +struct TileXRCcuDirectAllToAll2RankSpec { + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t remoteSendAddr = 0; + uint64_t remoteSendToken = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; + uint64_t bytes = 0; + uint32_t memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; +}; + struct TileXRCcuDirectSignalWaitSpec { TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; + bool overrideBarrierMode = false; + TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncCke; +}; + +struct TileXRCcuDirectSyncXnPingSpec { + uint32_t localRank = 0; + uint32_t peerRank = 1; + uint64_t payload = 0; + uint16_t remoteNotifyMask = 0; + uint16_t localWaitMask = 0; }; struct TileXRCcuDirectInstallAttempt { @@ -129,12 +155,24 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectAllToAll2RankInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAll2RankSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuRunDirectSignalWaitInstallAttempt( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectSignalWaitSpec& signalWait, TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectSyncXnPingInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectSyncXnPingSpec& syncXnPing, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt); int TileXRCcuSubmitPreparedTasks( diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp index f8cf6a49..71945955 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp @@ -349,6 +349,65 @@ int TileXRCcuDriverAdapter::ReadMissionContext( return TILEXR_SUCCESS; } +int TileXRCcuDriverAdapter::ReadResourceRange( + uint8_t dieId, + uint32_t opcode, + uint32_t startId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + ResetReport(report); + if (values == nullptr || count == 0) { + return Fail(report, "invalid CCU resource readback range"); + } + + uint32_t remaining = count; + uint32_t offset = startId; + uint32_t outputOffset = 0; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, opcode, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * sizeof(uint64_t); + + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, opcode, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + for (uint32_t i = 0; i < batch; ++i) { + std::memcpy(&values[outputOffset + i], &out.data.dataInfo.dataArray[i], sizeof(uint64_t)); + } + remaining -= batch; + offset += batch; + outputOffset += batch; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDriverAdapter::ReadXnRange( + uint8_t dieId, + uint32_t startXnId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + return ReadResourceRange(dieId, TILEXR_CCU_U_OP_GET_XN, startXnId, values, count, report); +} + +int TileXRCcuDriverAdapter::ReadCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const +{ + return ReadResourceRange(dieId, TILEXR_CCU_U_OP_GET_CKE, startCkeId, values, count, report); +} + int TileXRCcuDriverAdapter::InstallInstructions( uint8_t dieId, uint16_t instructionStartId, diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.h b/src/comm/ccu/tilexr_ccu_driver_adapter.h index 53ed1be3..ce6ce16a 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.h +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.h @@ -17,6 +17,8 @@ namespace TileXR { constexpr uint32_t TILEXR_CCU_U_OP_GET_BASIC_INFO = 11; constexpr uint32_t TILEXR_CCU_U_OP_GET_DIE_WORKING = 15; constexpr uint32_t TILEXR_CCU_U_OP_GET_INSTRUCTION = 201; +constexpr uint32_t TILEXR_CCU_U_OP_GET_XN = 203; +constexpr uint32_t TILEXR_CCU_U_OP_GET_CKE = 204; constexpr uint32_t TILEXR_CCU_U_OP_GET_MISSION_CTX = 208; constexpr uint32_t TILEXR_CCU_U_OP_SET_MSID_TOKEN = 53; constexpr uint32_t TILEXR_CCU_U_OP_SET_TASKKILL = 54; @@ -87,6 +89,18 @@ class TileXRCcuDriverAdapter { void* missionContext, uint32_t missionContextBytes, TileXRCcuDriverAdapterReport* report) const; + int ReadXnRange( + uint8_t dieId, + uint32_t startXnId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; + int ReadCkeRange( + uint8_t dieId, + uint32_t startCkeId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; int InstallInstructions( uint8_t dieId, uint16_t instructionStartId, @@ -145,6 +159,13 @@ class TileXRCcuDriverAdapter { const TileXRCcuCustomChannelIn& in, TileXRCcuCustomChannelOut* out, TileXRCcuDriverAdapterReport* report) const; + int ReadResourceRange( + uint8_t dieId, + uint32_t opcode, + uint32_t startId, + uint64_t* values, + uint32_t count, + TileXRCcuDriverAdapterReport* report) const; uint32_t devicePhyId_ = 0; TileXRCcuCustomChannelFn customChannel_ = nullptr; diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index 054e9272..fe0825af 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -118,6 +118,52 @@ bool RangeContains(uint32_t start, uint32_t count, uint32_t value) return count != 0 && value >= start && value < start + count; } +void NormalizeVerifiedEndpointRouteJettyWindow(TileXRCcuLowerLayerTransportSnapshot* snapshot) +{ + if (snapshot == nullptr || snapshot->routes.empty()) { + return; + } + + uint16_t minExplicitStart = 0; + uint32_t explicitEnd = 0; + uint32_t explicitStartCount = 0; + for (const auto& route : snapshot->routes) { + if (route.startJettyId == 0) { + continue; + } + if (explicitStartCount == 0) { + minExplicitStart = route.startJettyId; + } else { + minExplicitStart = std::min(minExplicitStart, route.startJettyId); + } + explicitEnd = std::max(explicitEnd, static_cast(route.startJettyId) + 1U); + ++explicitStartCount; + } + + if (minExplicitStart != 0) { + snapshot->startJettyId = minExplicitStart; + } + + uint32_t requiredJettyCount = static_cast(snapshot->routes.size()); + if (snapshot->startJettyId != 0) { + uint32_t end = static_cast(snapshot->startJettyId) + snapshot->routes.size(); + for (uint32_t i = 0; i < snapshot->routes.size(); ++i) { + const uint32_t start = snapshot->routes[i].startJettyId == 0 ? + static_cast(snapshot->startJettyId) + i : + snapshot->routes[i].startJettyId; + end = std::max(end, start + 1U); + } + requiredJettyCount = std::max( + requiredJettyCount, + end - static_cast(snapshot->startJettyId)); + } else if (explicitEnd != 0 && minExplicitStart != 0) { + requiredJettyCount = std::max( + requiredJettyCount, + explicitEnd - static_cast(minExplicitStart)); + } + snapshot->pfeJettyCount = CheckedU16(std::max(snapshot->pfeJettyCount, requiredJettyCount)); +} + uint32_t SelectLowerLayerPfeOffset(uint8_t dieId, uint32_t pfeId) { if (LowerLayerEnvEquals("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE", "hcomm_die")) { @@ -531,6 +577,7 @@ int TileXRCcuBuildLowerLayerTransportTemplate( const uint32_t requiredJettyCount = verifiedJettyEnd - verifiedStartJettyId; result.pfeJettyCount = CheckedU16(std::max(result.pfeJettyCount, requiredJettyCount)); } + NormalizeVerifiedEndpointRouteJettyWindow(&result); *snapshot = result; FillTemplateReport(*snapshot, report); @@ -575,10 +622,12 @@ int TileXRCcuOverlayVerifiedEndpointRoutes( routeIt->localDoorbellTokenId = verified.localDoorbellTokenId; routeIt->localDoorbellTokenValue = verified.localDoorbellTokenValue; routeIt->localSqDepth = verified.localSqDepth; + routeIt->startJettyId = verified.startJettyId; routeIt->endpointRouteVerified = true; routeIt->channelResourceOwnerVerified = verified.channelResourceOwnerVerified; routeIt->transportResourceExchangeVerified = verified.transportResourceExchangeVerified; } + NormalizeVerifiedEndpointRouteJettyWindow(snapshot); FillTemplateReport(*snapshot, report); return TILEXR_SUCCESS; @@ -602,28 +651,31 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( return Fail(plan, report, "too many lower-layer CCU transport routes"); } + TileXRCcuLowerLayerTransportSnapshot normalized = snapshot; + NormalizeVerifiedEndpointRouteJettyWindow(&normalized); + TileXRCcuLowerLayerPlanSpec spec; - spec.msidToken = snapshot.msidToken; - spec.pfe.dieId = snapshot.dieId; - spec.pfe.pfeOffset = snapshot.pfeOffset; - spec.pfe.startJettyId = snapshot.startJettyId; - spec.pfe.jettyCount = snapshot.pfeJettyCount; - spec.pfe.startLocalJettyCtxId = snapshot.startLocalJettyCtxId; - spec.xnClear.dieId = snapshot.dieId; - spec.xnClear.startXnId = snapshot.xnStartId; - spec.xnClear.count = snapshot.xnCount; - spec.xnClear.valid = snapshot.xnCount != 0; - spec.ckeClear.dieId = snapshot.dieId; - spec.ckeClear.startCkeId = snapshot.ckeStartId; - spec.ckeClear.count = snapshot.ckeCount; - spec.ckeClear.valid = snapshot.ckeCount != 0; + spec.msidToken = normalized.msidToken; + spec.pfe.dieId = normalized.dieId; + spec.pfe.pfeOffset = normalized.pfeOffset; + spec.pfe.startJettyId = normalized.startJettyId; + spec.pfe.jettyCount = normalized.pfeJettyCount; + spec.pfe.startLocalJettyCtxId = normalized.startLocalJettyCtxId; + spec.xnClear.dieId = normalized.dieId; + spec.xnClear.startXnId = normalized.xnStartId; + spec.xnClear.count = normalized.xnCount; + spec.xnClear.valid = normalized.xnCount != 0; + spec.ckeClear.dieId = normalized.dieId; + spec.ckeClear.startCkeId = normalized.ckeStartId; + spec.ckeClear.count = normalized.ckeCount; + spec.ckeClear.valid = normalized.ckeCount != 0; uint32_t routeIndex = 0; - for (const auto& route : snapshot.routes) { + for (const auto& route : normalized.routes) { TileXRCcuLowerLayerJettySpec jetty; - jetty.dieId = snapshot.dieId; - jetty.pfeId = snapshot.pfeId; - jetty.startJettyCtxId = static_cast(snapshot.startLocalJettyCtxId + routeIndex); + jetty.dieId = normalized.dieId; + jetty.pfeId = normalized.pfeId; + jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + routeIndex); jetty.doorbellVa = route.localDoorbellVa == 0 ? route.doorbellVa : route.localDoorbellVa; jetty.doorbellTokenId = route.localDoorbellTokenId == 0 ? route.doorbellTokenId : @@ -636,13 +688,13 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( spec.jettys.push_back(jetty); TileXRCcuLowerLayerChannelSpec channel; - channel.dieId = snapshot.dieId; + channel.dieId = normalized.dieId; channel.channelId = route.channelId; channel.remoteEid = route.remoteEid; channel.tpn = route.tpn; - channel.sourcePfeId = snapshot.pfeId; + channel.sourcePfeId = normalized.pfeId; channel.startJettyId = route.startJettyId == 0 ? - static_cast(snapshot.startJettyId + routeIndex) : + static_cast(normalized.startJettyId + routeIndex) : route.startJettyId; channel.jettyCount = 1; channel.memoryTokenId = route.memoryTokenId; @@ -659,16 +711,16 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( } TileXRCcuRemoteXnBindingProof remoteXn; - remoteXn.dieId = snapshot.dieId; + remoteXn.dieId = normalized.dieId; remoteXn.channelId = static_cast(route.channelId); - remoteXn.localXn = static_cast(snapshot.xnStartId + routeIndex); + remoteXn.localXn = static_cast(normalized.xnStartId + routeIndex); remoteXn.remoteXn = route.remoteXnId; remoteXn.notifyCke = route.remoteNotifyCke == 0 ? - static_cast(snapshot.ckeStartId + routeIndex) : + static_cast(normalized.ckeStartId + routeIndex) : route.remoteNotifyCke; remoteXn.peerRank = route.peerRank; remoteXn.peerExchangeObserved = route.remoteXnId != 0; - remoteXn.localWaitCke = static_cast(snapshot.ckeStartId + routeIndex); + remoteXn.localWaitCke = static_cast(normalized.ckeStartId + routeIndex); remoteXn.endpointRouteVerified = route.endpointRouteVerified; remoteXn.channelResourceOwnerVerified = route.channelResourceOwnerVerified; remoteXn.transportResourceExchangeVerified = route.transportResourceExchangeVerified; @@ -680,8 +732,8 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( if (ret != TILEXR_SUCCESS) { return ret; } - AppendRemoteXnClears(snapshot, plan); - AppendRemoteNotifyCkeClears(snapshot, plan); + AppendRemoteXnClears(normalized, plan); + AppendRemoteNotifyCkeClears(normalized, plan); FillReport(*plan, report); return TILEXR_SUCCESS; } diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index 979798fc..fd4e9b8b 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -77,6 +77,14 @@ constexpr const char* kP2pCcuCopyBytesEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COP constexpr const char* kP2pCcuCopyActiveRankEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK"; constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION"; constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; +constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; +constexpr const char* kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"; +constexpr const char* kAllToAllSingleRouteBidirectionalEnv = + "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL"; +constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; +constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; +constexpr const char* kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"; +constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; @@ -127,6 +135,25 @@ struct P2pCcuCopyState { bool passed = false; }; +struct AllToAllState { + DeviceBuffer source; + DeviceBuffer destination; + std::vector expected; + std::vector observed; + size_t bytes = 0; + int initRet = ACL_SUCCESS; + int readRet = ACL_SUCCESS; + uint32_t mismatchCount = 0; + size_t firstMismatchOffset = 0; + size_t lastMismatchOffset = 0; + uint32_t firstMismatchObserved = 0; + uint32_t firstMismatchExpected = 0; + uint32_t mismatchedBlockCount = 0; + uint32_t firstMismatchedBlock = 0; + uint32_t lastMismatchedBlock = 0; + bool passed = false; +}; + bool EnvFlag(const char* name) { const char* value = std::getenv(name); @@ -142,6 +169,26 @@ bool SignalWaitSmokeEnabled() return EnvFlag(kSignalWaitEnv); } +bool AllToAllSmokeEnabled() +{ + return EnvFlag(kAllToAllEnv); +} + +bool AllToAllLongMissionEnabled() +{ + return EnvFlag(kAllToAllLongMissionEnv); +} + +bool AllToAllSingleRouteBidirectionalEnabled() +{ + return EnvFlag(kAllToAllSingleRouteBidirectionalEnv); +} + +bool SyncXnPingSmokeEnabled() +{ + return EnvFlag(kSyncXnPingEnv); +} + bool BarrierSmokeEnabled() { return EnvFlag(kSignalWaitBarrierEnv); @@ -209,7 +256,8 @@ bool IsP2pCcuCopyActiveRank(int rank) TileXR::TileXRCcuMemoryCopyDirection P2pCcuCopyDirectionFromEnv() { const char* value = std::getenv(kP2pCcuCopyDirectionEnv); - if (value != nullptr && std::string(value) == "local_to_remote") { + const std::string direction = value == nullptr ? "" : std::string(value); + if (direction == "local_to_remote" || direction == "LocalToRemote" || direction == "1") { return TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote; } return TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal; @@ -518,6 +566,181 @@ int CheckP2pCcuCopyState(P2pCcuCopyState* state) return state->passed ? ACL_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; } +size_t AllToAllBytesFromEnv() +{ + return static_cast(EnvInt(kAllToAllBytesEnv, 2 * 1024 * 1024)); +} + +int AllToAllMemSlicePerLoopFromEnv() +{ + return EnvInt(kAllToAllMemSlicePerLoopEnv, 8); +} + +int AllToAllLoopCountFromEnv() +{ + const char* value = std::getenv(kAllToAllLoopCountEnv); + if (value == nullptr || value[0] == '\0') { + return 1; + } + char* end = nullptr; + const long parsed = std::strtol(value, &end, 10); + if (end == value || *end != '\0' || parsed < 1 || parsed > 1024) { + return 0; + } + return static_cast(parsed); +} + +uint64_t BuildAllToAllLoopMarker(int rank, int loopIndex) +{ + return 0x4343554c00000000ULL | + (static_cast(rank & 0xff) << 16U) | + static_cast(loopIndex & 0xffff); +} + +std::vector BuildAllToAllLoopPattern(int rank, int loopIndex, size_t bytes) +{ + std::vector pattern(bytes); + for (size_t i = 0; i < bytes; ++i) { + pattern[i] = static_cast( + (static_cast(rank + 1) * 17U + + static_cast(loopIndex + 1) * 29U + i * 13U) & 0xffU); + } + return pattern; +} + +int InitAllToAllState(int rank, int peer, AllToAllState* state) +{ + if (state == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->bytes = AllToAllBytesFromEnv(); + if (state->bytes != 2U * 1024U * 1024U || AllToAllMemSlicePerLoopFromEnv() != 8) { + state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + return state->initRet; + } + state->expected = BuildP2pCcuCopyPattern(peer, state->bytes); + state->observed.assign(state->bytes, 0); + const std::vector source = BuildP2pCcuCopyPattern(rank, state->bytes); + std::vector destination(state->bytes, 0xa5U); + + int ret = state->source.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = state->destination.Allocate(state->bytes); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->source.ptr, + state->bytes, + source.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + state->initRet = ret; + return ret; + } + ret = aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + state->bytes, + ACL_MEMCPY_HOST_TO_DEVICE); + state->initRet = ret; + return ret; +} + +int ResetAllToAllStateForLoop(int rank, int peer, int loopIndex, AllToAllState* state) +{ + if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || + state->bytes == 0 || loopIndex < 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->expected = BuildAllToAllLoopPattern(peer, loopIndex, state->bytes); + state->observed.assign(state->bytes, 0); + state->readRet = ACL_SUCCESS; + state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + state->passed = false; + + const std::vector source = BuildAllToAllLoopPattern(rank, loopIndex, state->bytes); + const std::vector destination( + state->bytes, + static_cast(0xa5U ^ static_cast(loopIndex))); + int ret = aclrtMemcpy( + state->source.ptr, + state->bytes, + source.data(), + source.size(), + ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + return ret; + } + return aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + destination.size(), + ACL_MEMCPY_HOST_TO_DEVICE); +} + +int CheckAllToAllState(AllToAllState* state) +{ + if (state == nullptr || state->destination.ptr == nullptr || state->bytes == 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->readRet = aclrtMemcpy( + state->observed.data(), + state->observed.size(), + state->destination.ptr, + state->bytes, + ACL_MEMCPY_DEVICE_TO_HOST); + if (state->readRet != ACL_SUCCESS) { + return state->readRet; + } + state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + const size_t blockBytes = 8U * 4096U; + uint32_t currentBlock = UINT32_MAX; + for (size_t i = 0; i < state->bytes; ++i) { + if (state->observed[i] != state->expected[i]) { + if (state->mismatchCount == 0) { + state->firstMismatchOffset = i; + state->firstMismatchObserved = state->observed[i]; + state->firstMismatchExpected = state->expected[i]; + } + state->lastMismatchOffset = i; + const uint32_t block = static_cast(i / blockBytes); + if (block != currentBlock) { + if (state->mismatchedBlockCount == 0) { + state->firstMismatchedBlock = block; + } + state->lastMismatchedBlock = block; + currentBlock = block; + ++state->mismatchedBlockCount; + } + ++state->mismatchCount; + } + } + state->passed = state->mismatchCount == 0; + return state->passed ? ACL_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + void PrintP2pCcuCopy( int rank, int peer, @@ -740,6 +963,8 @@ TileXR::TileXRCcuSignalWaitRequest MakeSignalWaitRequest( TileXR::TileXRCcuSignalWaitRequest request {}; request.peerRank = rankSize == 2 ? 1 - rank : (rank + 1) % rankSize; request.role = SignalWaitRoleForRank(rank); + request.overrideBarrierMode = BarrierSmokeEnabled(); + request.barrierMode = options.barrierMode; request.syncInstructionCount = std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr ? 0U : options.syncInstructionCount; @@ -1021,6 +1246,54 @@ uint16_t LoadLe16(const uint8_t* data, size_t index) static_cast(static_cast(data[index * 2U + 1U]) << 8U); } +bool ReadMissionContextAtEnd( + DirectCcuSmokeContext* context, + const TileXRDirectCcuTaskInfo& task, + const char* label) +{ + if (context == nullptr || label == nullptr) { + return false; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport adapterReport; + int ret = context->session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return false; + } + + uint8_t raw[TileXR::TILEXR_CCU_DATA_ARRAY_SLOT_BYTES] = {}; + ret = adapter.ReadMissionContext( + task.dieId, + task.missionId, + raw, + sizeof(raw), + &adapterReport); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " missionCtxRead ret=" << ret + << " opcode=" << adapterReport.opcode + << " driverRet=" << adapterReport.driverRet + << " opRet=" << adapterReport.opRet + << " message=\"" << adapterReport.message << "\"" + << std::endl; + return false; + } + + const uint16_t part4 = LoadLe16(raw, 4); + const uint16_t part5 = LoadLe16(raw, 5); + const uint16_t part6 = LoadLe16(raw, 6); + const uint16_t currentIns = static_cast(((part5 & 0x1fU) << 11U) | ((part4 >> 5U) & 0x7ffU)); + const uint16_t endIns = static_cast(((part6 & 0x1fU) << 11U) | ((part5 >> 5U) & 0x7ffU)); + const bool atEnd = currentIns == endIns; + std::cerr << label << " missionCtxAtEnd=" << (atEnd ? 1 : 0) + << " currentIns=" << currentIns + << " endIns=" << endIns + << std::endl; + return atEnd; +} + void PrintMissionContext( DirectCcuSmokeContext* context, const TileXRDirectCcuTaskInfo& task, @@ -1088,6 +1361,92 @@ void PrintMissionContext( std::cerr << std::dec << std::endl; } +void PrintCcuResourceState( + DirectCcuSmokeContext* context, + uint8_t dieId, + const TileXRDirectCcuPrepareOptions& options, + const char* label) +{ + if (context == nullptr || label == nullptr) { + return; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport report; + int ret = context->session.CreateDriverAdapter(&adapter, &report); + if (ret != TileXR::TILEXR_SUCCESS) { + std::cerr << label << " resourceState adapterRet=" << ret + << " message=\"" << report.message << "\"" << std::endl; + return; + } + + uint64_t localXn[3] {}; + uint64_t remoteXn[3] {}; + uint64_t localWaitCke[3] {}; + uint64_t remoteNotifyCke[3] {}; + const uint32_t localXnStartId = options.xnStartId; + const uint32_t remoteXnStartId = options.remoteXnStartId; + const uint32_t localWaitCkeStartId = options.localWaitCkeStartId; + const uint32_t remoteNotifyCkeStartId = options.remoteNotifyCkeStartId; + const int localXnRet = adapter.ReadXnRange(dieId, localXnStartId, localXn, 3, &report); + const int remoteXnRet = adapter.ReadXnRange(dieId, remoteXnStartId, remoteXn, 3, &report); + const int localCkeRet = adapter.ReadCkeRange( + dieId, localWaitCkeStartId, localWaitCke, 3, &report); + const int remoteCkeRet = adapter.ReadCkeRange( + dieId, remoteNotifyCkeStartId, remoteNotifyCke, 3, &report); + + std::cerr << label << " resourceState" + << " localXnStartId=" << localXnStartId + << " localXnRet=" << localXnRet + << " localXn=0x" << std::hex << localXn[0] << ",0x" << localXn[1] << ",0x" << localXn[2] + << std::dec + << " remoteXnStartId=" << remoteXnStartId + << " remoteXnRet=" << remoteXnRet + << " remoteXn=0x" << std::hex << remoteXn[0] << ",0x" << remoteXn[1] << ",0x" << remoteXn[2] + << std::dec + << " localWaitCkeStartId=" << localWaitCkeStartId + << " localCkeRet=" << localCkeRet + << " localCke=0x" << std::hex << localWaitCke[0] << ",0x" << localWaitCke[1] << ",0x" << localWaitCke[2] + << std::dec + << " remoteNotifyCkeStartId=" << remoteNotifyCkeStartId + << " remoteCkeRet=" << remoteCkeRet + << " remoteCke=0x" << std::hex << remoteNotifyCke[0] << ",0x" << remoteNotifyCke[1] << ",0x" << remoteNotifyCke[2] + << std::dec << std::endl; +} + +int ReadAndValidatePeerLoopMarker( + DirectCcuSmokeContext* context, + uint8_t dieId, + uint32_t markerXnId, + int rank, + int loopIndex, + uint64_t expectedPeerLoopMarker) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXR::TileXRCcuDriverAdapter adapter; + TileXR::TileXRCcuDriverAdapterReport report; + int readRet = context->session.CreateDriverAdapter(&adapter, &report); + uint64_t peerLoopMarker = 0; + if (readRet == TileXR::TILEXR_SUCCESS) { + readRet = adapter.ReadXnRange(dieId, markerXnId, &peerLoopMarker, 1, &report); + } + const bool matched = readRet == TileXR::TILEXR_SUCCESS && + peerLoopMarker == expectedPeerLoopMarker; + std::cout << "tilexr_ccu_alltoall peerLoopMarker" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " xnId=" << markerXnId + << " readRet=" << readRet + << " observed=0x" << std::hex << peerLoopMarker + << " expected=0x" << expectedPeerLoopMarker + << std::dec + << " matched=" << (matched ? 1 : 0) + << " message=\"" << report.message << "\"" + << std::endl; + return matched ? TileXR::TILEXR_SUCCESS : TileXR::TILEXR_ERROR_INTERNAL; +} + void PrintInstructionReadback(DirectCcuSmokeContext* context, TileXRDirectCcuPreparedTasksPtr prepared, uint32_t taskCount) { if (!EnvFlag(kReadbackInstructionsEnv)) { @@ -1156,18 +1515,27 @@ bool CollectiveSubmitReadyGateConfigured() return readyDir != nullptr && readyDir[0] != '\0'; } -std::string SubmitReadinessPath(int rank) +std::string RankPhaseFileStem(int rank, int phase) +{ + std::string stem = "/rank" + std::to_string(rank); + if (phase >= 0) { + stem += ".phase" + std::to_string(phase); + } + return stem; +} + +std::string SubmitReadinessPath(int rank, int phase = -1) { const char* readyDir = std::getenv(kReadyDirEnv); if (readyDir == nullptr || readyDir[0] == '\0') { return {}; } - return std::string(readyDir) + "/rank" + std::to_string(rank) + ".ready"; + return std::string(readyDir) + RankPhaseFileStem(rank, phase) + ".ready"; } -bool WriteSubmitReadiness(int rank, bool ready) +bool WriteSubmitReadiness(int rank, bool ready, int phase = -1) { - const std::string path = SubmitReadinessPath(rank); + const std::string path = SubmitReadinessPath(rank, phase); if (path.empty()) { return true; } @@ -1179,12 +1547,12 @@ bool WriteSubmitReadiness(int rank, bool ready) return static_cast(out); } -bool ReadSubmitReadiness(int rank, bool* ready) +bool ReadSubmitReadiness(int rank, bool* ready, int phase = -1) { if (ready == nullptr) { return false; } - const std::string path = SubmitReadinessPath(rank); + const std::string path = SubmitReadinessPath(rank, phase); if (path.empty()) { return false; } @@ -1201,12 +1569,12 @@ bool ReadSubmitReadiness(int rank, bool* ready) return true; } -bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) +bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady, int phase = -1) { if (!CollectiveSubmitReadyGateConfigured()) { return localReady; } - const bool wrote = WriteSubmitReadiness(rank, localReady); + const bool wrote = WriteSubmitReadiness(rank, localReady, phase); const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); const auto start = std::chrono::steady_clock::now(); for (;;) { @@ -1214,7 +1582,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) bool allReady = wrote && localReady; for (int peer = 0; peer < rankSize; ++peer) { bool peerReady = false; - if (!ReadSubmitReadiness(peer, &peerReady)) { + if (!ReadSubmitReadiness(peer, &peerReady, phase)) { allSeen = false; allReady = false; break; @@ -1224,6 +1592,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) if (allSeen) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" << " rank=" << rank + << " phase=" << phase << " localReady=" << (localReady ? 1 : 0) << " allRanksReady=" << (allReady ? 1 : 0) << std::endl; @@ -1234,6 +1603,7 @@ bool WaitForCollectiveSubmitReadiness(int rank, int rankSize, bool localReady) if (elapsedMs >= timeoutMs) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitReady" << " rank=" << rank + << " phase=" << phase << " localReady=" << (localReady ? 1 : 0) << " allRanksReady=0" << " timeoutMs=" << timeoutMs @@ -1250,18 +1620,18 @@ bool CollectiveSubmitDoneGateConfigured() return doneDir != nullptr && doneDir[0] != '\0'; } -std::string SubmitDonePath(int rank) +std::string SubmitDonePath(int rank, int phase = -1) { const char* doneDir = std::getenv(kDoneDirEnv); if (doneDir == nullptr || doneDir[0] == '\0') { return {}; } - return std::string(doneDir) + "/rank" + std::to_string(rank) + ".done"; + return std::string(doneDir) + RankPhaseFileStem(rank, phase) + ".done"; } -bool WriteSubmitDone(int rank, int result) +bool WriteSubmitDone(int rank, int result, int phase = -1) { - const std::string path = SubmitDonePath(rank); + const std::string path = SubmitDonePath(rank, phase); if (path.empty()) { return true; } @@ -1273,12 +1643,12 @@ bool WriteSubmitDone(int rank, int result) return static_cast(out); } -bool ReadSubmitDone(int rank, int* result) +bool ReadSubmitDone(int rank, int* result, int phase = -1) { if (result == nullptr) { return false; } - const std::string path = SubmitDonePath(rank); + const std::string path = SubmitDonePath(rank, phase); if (path.empty()) { return false; } @@ -1290,12 +1660,12 @@ bool ReadSubmitDone(int rank, int* result) return static_cast(in); } -bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) +bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult, int phase = -1) { if (!CollectiveSubmitDoneGateConfigured()) { return true; } - const bool wrote = WriteSubmitDone(rank, localResult); + const bool wrote = WriteSubmitDone(rank, localResult, phase); const int timeoutMs = EnvInt(kReadyTimeoutMsEnv, 5000); const auto start = std::chrono::steady_clock::now(); for (;;) { @@ -1303,7 +1673,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) bool allSucceeded = wrote && localResult == 0; for (int peer = 0; peer < rankSize; ++peer) { int peerResult = 0; - if (!ReadSubmitDone(peer, &peerResult)) { + if (!ReadSubmitDone(peer, &peerResult, phase)) { allSeen = false; allSucceeded = false; break; @@ -1313,6 +1683,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) if (allSeen) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" << " rank=" << rank + << " phase=" << phase << " localResult=" << localResult << " allRanksDone=1" << " allRanksSucceeded=" << (allSucceeded ? 1 : 0) @@ -1324,6 +1695,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult) if (elapsedMs >= timeoutMs) { std::cout << "tilexr_ccu_direct_smoke collectiveSubmitDone" << " rank=" << rank + << " phase=" << phase << " localResult=" << localResult << " allRanksDone=0" << " timeoutMs=" << timeoutMs @@ -1356,6 +1728,578 @@ int RunInactiveP2pCcuCopyRank(int rank, int peer, int rankSize, P2pCcuCopyState* return checkRet == ACL_SUCCESS ? 0 : 14; } +int RunAllToAllCopyPhase( + DirectCcuSmokeContext* context, + int rank, + int rankSize, + int device, + int phase, + AllToAllState* alltoall) +{ + if (context == nullptr || alltoall == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 7; + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + const int peer = 1 - rank; + const bool singleRouteBidirectional = AllToAllSingleRouteBidirectionalEnabled(); + const bool active = singleRouteBidirectional || rank == phase; + const bool submitRequested = EnvFlag(kSubmitEnv); + + std::cout << "tilexr_ccu_alltoall phase" + << " rank=" << rank + << " phase=" << phase + << " direction=" << (singleRouteBidirectional ? "LocalToRemote" : "RemoteToLocal") + << " singleRouteBidirectional=" << (singleRouteBidirectional ? 1 : 0) + << std::endl; + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall->initRet != ACL_SUCCESS ? + alltoall->initRet : + context->planner.PrepareDirectCcuMemoryCopyInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall->source.ptr), + reinterpret_cast(alltoall->destination.ptr), + alltoall->bytes, + static_cast(peer), + singleRouteBidirectional ? + TileXR::TileXRCcuMemoryCopyDirection::LocalToRemote : + TileXR::TileXRCcuMemoryCopyDirection::RemoteToLocal, + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + if (!active) { + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested) { + const bool phaseReady = WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady, phase); + finalRet = phaseReady ? 0 : 13; + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase) && finalRet == 0) { + finalRet = 13; + } + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; + } + + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady, + phase) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + finalRet = 6; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + bool skipStreamDestroy = false; + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before alltoall aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after alltoall aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " phase=" << phase + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + bool missionAtEnd = false; + if (!attempt.submitTasks.empty()) { + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + missionAtEnd = ReadMissionContextAtEnd( + context, + attempt.submitTasks.front(), + "tilexr_ccu_alltoall"); + } + PrintCcuResourceState( + context, + attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall"); + if (missionAtEnd) { + std::cout << "tilexr_ccu_alltoall streamTimeoutAtMissionEnd=1" + << " rank=" << rank + << " reason=\"continuing to device buffer validation\"" + << std::endl; + skipStreamDestroy = true; + } else { + finalRet = 8; + } + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase) && finalRet == 0) { + finalRet = 13; + } + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" + << " rank=" << rank + << " reason=\"stream timeout after mission reached end\"" + << std::endl; + } else { + aclrtDestroyStream(stream); + } + } + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + +void PrintAllToAllResult(int rank, int loopIndex, int finalRet, const AllToAllState& alltoall) +{ + if (finalRet == 0) { + std::cout << "tilexr_ccu_alltoall result passed=1" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << std::endl; + } else { + std::cout << "tilexr_ccu_alltoall result passed=0" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet + << " readRet=" << alltoall.readRet + << " mismatches=" << alltoall.mismatchCount + << " firstMismatchOffset=" << alltoall.firstMismatchOffset + << " lastMismatchOffset=" << alltoall.lastMismatchOffset + << " firstMismatchObserved=0x" << std::hex << alltoall.firstMismatchObserved + << " firstMismatchExpected=0x" << alltoall.firstMismatchExpected + << std::dec + << " mismatchedBlocks=" << alltoall.mismatchedBlockCount + << " firstMismatchedBlock=" << alltoall.firstMismatchedBlock + << " lastMismatchedBlock=" << alltoall.lastMismatchedBlock + << std::endl; + } +} + +void MaybeFastExitAfterAllToAllRun(int finalRet) +{ + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_alltoall fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } +} + +int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + const int loopCount = AllToAllLoopCountFromEnv(); + AllToAllState alltoall; + alltoall.initRet = InitAllToAllState(rank, peer, &alltoall); + if (loopCount == 0 && alltoall.initRet == ACL_SUCCESS) { + alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 3; + options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; + if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { + options.syncInstructionCount = 5 + 64 * 7; + } + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << alltoall.bytes + << " loopCount=" << loopCount + << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() + << " blockCount=64" + << " longMission=1" + << " preSync=1" + << " postSync=0" + << " finish=0" + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall.initRet != ACL_SUCCESS ? + alltoall.initRet : + context->planner.PrepareDirectCcuAllToAll2RankInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall.source.ptr), + reinterpret_cast(alltoall.destination.ptr), + alltoall.bytes, + static_cast(peer), + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (attempt.submitTasks.empty() || + attempt.submitTasks.front().argSize != TILEXR_DIRECT_CCU_SQE_ARGS_LEN) { + std::cerr << "tilexr_ccu_alltoall invalidPreparedTask" + << " rank=" << rank + << " taskCount=" << attempt.submitTasks.size() + << " argSize=" << (attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().argSize) + << std::endl; + finalRet = 6; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + bool skipStreamDestroy = false; + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { + const int resetRet = ResetAllToAllStateForLoop(rank, peer, loopIndex, &alltoall); + const uint64_t localLoopMarker = BuildAllToAllLoopMarker(rank, loopIndex); + attempt.submitTasks.front().args[0] = localLoopMarker; + const bool collectiveSubmitReady = WaitForCollectiveSubmitReadiness( + rank, + rankSize, + resetRet == ACL_SUCCESS && installReport.submitReady, + loopIndex); + if (resetRet != ACL_SUCCESS) { + finalRet = 14; + } else if (!collectiveSubmitReady) { + finalRet = 13; + } + + if (finalRet == 0) { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + TraceLifecycle("before alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after alltoall long mission aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " longMission=1" + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } else if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtSynchronizeStreamWithTimeout" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + finalRet = 8; + skipStreamDestroy = true; + } + } + + if (finalRet == 0) { + const int markerRet = ReadAndValidatePeerLoopMarker( + context, + attempt.submitTasks.front().dieId, + options.remoteXnStartId, + rank, + loopIndex, + BuildAllToAllLoopMarker(peer, loopIndex)); + if (markerRet != TileXR::TILEXR_SUCCESS) { + finalRet = 15; + } + } + if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { + finalRet = 14; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex) && finalRet == 0) { + finalRet = 13; + } + PrintAllToAllResult(rank, loopIndex, finalRet, alltoall); + if (finalRet != 0) { + std::cerr << "tilexr_ccu_alltoall loopFailure" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet << std::endl; + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + PrintCcuResourceState( + context, + attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall"); + break; + } + } + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1" + << " rank=" << rank + << " reason=\"stream timeout after mission reached end\"" + << std::endl; + } else { + aclrtDestroyStream(stream); + } + } + } + + if (!submitRequested && finalRet == 0) { + const int checkRet = CheckAllToAllState(&alltoall); + if (checkRet != ACL_SUCCESS) { + finalRet = 14; + } + PrintAllToAllResult(rank, -1, finalRet, alltoall); + } + MaybeFastExitAfterAllToAllRun(finalRet); + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + +int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AllToAllLongMissionEnabled()) { + return RunAllToAllLongMissionSmokeForRank(context, rank, rankSize, device); + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + AllToAllState alltoall; + alltoall.initRet = InitAllToAllState(rank, peer, &alltoall); + + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 7; + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " peer=" << peer + << " bytes=" << alltoall.bytes + << " memSlicePerLoop=" << AllToAllMemSlicePerLoopFromEnv() + << " blockCount=64" + << " hostPhases=2" + << std::endl; + PrintConfig(options, rankSize); + + int finalRet = alltoall.initRet == ACL_SUCCESS ? 0 : alltoall.initRet; + const int phaseCount = AllToAllSingleRouteBidirectionalEnabled() ? 1 : 2; + for (int phase = 0; phase < phaseCount && finalRet == 0; ++phase) { + finalRet = RunAllToAllCopyPhase(context, rank, rankSize, device, phase, &alltoall); + } + if (finalRet == 0) { + const int checkRet = CheckAllToAllState(&alltoall); + if (checkRet != ACL_SUCCESS) { + finalRet = 14; + } + } + + PrintAllToAllResult(rank, -1, finalRet, alltoall); + MaybeFastExitAfterAllToAllRun(finalRet); + return finalRet; +} + +int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 2) { + std::cout << "tilexr_ccu_sync_xn_ping skipped rankSize=" << rankSize + << " reason=\"direct CCU SyncXn ping requires two ranks\"" << std::endl; + return 0; + } + + const int peer = 1 - rank; + AllToAllState routeState; + routeState.initRet = InitAllToAllState(rank, peer, &routeState); + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 1; + options.sqeArgCount = 0; + options.syncInstructionCount = 3; + + std::cout << "tilexr_ccu_sync_xn_ping config" + << " rank=" << rank + << " peer=" << peer + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = routeState.initRet != ACL_SUCCESS ? + routeState.initRet : + context->planner.PrepareDirectCcuSyncXnPingInstallAttempt( + context->session, + options, + reinterpret_cast(routeState.source.ptr), + reinterpret_cast(routeState.destination.ptr), + routeState.bytes, + static_cast(peer), + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_sync_xn_ping prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + const bool collectiveSubmitReady = submitRequested ? + WaitForCollectiveSubmitReadiness( + rank, + rankSize, + prepareRet == TileXR::TILEXR_SUCCESS && installReport.submitReady) : + false; + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (submitRequested && !collectiveSubmitReady && CollectiveSubmitReadyGateConfigured()) { + std::cout << "tilexr_ccu_sync_xn_ping submit skipped reason=\"collective submitReady gate did not pass\"" + << " localSubmitReady=" << (installReport.submitReady ? 1 : 0) + << std::endl; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_sync_xn_ping submit skipped reason=\"prepare did not reach submitReady\"" << std::endl; + } else if (submitRequested) { + aclrtStream stream = nullptr; + int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_sync_xn_ping aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + TileXRDirectCcuSubmitReport submitReport; + const auto submitBegin = std::chrono::steady_clock::now(); + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + const auto submitEnd = std::chrono::steady_clock::now(); + PrintSubmitReport("tilexr_ccu_sync_xn_ping submit", submitRet, submitReport); + const auto syncBegin = std::chrono::steady_clock::now(); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before SyncXn ping aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after SyncXn ping aclrtSynchronizeStreamWithTimeout"); + const auto syncEnd = std::chrono::steady_clock::now(); + std::cout << "tilexr_ccu_sync_xn_ping timing" + << " rank=" << rank + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << " submitMs=" + << std::chrono::duration_cast(submitEnd - submitBegin).count() + << " syncMs=" + << std::chrono::duration_cast(syncEnd - syncBegin).count() + << std::endl; + if (syncRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_sync_xn_ping aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; + if (!attempt.submitTasks.empty()) { + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_sync_xn_ping"); + } + finalRet = 8; + } else if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet) && finalRet == 0) { + finalRet = 13; + } + aclrtDestroyStream(stream); + } + } + + if (finalRet == 0) { + std::cout << "tilexr_ccu_sync_xn_ping result passed=1" + << " rank=" << rank + << " ret=" << finalRet + << std::endl; + } else { + std::cout << "tilexr_ccu_sync_xn_ping result passed=0" + << " rank=" << rank + << " ret=" << finalRet + << std::endl; + } + if (ShouldFastExitAfterRun()) { + std::cout << "tilexr_ccu_sync_xn_ping fastExitAfterRun=1" + << " ret=" << finalRet + << " reason=\"skipping prepared-task cleanup to isolate cleanup hangs\"" + << std::endl; + std::fflush(stdout); + std::fflush(stderr); + std::_Exit(finalRet); + } + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { if (context == nullptr) { @@ -1425,9 +2369,10 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank const auto submitEnd = std::chrono::steady_clock::now(); PrintSubmitReport("tilexr_ccu_signal_wait submit", submitRet, submitReport); const auto syncBegin = std::chrono::steady_clock::now(); - TraceLifecycle("before signal/wait aclrtSynchronizeStream"); - const int syncRet = aclrtSynchronizeStream(stream); - TraceLifecycle("after signal/wait aclrtSynchronizeStream"); + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + TraceLifecycle("before signal/wait aclrtSynchronizeStreamWithTimeout"); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + TraceLifecycle("after signal/wait aclrtSynchronizeStreamWithTimeout"); const auto syncEnd = std::chrono::steady_clock::now(); std::cout << "tilexr_ccu_signal_wait timing" << " rank=" << rank @@ -1435,13 +2380,15 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank << " preSubmitDelayMs=" << effectiveDelayMs << " submitRet=" << submitRet << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs << " submitMs=" << std::chrono::duration_cast(submitEnd - submitBegin).count() << " syncMs=" << std::chrono::duration_cast(syncEnd - syncBegin).count() << std::endl; if (syncRet != ACL_SUCCESS) { - std::cerr << "tilexr_ccu_signal_wait aclrtSynchronizeStream ret=" << syncRet << std::endl; + std::cerr << "tilexr_ccu_signal_wait aclrtSynchronizeStreamWithTimeout ret=" << syncRet + << " timeoutMs=" << syncTimeoutMs << std::endl; if (!plan.submitTasks.empty()) { PrintMissionContext(context, plan.submitTasks.front(), "tilexr_ccu_signal_wait"); } @@ -1489,6 +2436,12 @@ int RunSignalWaitSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank int RunPreparedSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { + if (SyncXnPingSmokeEnabled()) { + return RunSyncXnPingSmokeForRank(context, rank, rankSize, device); + } + if (AllToAllSmokeEnabled()) { + return RunAllToAllSmokeForRank(context, rank, rankSize, device); + } if (SignalWaitSmokeEnabled() || BarrierSmokeEnabled()) { return RunSignalWaitSmokeForRank(context, rank, rankSize, device); } diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index e8fac065..ef704493 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -87,6 +87,21 @@ signal_wait_mode_enabled() [ "${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0}" = "1" ] } +alltoall_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0}" = "1" ] +} + +alltoall_long_mission_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ] +} + +sync_xn_ping_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0}" = "1" ] +} + apply_p2p_ccu_copy_defaults() { if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0}" != "1" ]; then @@ -154,8 +169,89 @@ apply_signal_wait_defaults() export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" } +apply_sync_xn_ping_defaults() +{ + if ! sync_xn_ping_mode_enabled; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}" + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" + export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}" +} + +apply_alltoall_defaults() +{ + if ! alltoall_mode_enabled; then + return + fi + + export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" + export TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" + export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" + export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" + export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" + if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453}" + else + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7}" + fi + export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" + export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" + export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" + export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" + export TILEXR_CCU_PROBE_GSA_START="${TILEXR_CCU_PROBE_GSA_START:-510}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-8}" + export TILEXR_CCU_PROBE_CHANNEL_START="${TILEXR_CCU_PROBE_CHANNEL_START:-2}" + export TILEXR_CCU_DIRECT_BARRIER_MODE="${TILEXR_CCU_DIRECT_BARRIER_MODE:-sync_cke}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE="${TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE:-hcomm_cap}" + export TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}" +} + apply_p2p_ccu_copy_defaults apply_signal_wait_defaults +apply_sync_xn_ping_defaults +apply_alltoall_defaults if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" @@ -170,6 +266,12 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT \ TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK \ TILEXR_CCU_DIRECT_SMOKE_BARRIER \ + TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION \ + TILEXR_CCU_ALLTOALL_BYTES \ + TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ + TILEXR_CCU_ALLTOALL_LOOP_COUNT \ TILEXR_CCU_PROBE_SQE_ARG_COUNT \ TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START; do diagnostic_value="${!diagnostic_var:-}" @@ -428,6 +530,42 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION}") +fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC=${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT=${TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT}") +fi +if [ "${TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE=${TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE}") +fi +if [ "${TILEXR_CCU_ALLTOALL_BYTES:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_BYTES=${TILEXR_CCU_ALLTOALL_BYTES}") +fi +if [ "${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP=${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP}") +fi +if [ "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-}" != "" ]; then + common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN=${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN}") fi @@ -559,7 +697,7 @@ for token_field in "${resource_window_token_fields[@]}"; do fi done -echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" +echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} syncXnPing=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} alltoallLongMission=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then thread_log="${work_dir}/ccu_thread.log" @@ -574,7 +712,34 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then echo "thread log: ${thread_log}" >&2 exit 4 fi - if signal_wait_mode_enabled; then + if alltoall_mode_enabled; then + if [ "$(grep -c "tilexr_ccu_alltoall prepare ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode prepare did not return success for both ranks" >&2 + exit 5 + fi + if [ "$(grep -c "installSucceeded=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode prepare did not complete install attempt for both ranks" >&2 + exit 6 + fi + if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then + if [ "$(grep -c "submitReady=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode submit requested but prepare did not reach submitReady=1" >&2 + exit 6 + fi + if [ "$(grep -c "tilexr_ccu_alltoall submit ret=0" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode submit did not return success for both ranks" >&2 + exit 7 + fi + if [ "$(grep -c "tilexr_ccu_alltoall timing" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode timing was not reported for both ranks" >&2 + exit 8 + fi + fi + if [ "$(grep -c "tilexr_ccu_alltoall result passed=1" "${thread_log}")" -lt 2 ]; then + echo "ERROR: direct CCU alltoall thread-mode result did not pass for both ranks" >&2 + exit 8 + fi + elif signal_wait_mode_enabled; then if [ "$(grep -c "tilexr_ccu_signal_wait prepare ret=0" "${thread_log}")" -lt 2 ]; then echo "ERROR: direct CCU signal/wait thread-mode prepare did not return success for both ranks" >&2 exit 5 @@ -666,7 +831,18 @@ if [ "${rank0_status}" -ne 0 ] || [ "${rank1_status}" -ne 0 ]; then exit 4 fi -if signal_wait_mode_enabled; then +if alltoall_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_alltoall prepare ret=0" "${log}"; then + echo "ERROR: direct CCU alltoall prepare did not return success in ${log}" >&2 + exit 5 + fi + if ! grep -q "installSucceeded=1" "${log}"; then + echo "ERROR: direct CCU alltoall prepare did not complete install attempt in ${log}" >&2 + exit 6 + fi + done +elif signal_wait_mode_enabled; then for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_signal_wait prepare ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait prepare did not return success in ${log}" >&2 @@ -705,6 +881,17 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then fi done for log in "${rank0_log}" "${rank1_log}"; do + if alltoall_mode_enabled; then + if ! grep -q "tilexr_ccu_alltoall submit ret=0" "${log}"; then + echo "ERROR: direct CCU alltoall submit did not return success in ${log}" >&2 + exit 7 + fi + if ! grep -q "tilexr_ccu_alltoall timing" "${log}"; then + echo "ERROR: direct CCU alltoall timing was not reported in ${log}" >&2 + exit 8 + fi + continue + fi if signal_wait_mode_enabled; then if ! grep -q "tilexr_ccu_signal_wait submit ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait submit did not return success in ${log}" >&2 @@ -733,7 +920,14 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then done fi -if signal_wait_mode_enabled; then +if alltoall_mode_enabled; then + for log in "${rank0_log}" "${rank1_log}"; do + if ! grep -q "tilexr_ccu_alltoall result passed=1" "${log}"; then + echo "ERROR: direct CCU alltoall result did not pass in ${log}" >&2 + exit 9 + fi + done +elif signal_wait_mode_enabled; then for log in "${rank0_log}" "${rank1_log}"; do if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then echo "ERROR: direct CCU signal/wait result did not pass in ${log}" >&2 diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py new file mode 100644 index 00000000..45a77265 --- /dev/null +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2026 TileXR Project +# + +import shutil +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[2] +INCLUDE_DIR = REPO_ROOT / "src" / "include" +COMM_DIR = REPO_ROOT / "src" / "comm" +ALLTOALL_HEADER = COMM_DIR / "ccu" / "tilexr_ccu_alltoall_program.h" +ALLTOALL_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_alltoall_program.cpp" +MICROCODE_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_microcode.cpp" +MEMORY_SOURCE = COMM_DIR / "ccu" / "tilexr_ccu_memory_program.cpp" +COMM_CMAKE = COMM_DIR / "CMakeLists.txt" + + +PRIVATE_CCU_PRODUCER_NEEDLES = [ + "#include + #include + + using namespace TileXR; + + constexpr uint16_t kSetCkeHeader = 0x0802U; + constexpr uint16_t kClearCkeHeader = 0x0804U; + constexpr uint16_t kLoadSqeArgsToXnHeader = 0x0001U; + constexpr uint16_t kLoadImdToXnHeader = 0x0003U; + constexpr uint16_t kSyncXnHeader = 0x100dU; + + uint16_t Slot(const TileXRCcuInstr& instr, uint32_t slot) + { + const uint32_t word = slot / 4U; + const uint32_t shift = (slot % 4U) * 16U; + return static_cast((instr.words[word] >> shift) & 0xffffU); + } + + int main() + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12346, 0x22223333U, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(0x23457, 0x44445555U, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 0x101; + spec.remoteGsa = 0x102; + spec.localXn = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x203; + spec.preSyncLocalAddrXn = 0x209; + spec.preSyncLocalTokenXn = 0x20a; + spec.preSyncLocalMarkerXn = 0x208; + spec.preSyncRemoteMarkerXn = 0x210; + spec.preSyncMarkerArgIndex = 0; + spec.preSyncMarkerEnabled = true; + spec.channelId = 0x12; + spec.preSyncChannelId = 0x13; + spec.preSyncTokenChannelId = 0x13; + spec.copyCompletionCke = 0x301; + spec.preSyncRemoteAddrXn = 0x211; + spec.preSyncRemoteTokenXn = 0x212; + spec.preSyncLocalWaitCke = 0x302; + spec.preSyncRemoteNotifyCke = 0x303; + spec.preSyncTokenLocalWaitCke = 0x302; + spec.preSyncRemoteTokenNotifyCke = 0x303; + spec.postSyncLocalWaitCke = 0x306; + spec.postSyncRemoteNotifyCke = 0x307; + spec.sourceCke = 0x308; + spec.ckeMask = 1; + + std::vector program; + TileXRCcuAllToAllProgramReport report; + int ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS) { + std::cerr << "builder failed: " << report.message << "\n"; + return 1; + } + const uint32_t expectedBlocks = 64; + const uint32_t expectedInstructions = 7 + expectedBlocks * 7 + 3 + 1; + if (report.blockCount != expectedBlocks || + report.copyInstructionCount != expectedBlocks * 7 || + report.preSyncInstructionCount != 7 || + report.postSyncInstructionCount != 3 || + report.finishInstructionCount != 1 || + report.totalInstructionCount != expectedInstructions || + program.size() != expectedInstructions) { + std::cerr << "unexpected report counts" + << " blocks=" << report.blockCount + << " copyInst=" << report.copyInstructionCount + << " pre=" << report.preSyncInstructionCount + << " post=" << report.postSyncInstructionCount + << " finish=" << report.finishInstructionCount + << " total=" << report.totalInstructionCount + << " size=" << program.size() << "\n"; + return 2; + } + if (report.bytesPerBlock != 32768 || report.message != "ok") { + std::cerr << "unexpected block size or message\n"; + return 3; + } + const uint16_t markerMask = 1U; + const uint16_t outputMask = 1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID; + const uint16_t tokenMask = 1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID; + const uint16_t waitMask = markerMask | outputMask | tokenMask; + const uint16_t syncMask = spec.ckeMask; + const uint32_t postSyncSetIndex = expectedInstructions - 4; + const uint32_t postSyncWaitIndex = expectedInstructions - 2; + if (program.size() < 7 || + Slot(program[0], 0) != kLoadSqeArgsToXnHeader || + Slot(program[0], 1) != spec.preSyncLocalMarkerXn || + Slot(program[0], 2) != spec.preSyncMarkerArgIndex || + Slot(program[1], 0) != kSyncXnHeader || + Slot(program[1], 1) != spec.preSyncRemoteMarkerXn || + Slot(program[1], 2) != spec.preSyncLocalMarkerXn || + Slot(program[1], 4) != spec.preSyncChannelId || + Slot(program[1], 5) != spec.preSyncRemoteNotifyCke || + Slot(program[1], 6) != markerMask || + Slot(program[2], 0) != kLoadImdToXnHeader || + Slot(program[2], 1) != spec.preSyncLocalAddrXn || + Slot(program[3], 0) != kSyncXnHeader || + Slot(program[3], 1) != spec.preSyncRemoteAddrXn || + Slot(program[3], 2) != spec.preSyncLocalAddrXn || + Slot(program[3], 6) != outputMask || + Slot(program[4], 0) != kLoadImdToXnHeader || + Slot(program[4], 1) != spec.preSyncLocalTokenXn || + Slot(program[5], 0) != kSyncXnHeader || + Slot(program[5], 1) != spec.preSyncRemoteTokenXn || + Slot(program[5], 2) != spec.preSyncLocalTokenXn || + Slot(program[5], 4) != spec.preSyncTokenChannelId || + Slot(program[5], 5) != spec.preSyncRemoteTokenNotifyCke || + Slot(program[5], 6) != tokenMask || + Slot(program[6], 0) != kSetCkeHeader || + Slot(program[6], 4) != spec.preSyncLocalWaitCke || + Slot(program[6], 5) != waitMask) { + std::cerr << "PreSync should publish marker, output, and token then wait for all" + << " loadHeader=0x" << std::hex << Slot(program[0], 0) + << " loadXn=0x" << Slot(program[0], 1) + << " notifyHeader=0x" << Slot(program[1], 0) + << " remoteXn=0x" << Slot(program[1], 1) + << " localXn=0x" << Slot(program[1], 2) + << " channel=0x" << Slot(program[1], 4) + << " remoteCke=0x" << Slot(program[1], 5) + << " notifyMask=0x" << Slot(program[1], 6) + << " tokenNotifyHeader=0x" << Slot(program[5], 0) + << " tokenRemoteXn=0x" << Slot(program[5], 1) + << " tokenLocalXn=0x" << Slot(program[5], 2) + << " tokenChannel=0x" << Slot(program[5], 4) + << " tokenRemoteCke=0x" << Slot(program[5], 5) + << " tokenNotifyMask=0x" << Slot(program[5], 6) + << " waitHeader=0x" << Slot(program[6], 0) + << " waitCke=0x" << Slot(program[6], 4) + << " waitMask=0x" << Slot(program[6], 5) + << std::dec << "\n"; + return 4; + } + if (Slot(program[postSyncSetIndex], 0) != kSetCkeHeader || + Slot(program[postSyncSetIndex], 3) != syncMask || + Slot(program[postSyncWaitIndex], 0) != kClearCkeHeader || + Slot(program[postSyncWaitIndex], 4) != spec.postSyncLocalWaitCke || + Slot(program[postSyncWaitIndex], 5) != syncMask) { + std::cerr << "PostSync should use the allocated post CKE resource with the resource mask" + << " postSetMask=0x" << std::hex << Slot(program[postSyncSetIndex], 3) + << " postWaitCke=0x" << Slot(program[postSyncWaitIndex], 4) + << " postWaitMask=0x" << Slot(program[postSyncWaitIndex], 5) + << " syncMask=0x" << spec.ckeMask + << std::dec << "\n"; + return 5; + } + + spec.preSyncNotify = false; + spec.postSyncNotify = false; + spec.emitFinish = false; + ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS || + report.preSyncInstructionCount != 0 || + report.postSyncInstructionCount != 0 || + report.finishInstructionCount != 0 || + report.totalInstructionCount != expectedBlocks * 7 || + program.size() != expectedBlocks * 7) { + std::cerr << "copy-only diagnostic program has unexpected counts\n"; + return 6; + } + + spec.preSyncNotify = true; + spec.preSyncWait = false; + spec.sourceCke = 0; + ret = TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS || + report.preSyncInstructionCount != 6 || + report.totalInstructionCount != expectedBlocks * 7 + 6 || + program.size() != expectedBlocks * 7 + 6) { + std::cerr << "notify-only PreSync program has unexpected counts\n"; + return 7; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + + using namespace TileXR; + + constexpr uint16_t kTransLocMemToRmtMemHeader = 0x1009U; + + uint16_t Header(const TileXRCcuInstr& instr) + { + return static_cast(instr.words[0] & 0xffffU); + } + + TileXRCcuAllToAll2RankProgramSpec ValidSpec(uint32_t localRank) + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localRank = localRank; + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(0x12345, 0x11112222U, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(0x12346, 0x22223333U, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(0x23456, 0x33334444U, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(0x23457, 0x44445555U, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 0x101; + spec.remoteGsa = 0x102; + spec.localXn = 0x201; + spec.remoteXn = 0x202; + spec.lengthXn = 0x203; + spec.preSyncLocalMarkerXn = 0x201; + spec.preSyncRemoteMarkerXn = 0x213; + spec.preSyncMarkerArgIndex = 0; + spec.preSyncMarkerEnabled = true; + spec.channelId = 0x12; + spec.copyCompletionCke = 0x301; + spec.preSyncRemoteAddrXn = 0x211; + spec.preSyncRemoteTokenXn = 0x212; + spec.preSyncLocalWaitCke = 0x302; + spec.preSyncRemoteNotifyCke = 0x303; + spec.preSyncTokenLocalWaitCke = 0x304; + spec.preSyncRemoteTokenNotifyCke = 0x305; + spec.postSyncLocalWaitCke = 0x306; + spec.postSyncRemoteNotifyCke = 0x307; + spec.sourceCke = 0x308; + spec.ckeMask = 1; + return spec; + } + + size_t FirstCopyIndex(const std::vector& program) + { + for (size_t i = 0; i < program.size(); ++i) { + if (Header(program[i]) == kTransLocMemToRmtMemHeader) { + return i; + } + } + return program.size(); + } + + uint32_t CopyInstructionCount(const std::vector& program) + { + uint32_t count = 0; + for (const auto& instr : program) { + if (Header(instr) == kTransLocMemToRmtMemHeader) { + ++count; + } + } + return count; + } + + int main() + { + std::vector rank0; + std::vector rank1; + TileXRCcuAllToAllProgramReport report0; + TileXRCcuAllToAllProgramReport report1; + int ret0 = TileXRCcuBuildAllToAll2RankProgram(ValidSpec(0), &rank0, &report0); + int ret1 = TileXRCcuBuildAllToAll2RankProgram(ValidSpec(1), &rank1, &report1); + if (ret0 != TILEXR_SUCCESS || ret1 != TILEXR_SUCCESS) { + std::cerr << "builder failed rank0=" << report0.message + << " rank1=" << report1.message << "\n"; + return 1; + } + const uint32_t expectedInstructions = 7 + 64 * 7 + 3 + 1; + if (report0.totalInstructionCount != expectedInstructions || + report1.totalInstructionCount != expectedInstructions || + rank0.size() != expectedInstructions || + rank1.size() != expectedInstructions) { + std::cerr << "unexpected instruction count" + << " rank0=" << rank0.size() + << " rank1=" << rank1.size() + << " report0=" << report0.totalInstructionCount + << " report1=" << report1.totalInstructionCount << "\n"; + return 2; + } + if (CopyInstructionCount(rank0) != 64 || CopyInstructionCount(rank1) != 64) { + std::cerr << "each rank should issue exactly 64 local-to-remote writes\n"; + return 3; + } + const size_t rank0FirstCopy = FirstCopyIndex(rank0); + const size_t rank1FirstCopy = FirstCopyIndex(rank1); + if (rank0FirstCopy != 12 || rank1FirstCopy != 12) { + std::cerr << "copy region should start at the same instruction after the single PreSync" + << " rank0FirstCopy=" << rank0FirstCopy + << " rank1FirstCopy=" << rank1FirstCopy << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builder_rejects_invalid_slice_configuration(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + + using namespace TileXR; + + TileXRCcuAllToAll2RankProgramSpec ValidSpec() + { + TileXRCcuAllToAll2RankProgramSpec spec; + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.localRecvAddr = 0x18000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + spec.remoteSendAddr = 0x20000000ULL; + spec.remoteSendToken = TileXRCcuPackMemoryToken(3, 4, true); + spec.remoteRecvAddr = 0x28000000ULL; + spec.remoteRecvToken = TileXRCcuPackMemoryToken(4, 5, true); + spec.bytes = 2ULL * 1024ULL * 1024ULL; + spec.memorySliceBytes = TILEXR_CCU_ALLTOALL_MEMORY_SLICE_BYTES; + spec.memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; + spec.localGsa = 1; + spec.remoteGsa = 2; + spec.localXn = 3; + spec.remoteXn = 4; + spec.lengthXn = 5; + spec.channelId = 6; + spec.copyCompletionCke = 7; + spec.preSyncRemoteAddrXn = 13; + spec.preSyncRemoteTokenXn = 14; + spec.preSyncLocalWaitCke = 8; + spec.preSyncRemoteNotifyCke = 9; + spec.preSyncTokenLocalWaitCke = 10; + spec.preSyncRemoteTokenNotifyCke = 11; + spec.postSyncLocalWaitCke = 12; + spec.postSyncRemoteNotifyCke = 15; + spec.sourceCke = 16; + spec.ckeMask = 1; + return spec; + } + + int main() + { + std::vector program; + TileXRCcuAllToAllProgramReport report; + + auto spec = ValidSpec(); + spec.memSlicePerBlock = 9; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("memSlicePerBlock") == std::string::npos) { + std::cerr << "memSlicePerBlock > 8 accepted: " << report.message << "\n"; + return 1; + } + + spec = ValidSpec(); + spec.bytes = 4097; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("4KB") == std::string::npos) { + std::cerr << "non-4KB size accepted: " << report.message << "\n"; + return 2; + } + + spec = ValidSpec(); + spec.remoteRecvToken = 0; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("token") == std::string::npos) { + std::cerr << "missing token accepted: " << report.message << "\n"; + return 3; + } + + spec = ValidSpec(); + spec.localRank = 2; + if (TileXRCcuBuildAllToAll2RankProgram(spec, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("localRank") == std::string::npos) { + std::cerr << "invalid localRank accepted: " << report.message << "\n"; + return 4; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + + def test_builder_is_wired_and_has_no_hccl_dependency_surface(self): + cmake = COMM_CMAKE.read_text(encoding="utf-8") + header = ALLTOALL_HEADER.read_text(encoding="utf-8") + source = ALLTOALL_SOURCE.read_text(encoding="utf-8") + + self.assertIn("ccu/tilexr_ccu_alltoall_program.h", cmake) + self.assertIn("ccu/tilexr_ccu_alltoall_program.cpp", cmake) + self.assertIn("TileXRCcuBuildAllToAll2RankProgram", header) + self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) + self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) + self.assertIn("preSyncRemoteTokenNotifyCke", header) + self.assertIn("preSyncTokenLocalWaitCke", header) + self.assertNotIn("tokenLocalWaitCke", source) + self.assertIn("PreSyncSignalMask", source) + self.assertIn("PostSyncSignalMask", source) + self.assertIn("1U << TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID", source) + self.assertIn("1U << TILEXR_CCU_ALLTOALL_TOKEN_XN_ID", source) + self.assertNotIn("1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID", source) + self.assertIn("post.clearWait = true;", source) + self.assertIn("TILEXR_CCU_ALLTOALL_SIGNAL_MASK", header) + self.assertIn("TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK", header) + self.assertIn("TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK", header) + + combined = header + "\n" + source + for needle in PRIVATE_CCU_PRODUCER_NEEDLES: + with self.subTest(needle=needle): + self.assertNotIn(needle, combined) + + def test_local_rank_does_not_split_the_long_mission_into_copy_phases(self): + header = ALLTOALL_HEADER.read_text(encoding="utf-8") + source = ALLTOALL_SOURCE.read_text(encoding="utf-8") + orchestrator = (COMM_DIR / "ccu" / "tilexr_ccu_direct_orchestrator.cpp").read_text(encoding="utf-8") + planner = (COMM_DIR / "ccu" / "tilexr_ccu_collective_planner.cpp").read_text(encoding="utf-8") + + self.assertIn("uint32_t localRank = 0", header) + self.assertNotIn("append copy only for the local rank's active phase", source) + self.assertNotIn("for (uint32_t phase = 0; phase < 2U; ++phase)", source) + self.assertIn("alltoallSpec.localRank = alltoall.localRank", orchestrator) + self.assertIn("alltoall.localRank = static_cast(rank)", planner) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 55e645b8..8118445d 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -223,6 +223,24 @@ def test_p2p_ccu_copy_uses_original_va_for_microcode_and_imported_segva_for_rout self.assertNotIn("QueryDirectCcuProcessMemoryToken(sourceAddr", endpoint_builder) self.assertNotIn("QueryDirectCcuProcessMemoryToken(destinationAddr", endpoint_builder) + def test_alltoall_overrides_only_copy_route_memory_not_sync_routes(self): + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + prepare_alltoall = planner[ + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt"): + planner.index("#endif", planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt")) + ] + override_apply = planner[ + planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride"): + planner.index("#endif", planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride")) + ] + + self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(", prepare_alltoall) + self.assertIn("0U", prepare_alltoall) + self.assertIn("uint32_t routeIndex = 0", override_apply) + self.assertIn("routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_", override_apply) + self.assertIn("++routeIndex", override_apply) + self.assertIn("directCcuRemoteRouteMemoryOverrideAllRoutes_", override_apply) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index b2bb6587..006553b3 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -16,6 +16,8 @@ REPO_ROOT = Path(__file__).resolve().parents[2] DIRECT_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_orchestrator.h" DIRECT_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_orchestrator.cpp" +PLANNER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.h" +PLANNER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp" INSTALL_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_install_provider.cpp" PROVIDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_provider.cpp" PACKAGE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_launch_package.cpp" @@ -27,6 +29,7 @@ BARRIER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_barrier_program.cpp" MICROCODE_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_microcode.cpp" MEMORY_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_memory_program.cpp" +ALLTOALL_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_alltoall_program.cpp" SIGNAL_WAIT_PROGRAM_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_signal_wait_program.cpp" RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_runtime.cpp" LOWER_LAYER_PLAN_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_plan_builder.cpp" @@ -138,6 +141,7 @@ def compile_and_run(self, code: str): str(BARRIER_SOURCE), str(MICROCODE_SOURCE), str(MEMORY_PROGRAM_SOURCE), + str(ALLTOALL_PROGRAM_SOURCE), str(SIGNAL_WAIT_PROGRAM_SOURCE), str(RUNTIME_SOURCE), str(LOWER_LAYER_PLAN_SOURCE), @@ -1684,11 +1688,13 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): cmake = COMM_CMAKE.read_text(encoding="utf-8") header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") self.assertIn("ccu/tilexr_ccu_direct_orchestrator.h", cmake) self.assertIn("ccu/tilexr_ccu_direct_orchestrator.cpp", cmake) self.assertIn("struct TileXRCcuDirectInstallOptions", header) self.assertIn("struct TileXRCcuDirectMemoryCopySpec", header) + self.assertIn("struct TileXRCcuDirectAllToAll2RankSpec", header) self.assertIn("struct TileXRCcuDirectSignalWaitSpec", header) self.assertIn("struct TileXRCcuDirectInstallAttempt", header) self.assertIn("struct TileXRCcuDirectInstallReport", header) @@ -1703,6 +1709,7 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TileXRCcuSubmitPreparedTasks", header) self.assertIn("TileXRCcuRunDirectInstallAttempt", header) self.assertIn("TileXRCcuRunDirectMemoryCopyInstallAttempt", header) + self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", header) self.assertIn("TileXRCcuRunDirectSignalWaitInstallAttempt", header) self.assertIn("TileXRCcuDecodeBasicInfo", source) self.assertIn("TileXRCcuBuildResourceSpec", source) @@ -1710,6 +1717,8 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TileXRCcuBuildLaunchPackage", source) self.assertIn("TileXRCcuBuildMemoryCopyProgram", source) self.assertIn("BuildDirectMemoryCopyLaunchPackage", source) + self.assertIn("TileXRCcuBuildAllToAll2RankProgram", source) + self.assertIn("BuildDirectAllToAll2RankLaunchPackage", source) self.assertIn("TileXRCcuBuildSignalWaitProgram", source) self.assertIn("BuildDirectSignalWaitLaunchPackage", source) self.assertIn("spec.localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke", source) @@ -1735,6 +1744,81 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER", source) + def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", planner) + self.assertIn("alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT", source) + self.assertIn("alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn", source) + self.assertIn("const TileXRCcuSyncResource& copyResource = attempt->plan.syncResources[0]", source) + self.assertIn("const TileXRCcuSyncResource& preResource = attempt->plan.syncResources[1]", source) + self.assertIn("const TileXRCcuSyncResource& postResource = attempt->plan.syncResources[2]", source) + self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(\n 0U,", planner) + self.assertIn( + "static_cast(peerResources.localWaitCkeStartId) +\n peerLocalWaitCkeOffset", + planner, + ) + self.assertNotIn( + "peerResources.remoteNotifyCkeStartId) +\n peerLocalWaitCkeOffset", planner + ) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.localXn : preResource.localXn", source + ) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.localXn : preSyncRemoteAddrXn", source + ) + self.assertIn( + "preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn", source + ) + self.assertIn("preSyncTokenChannelId = preResource.channelId", source) + self.assertIn("preSyncLocalMarkerXn = copyResource.localXn", source) + self.assertIn("preSyncRemoteMarkerXn = copyResource.remoteXn", source) + self.assertIn("preSyncMarkerArgIndex = 0", source) + self.assertIn("preSyncMarkerEnabled = true", source) + self.assertNotIn("preSyncTokenChannelId = postResource.channelId", source) + self.assertIn( + "preSyncOnCopyRoute ? attempt->allocation.remoteNotifyCke.startId : preResource.notifyCke", source + ) + self.assertIn("preSyncRemoteTokenNotifyCke = preResource.notifyCke", source) + self.assertIn("preSyncTokenLocalWaitCke =", source) + self.assertIn("preResource.localWaitCke == 0 ? preResource.notifyCke : preResource.localWaitCke", source) + self.assertIn("alltoallSpec.copyCompletionCke =", source) + self.assertIn("copyResource.localWaitCke == 0 ? copyResource.notifyCke : copyResource.localWaitCke", source) + self.assertIn("postSyncRemoteNotifyCke = postResource.notifyCke", source) + self.assertNotIn("postSyncRemoteNotifyCke = preResource.notifyCke", source) + self.assertIn("postResource.localWaitCke == 0 ? postResource.notifyCke : postResource.localWaitCke", source) + self.assertIn( + "preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId", source + ) + self.assertIn("copyChannelId = copyResource.channelId", source) + self.assertIn("postSyncChannelId = postResource.channelId", source) + self.assertNotIn("postSyncChannelId = preResource.channelId", source) + self.assertIn("postSyncNotify = false", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC", source) + self.assertIn("alltoallSpec.preSyncNotify", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SKIP_PRE_SYNC_WAIT", source) + self.assertIn("alltoallSpec.preSyncWait", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_ON_COPY_ROUTE", source) + self.assertIn("preSyncOnCopyRoute", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_PRE_SYNC_PEER_LOCAL_XN", source) + self.assertIn( + "preSyncPeerLocalXn ? preResource.localXn : preResource.remoteXn", source + ) + self.assertIn( + "preSyncPeerLocalXn ? postResource.localXn : postResource.remoteXn", source + ) + self.assertIn("postSyncWait = false", source) + self.assertIn("emitFinish = false", source) + self.assertNotIn("postSyncNotify = true", source) + self.assertNotIn("postSyncWait = true", source) + self.assertNotIn("emitFinish = true", source) + self.assertIn("LocalToRemote", source) + self.assertIn("uint32_t memSlicePerBlock", header) + combined = header + "\n" + source for needle in PRIVATE_CCU_PRODUCER_NEEDLES: with self.subTest(needle=needle): @@ -1763,6 +1847,44 @@ def test_direct_memory_copy_submit_tasks_keep_planned_runtime_payload(self): self.assertNotIn("NormalizeDirectMemoryCopySubmitTasks", source) self.assertNotIn("task.argSize = 1", source) + def test_collective_planner_has_private_alltoall_prepare_path(self): + header = PLANNER_HEADER.read_text(encoding="utf-8") + source = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", header) + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", source) + self.assertIn("rankSize != 2", source) + self.assertIn("alltoall.localSendAddr = localEndpoint.sourceAddr", source) + self.assertIn("alltoall.localSendToken = localEndpoint.sourceToken", source) + self.assertIn("remoteImportRequest = peerEndpoint.destinationRemoteImport", source) + self.assertIn("alltoall.remoteRecvAddr = remoteImportRequest.addr", source) + self.assertIn("alltoall.remoteRecvToken", source) + self.assertNotIn("alltoall.remoteRecvAddr = peerEndpoint.destinationAddr", source) + self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT", source) + self.assertIn("tilexr-comm-direct-ccu-alltoall", source) + self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) + + def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): + header = DIRECT_HEADER.read_text(encoding="utf-8") + source = DIRECT_SOURCE.read_text(encoding="utf-8") + planner_header = PLANNER_HEADER.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("struct TileXRCcuDirectSyncXnPingSpec", header) + self.assertIn("uint16_t remoteNotifyMask = 0", header) + self.assertIn("uint16_t localWaitMask = 0", header) + self.assertIn("TileXRCcuRunDirectSyncXnPingInstallAttempt", header) + self.assertIn("BuildDirectSyncXnPingLaunchPackage", source) + self.assertIn("TileXRCcuEncodeSyncXn", source) + self.assertIn("defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank)", source) + self.assertIn("defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank)", source) + self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U", source) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner_header) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK", planner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK", planner) + self.assertIn("RegisterCcuResourceRmaBuffer", planner) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index 960cdadc..c76a80a9 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -245,6 +245,8 @@ def test_p2p_ccu_copy_mode_prepares_memory_copy_task(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION", source) self.assertIn("P2pCcuCopyDirectionFromEnv", source) self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) + self.assertIn('"LocalToRemote"', source) + self.assertIn('"1"', source) self.assertIn("ACL_MEMCPY_HOST_TO_DEVICE", source) self.assertIn("ACL_MEMCPY_DEVICE_TO_HOST", source) self.assertIn("p2pCcuCopy", source) @@ -574,6 +576,33 @@ def test_probe_default_sync_instruction_count_includes_hcomm_style_task1_prelude default_body, ) + def test_alltoall_submit_uses_bounded_stream_synchronize(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + alltoall_body = source[ + source.index("int RunAllToAllCopyPhase"): + source.index("int RunSignalWaitSmokeForRank") + ] + + self.assertIn("aclrtSynchronizeStreamWithTimeout", alltoall_body) + self.assertIn("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", alltoall_body) + self.assertIn("PrintMissionContext(context, attempt.submitTasks.front(), \"tilexr_ccu_alltoall\")", alltoall_body) + self.assertNotIn("const int syncRet = aclrtSynchronizeStream(stream);", alltoall_body) + + def test_alltoall_inactive_rank_still_prepares_before_host_phase_wait(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + alltoall_body = source[ + source.index("int RunAllToAllCopyPhase"): + source.index("int RunAllToAllSmokeForRank") + ] + + prepare_index = alltoall_body.index("PrepareDirectCcuMemoryCopyInstallAttempt") + inactive_index = alltoall_body.index("if (!active)") + self.assertLess(prepare_index, inactive_index) + self.assertIn("const bool phaseReady =", alltoall_body) + self.assertIn("WaitForCollectiveSubmitReadiness(rank, rankSize, installReport.submitReady, phase)", alltoall_body) + self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, phase)", alltoall_body) + self.assertIn("RankPhaseFileStem", source) + def test_thread_mode_rank_specific_resource_env_overrides_common_prepare_options(self): source = PROBE_SOURCE.read_text(encoding="utf-8") prepare_options_body = source[ @@ -622,6 +651,113 @@ def test_probe_keeps_hcomm_hccl_and_runtime_launch_out_of_source(self): self.assertNotIn("runtime/kernel.h", source) self.assertNotIn("rtCCULaunch", source) + def test_alltoall_smoke_mode_is_opt_in_and_validates_peer_pattern(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn('kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"', source) + self.assertIn('kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"', source) + self.assertIn("kAllToAllSingleRouteBidirectionalEnv", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL", source) + self.assertIn('kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"', source) + self.assertIn('kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"', source) + self.assertIn("struct AllToAllState", source) + self.assertIn("AllToAllSmokeEnabled", source) + self.assertIn("AllToAllLongMissionEnabled", source) + self.assertIn("AllToAllSingleRouteBidirectionalEnabled", source) + self.assertIn("TileXRCcuMemoryCopyDirection::LocalToRemote", source) + self.assertIn("singleRouteBidirectional", source) + self.assertIn("InitAllToAllState", source) + self.assertIn("RunAllToAllCopyPhase", source) + self.assertIn("RunAllToAllLongMissionSmokeForRank", source) + self.assertIn("RunAllToAllSmokeForRank", source) + self.assertIn("PrepareDirectCcuMemoryCopyInstallAttempt", source) + self.assertIn("PrepareDirectCcuAllToAll2RankInstallAttempt", source) + self.assertIn("TileXRCcuMemoryCopyDirection::RemoteToLocal", source) + self.assertIn("tilexr_ccu_alltoall config", source) + self.assertIn("tilexr_ccu_alltoall result passed=1", source) + self.assertIn("BuildP2pCcuCopyPattern(peer", source) + + def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + body = source[ + source.index("int RunAllToAllLongMissionSmokeForRank"): + source.index("int RunAllToAllSmokeForRank") + ] + + self.assertIn('kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"', source) + self.assertIn("AllToAllLoopCountFromEnv", source) + loop_count_body = source[ + source.index("int AllToAllLoopCountFromEnv"): + source.index("uint64_t BuildAllToAllLoopMarker") + ] + self.assertIn("std::strtol", loop_count_body) + self.assertIn("parsed < 1 || parsed > 1024", loop_count_body) + self.assertIn("BuildAllToAllLoopMarker", source) + self.assertIn("BuildAllToAllLoopPattern", source) + self.assertIn("ResetAllToAllStateForLoop", source) + self.assertIn("ReadAndValidatePeerLoopMarker", source) + self.assertIn("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)", body) + self.assertIn("attempt.submitTasks.front().args[0] =", body) + self.assertIn("WaitForCollectiveSubmitReadiness(", body) + self.assertIn("loopIndex);", body) + self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex)", body) + self.assertIn("adapter.ReadXnRange", source) + self.assertIn("peerLoopMarker", source) + self.assertIn("loopIndex=", body) + self.assertLess( + body.index("PrepareDirectCcuAllToAll2RankInstallAttempt"), + body.index("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)"), + ) + + def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn('kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"', source) + self.assertIn("SyncXnPingSmokeEnabled", source) + self.assertIn("RunSyncXnPingSmokeForRank", source) + self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", source) + sync_ping_body = source[ + source.index("int RunSyncXnPingSmokeForRank"): + source.index("int RunSignalWaitSmokeForRank") + ] + self.assertIn("AllToAllState routeState", sync_ping_body) + self.assertIn("InitAllToAllState(rank, peer, &routeState)", sync_ping_body) + self.assertIn("tilexr_ccu_sync_xn_ping prepare", source) + self.assertIn("tilexr_ccu_sync_xn_ping submit", source) + self.assertIn("tilexr_ccu_sync_xn_ping timing", source) + self.assertIn("aclrtSynchronizeStreamWithTimeout", source) + + def test_alltoall_timeout_prints_xn_and_cke_readback(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrintCcuResourceState", source) + self.assertIn("adapter.ReadXnRange", source) + self.assertIn("adapter.ReadCkeRange", source) + self.assertIn("localXnStartId", source) + self.assertIn("remoteXnStartId", source) + self.assertIn("localWaitCkeStartId", source) + self.assertIn("remoteNotifyCkeStartId", source) + self.assertIn("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", source) + self.assertIn("tilexr_ccu_sync_xn_ping result passed=1", source) + + def test_smoke_runner_forwards_alltoall_env(self): + runner = (REPO_ROOT / "tests" / "ccu" / "run_tilexr_ccu_direct_smoke.sh").read_text(encoding="utf-8") + + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL", runner) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_BYTES", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP", runner) + self.assertIn("TILEXR_CCU_ALLTOALL_LOOP_COUNT", runner) + self.assertIn('if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then', runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-451", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-452", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-458", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-454", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-455", runner) + self.assertNotIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-476", runner) + self.assertIn("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-7", runner) + def test_probe_compiles_and_default_run_skips_without_touching_hardware(self): temp_dir, probe_bin, tile_comm_dir, cann_lib_dir, driver_lib_dir = self.compile_probe() try: diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index c1248231..cad3e2ba 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -34,6 +34,8 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY", source) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_BYTES", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK", source) + self.assertIn("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION", source) self.assertIn("TILEXR_CCU_DIRECT_BARRIER_MODE", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW", source) self.assertIn("TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE", source) @@ -143,6 +145,8 @@ def test_runner_p2p_mode_applies_direct_ccu_resource_defaults(self): self.assertIn('TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}"', source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN="${TILEXR_CCU_DIRECT_SMOKE_FAST_EXIT_AFTER_RUN:-1}"', source) self.assertIn('common_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE=${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_ACTIVE_RANK}")', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_DIRECTION}")', source) self.assertIn("p2p_passed_count=0", source) self.assertIn('grep -q "tilexr_ccu_direct_smoke p2pCcuCopy skipped"', source) self.assertIn("direct CCU P2P CCU-copy produced no passing receiver result", source) @@ -167,6 +171,21 @@ def test_runner_signal_wait_mode_uses_internal_backend_defaults_and_asserts_resu self.assertIn('grep -q "tilexr_ccu_signal_wait submit ret=0"', source) self.assertIn('grep -q "tilexr_ccu_signal_wait result passed=1"', source) + def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): + source = RUNNER.read_text(encoding="utf-8") + + self.assertIn("sync_xn_ping_mode_enabled", source) + self.assertIn("apply_sync_xn_ping_defaults", source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0', source) + self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}"', source) + self.assertIn('TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}"', source) + self.assertIn('common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}")', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}"', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}"', source) + self.assertLess(source.index("apply_sync_xn_ping_defaults"), source.index("apply_alltoall_defaults")) + def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): source = RUNNER.read_text(encoding="utf-8") submit_check = source[ diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py index 5244d89e..57066197 100644 --- a/tests/ccu/test_tilexr_ccu_driver_adapter.py +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -488,6 +488,70 @@ def test_adapter_reads_each_instruction_from_its_own_data_array_slot(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_adapter_reads_xn_and_cke_ranges_from_driver_slots(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_driver_adapter.h" + + #include + #include + + using namespace TileXR; + + struct FakeState { + int calls = 0; + }; + + int FakeCustomChannel( + uint32_t, + const TileXRCcuCustomChannelIn& in, + TileXRCcuCustomChannelOut* out, + void* userData) + { + auto* state = static_cast(userData); + state->calls++; + if (in.data.dataInfo.udieIdx != 1 || in.offsetStartIdx != (in.op == 203 ? 2362 : 332) || + in.data.dataInfo.dataArraySize != 3 || in.data.dataInfo.dataLen != 24) { + return -1; + } + for (uint32_t i = 0; i < 3; ++i) { + const uint64_t value = (static_cast(in.op) << 32U) | i; + std::memcpy(&out->data.dataInfo.dataArray[i], &value, sizeof(value)); + } + out->opRet = 0; + return 0; + } + + int main() + { + FakeState state; + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport report; + if (adapter.Init(5, FakeCustomChannel, &state, &report) != TILEXR_SUCCESS) { + return 1; + } + uint64_t xn[3] {}; + uint64_t cke[3] {}; + if (adapter.ReadXnRange(1, 2362, xn, 3, &report) != TILEXR_SUCCESS || + adapter.ReadCkeRange(1, 332, cke, 3, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 2; + } + if (state.calls != 2 || xn[0] != (203ULL << 32U) || xn[2] != ((203ULL << 32U) | 2U) || + cke[0] != (204ULL << 32U) || cke[2] != ((204ULL << 32U) | 2U)) { + std::cerr << "resource readback mismatch\n"; + return 3; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_adapter_set_instruction_trailer_wire_word_is_offset_then_opcode(self): code = textwrap.dedent( r''' diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index 8db3c238..d9a8d372 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -1576,6 +1576,119 @@ def test_overlay_verified_endpoint_routes_updates_only_matching_endpoint_fields( self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_overlay_verified_endpoint_route_reuses_shared_jetty_for_multi_route_snapshot(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + uint16_t Read16(const uint8_t* raw, uint32_t offset) + { + return static_cast(raw[offset]) | + static_cast(static_cast(raw[offset + 1]) << 8U); + } + + uint16_t DecodeChannelStartJettyId(const TileXRCcuChannelCtxDataV1& ctx) + { + const uint16_t word18 = Read16(ctx.raw, 18); + const uint16_t word20 = Read16(ctx.raw, 20); + return static_cast(((word18 >> 12U) & 0xfU) | ((word20 & 0xfffU) << 4U)); + } + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {1, 2, 3}; + allocation.localXn = {1, 0x1a0, 3}; + allocation.remoteXn = {1, 0x2a0, 3}; + allocation.notifyCke = {1, 0x360, 3}; + allocation.localWaitCke = {1, 0x220, 3}; + allocation.remoteNotifyCke = {1, 0x360, 3}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0x0000009234000000ULL; + remote.memoryTokenId = 0x23456; + remote.memoryTokenValue = 0x5678; + remote.remoteXnId = 0x2a0; + remote.remoteNotifyCke = 0x360; + for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { + remote.remoteEid[i] = static_cast(0x40 + i); + } + remote.tpn = 0x010203; + remote.doorbellVa = 0x1111222233334444ULL; + remote.doorbellTokenId = 0x12345; + remote.doorbellTokenValue = 0; + remote.sqDepth = 8; + remote.startJettyId = 0x400; + remote.endpointRouteVerified = true; + + std::vector remoteCcuBuffers {remote, remote, remote}; + for (uint32_t i = 0; i < remoteCcuBuffers.size(); ++i) { + remoteCcuBuffers[i].remoteCcuVa += i * 0x1000ULL; + remoteCcuBuffers[i].memoryTokenValue += i; + remoteCcuBuffers[i].remoteXnId = static_cast(0x2a0 + i); + remoteCcuBuffers[i].remoteNotifyCke = static_cast(0x360 + i); + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, remoteCcuBuffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template build failed: " << report.message << "\n"; + return 1; + } + + TileXRCcuLowerLayerTransportRoute verified = snapshot.routes[0]; + std::vector verifiedRoutes {verified}; + if (TileXRCcuOverlayVerifiedEndpointRoutes(verifiedRoutes, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "overlay failed: " << report.message << "\n"; + return 2; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "install plan build failed: " << report.message + << " pfeJettyCount=" << snapshot.pfeJettyCount + << " routeCount=" << snapshot.routes.size() << "\n"; + return 3; + } + if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 3 || plan.pfes.empty()) { + std::cerr << "install plan shape mismatch\n"; + return 4; + } + if (plan.channels.size() != 3 || + plan.remoteXnBindings.size() != 3 || + snapshot.pfeJettyCount < snapshot.routes.size()) { + std::cerr << "multi-route lower-layer plan did not preserve all routes\n"; + return 5; + } + if (DecodeChannelStartJettyId(plan.channels[0].ctx) != 0x400 || + DecodeChannelStartJettyId(plan.channels[1].ctx) != 0x400 || + DecodeChannelStartJettyId(plan.channels[2].ctx) != 0x400) { + std::cerr << "multi-route channels did not reuse the verified endpoint jetty\n"; + return 6; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_transport_template_uses_peer_exchanged_remote_xn_ids_when_present(self): code = textwrap.dedent( r''' @@ -2060,8 +2173,8 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): ] self.assertNotIn("SelectDirectCcuRemoteBindingOverride", exchange_body) self.assertIn("peerLocalWaitCkeOffset", exchange_body) - self.assertIn("peerResources.localWaitCkeStartId", exchange_body) self.assertIn("peerResources.localWaitCkeCount", exchange_body) + self.assertIn("peerResources.remoteNotifyCkeCount", exchange_body) self.assertNotIn("allocation.remoteNotifyCke.startId,\n routeIndex", exchange_body) self.assertIn("allocation.localXn.startId", planner_source) self.assertIn("remoteXnStartId", exchange_body) @@ -2131,7 +2244,7 @@ def test_direct_ccu_runtime_owns_resource_window_boundary(self): self.assertNotIn("RefreshDirectCcuLowerLayerPlan();", register_memory_body) self.assertNotIn("ResetDirectCcuLowerLayerPlan();", register_memory_body) - def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): + def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): @@ -2139,13 +2252,12 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): ] compact_body = " ".join(exchange_body.split()) - self.assertIn( - "channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( peerResources.remoteXnStartId, peerLocalIndex, syncIndex, peerRouteCount)", - compact_body) self.assertIn( "peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset)", compact_body) self.assertIn("selectedRemoteXnOffset >= peerResources.remoteXnCount", compact_body) + self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertNotIn("SelectDirectCcuRemoteBindingOverride", compact_body) self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn("(*remoteCcuBuffers)[routeIndex].remoteCcuVa +=", compact_body) @@ -2161,7 +2273,7 @@ def test_remote_xn_exchange_uses_peer_channel_local_xn_operand(self): "static_cast((*remoteCcuBuffers)[routeIndex].remoteXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) - def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): + def test_remote_notify_cke_targets_peer_local_wait_cke(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ planner_source.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): @@ -2171,9 +2283,12 @@ def test_remote_notify_cke_comes_from_peer_exported_local_wait_cke(self): self.assertIn("peerLocalWaitCkeOffset", exchange_body) self.assertIn("peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount", compact_body) + self.assertIn("peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount", compact_body) self.assertIn( "remoteNotifyCke = static_cast(static_cast(peerResources.localWaitCkeStartId) + peerLocalWaitCkeOffset)", compact_body) + self.assertIn("remoteNotifyCke) >= peerResources.localWaitCkeStartId", compact_body) + self.assertIn("peerResources.localWaitCkeStartId) + peerResources.localWaitCkeCount", compact_body) self.assertNotIn( "remoteNotifyCke = SelectDirectCcuRemoteNotifyCkeId( allocation.remoteNotifyCke.startId, routeIndex)", compact_body) @@ -2197,7 +2312,8 @@ def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) self.assertIn("const size_t peerBufferIndex = syncIndex % peerRouteCount", compact_body) self.assertIn("(*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]", compact_body) - self.assertIn("channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self):