diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt index b6749cd8..5c64771e 100644 --- a/src/comm/CMakeLists.txt +++ b/src/comm/CMakeLists.txt @@ -149,6 +149,8 @@ set(TILEXR_SOURCE_FILE tilexr_comm.cpp ccu/tilexr_ccu_runtime.cpp ccu/tilexr_ccu_signal_wait_program.h ccu/tilexr_ccu_signal_wait_program.cpp + ccu/tilexr_ccu_topology.h + ccu/tilexr_ccu_topology.cpp ccu/tilexr_ccu_runtime_session.h ccu/tilexr_ccu_runtime_session.cpp ccu/tilexr_ccu_executor.h diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp index 16a23e26..7e32cf0a 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.cpp +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.cpp @@ -5,9 +5,24 @@ #include "ccu/tilexr_ccu_alltoall_program.h" +#include +#include + namespace TileXR { namespace { +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_SQE_ARGS_TO_X_HEADER = 0x0001U; +constexpr uint16_t TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER = 0x0003U; +constexpr uint16_t TILEXR_CCU_TRACE_SET_CKE_HEADER = 0x0802U; +constexpr uint16_t TILEXR_CCU_TRACE_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER = 0x1000U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MS_TO_LOC_MEM_HEADER = 0x1002U; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; +constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; +constexpr uint16_t TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK = 0xffffU; + uint16_t PreSyncSignalMask(const TileXRCcuAllToAll2RankProgramSpec& spec) { (void)spec; @@ -200,6 +215,33 @@ int AppendRemoteNotify( return TILEXR_SUCCESS; } +int AppendSyncXnNotify( + uint16_t remoteNotifyCke, + uint16_t channelId, + uint16_t localXn, + uint16_t remoteXn, + uint16_t mask, + const char* phase, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuSyncXnSpec notify; + notify.remoteXn = remoteXn; + notify.localXn = localXn; + notify.channelId = channelId; + notify.notifyCke = remoteNotifyCke; + notify.notifyMask = mask; + notify.clearWait = true; + + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncXn(notify, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, std::string("failed to encode direct CCU alltoall ") + phase + + " SyncXn notify"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + int AppendRemoteMarkerNotify( uint16_t remoteNotifyCke, uint16_t channelId, @@ -248,12 +290,14 @@ int AppendPreSyncPhase( spec.preSyncLocalTokenXn == 0 ? spec.lengthXn : spec.preSyncLocalTokenXn; const uint16_t tokenChannelId = spec.preSyncTokenChannelId == 0 ? outputChannelId : spec.preSyncTokenChannelId; + const uint16_t markerChannelId = + spec.preSyncMarkerChannelId == 0 ? outputChannelId : spec.preSyncMarkerChannelId; const uint16_t tokenNotifyCke = spec.preSyncRemoteTokenNotifyCke == 0 ? remoteNotifyCke : spec.preSyncRemoteTokenNotifyCke; if (spec.preSyncMarkerEnabled && AppendRemoteMarkerNotify( remoteNotifyCke, - outputChannelId, + markerChannelId, spec, program, report) != TILEXR_SUCCESS) { @@ -364,6 +408,42 @@ int AppendCopyBlock( return TILEXR_SUCCESS; } +int AppendMeshRemoteCopyBlock( + const TileXRCcuAllToAll2RankProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + uint16_t completionCke, + uint16_t completionMask, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuMemoryCopySpec copy; + copy.direction = TileXRCcuMemoryCopyDirection::LocalToRemote; + copy.localGsa = spec.localGsa; + copy.localXn = spec.localXn; + copy.remoteGsa = spec.remoteGsa; + copy.remoteXn = spec.remoteXn; + copy.lengthXn = spec.lengthXn; + copy.localAddr = spec.localSendAddr + offset; + copy.localToken = spec.localSendToken; + copy.remoteAddr = spec.remoteRecvAddr + offset; + copy.remoteToken = spec.remoteRecvToken; + copy.lengthBytes = bytesPerBlock; + copy.channelId = spec.copyChannelId == 0 ? spec.channelId : spec.copyChannelId; + copy.completionCke = completionCke; + copy.completionMask = completionMask; + + std::vector block; + TileXRCcuMemoryProgramReport memoryReport; + if (TileXRCcuBuildMemoryCopyProgram(copy, &block, &memoryReport) != TILEXR_SUCCESS || block.size() != 7U) { + return Fail(program, report, memoryReport.message.empty() ? + "failed to build direct CCU alltoall mesh remote copy block" : memoryReport.message); + } + block.pop_back(); + program->insert(program->end(), block.begin(), block.end()); + return TILEXR_SUCCESS; +} + int AppendFinish( const TileXRCcuAllToAll2RankProgramSpec& spec, std::vector* program, @@ -398,8 +478,442 @@ void FillReport( report->message = "ok"; } +int ValidateMeshSpec( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (program == nullptr) { + return Fail(program, report, "missing output direct CCU alltoall mesh program"); + } + if (spec.rankSize != 4U || spec.localRank >= spec.rankSize || spec.peers.size() != 3U) { + return Fail(program, report, "direct CCU alltoall mesh requires four ranks and three peers"); + } + if (spec.localSendAddr == 0 || spec.localRecvAddr == 0 || + spec.localSendToken == 0 || spec.localRecvToken == 0 || spec.chunkBytes == 0 || + spec.chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return Fail(program, report, "invalid direct CCU alltoall mesh local buffer"); + } + if (spec.selfSourceGsa == 0 || spec.selfDestinationGsa == 0 || spec.selfSourceXn == 0 || + spec.selfDestinationXn == 0 || spec.selfLengthXn == 0 || + spec.selfCompletionCke == 0 || spec.remoteCompletionCke == 0) { + return Fail(program, report, "missing direct CCU alltoall mesh self-copy resource"); + } + bool peerRanks[4] = {}; + std::set channelIds; + const auto& sharedRoute = spec.peers.front().route; + if (spec.remoteCompletionCke == sharedRoute.sourceCke) { + return Fail(program, report, "alltoall mesh completion CKE overlaps source CKE"); + } + for (const auto& peer : spec.peers) { + if (peer.peerRank >= spec.rankSize || peer.peerRank == spec.localRank || peerRanks[peer.peerRank]) { + return Fail(program, report, "invalid direct CCU alltoall mesh peer rank"); + } + peerRanks[peer.peerRank] = true; + if (peer.route.localRank != spec.localRank || peer.route.localSendAddr != spec.localSendAddr || + peer.route.localSendToken != spec.localSendToken || peer.route.localRecvAddr != spec.localRecvAddr || + peer.route.localRecvToken != spec.localRecvToken || peer.route.bytes != spec.chunkBytes || + peer.route.preSyncMarkerEnabled || !peer.route.preSyncNotify || !peer.route.preSyncWait || + !peer.route.postSyncNotify || !peer.route.postSyncWait) { + return Fail(program, report, "invalid direct CCU alltoall mesh peer route"); + } + if (peer.route.preSyncLocalAddrXn != sharedRoute.preSyncLocalAddrXn || + peer.route.preSyncLocalTokenXn != sharedRoute.preSyncLocalTokenXn || + peer.route.preSyncChannelId != peer.route.copyChannelId || + peer.route.preSyncTokenChannelId != peer.route.copyChannelId || + peer.route.postSyncChannelId != peer.route.copyChannelId || + peer.route.copyCompletionCke != spec.remoteCompletionCke || + peer.route.ckeMask != TILEXR_CCU_ALLTOALL_POST_SYNC_MASK || + !channelIds.insert(peer.route.copyChannelId).second) { + return Fail(program, report, "duplicate direct CCU alltoall mesh peer resource"); + } + TileXRCcuAllToAll2RankProgramSpec validationRoute = peer.route; + validationRoute.localRank = 0; + std::vector ignored; + TileXRCcuAllToAllProgramReport ignoredReport; + if (ValidateSpec(validationRoute, &ignored, &ignoredReport) != TILEXR_SUCCESS) { + return Fail(program, report, ignoredReport.message); + } + } + return TILEXR_SUCCESS; +} + +int AppendMeshPeerPosts( + const std::vector& peers, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + if (peers.empty()) { + return Fail(program, report, "missing direct CCU alltoall mesh peers"); + } + const auto& shared = peers.front().route; + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToXn(shared.preSyncLocalAddrXn, shared.localRecvAddr, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh output variable"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(shared.preSyncLocalTokenXn, shared.localRecvToken, 1, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh token variable"); + } + program->push_back(instr); + + if (AppendSetSourceCke(shared, TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK, program, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + for (const auto& peer : peers) { + const auto& route = peer.route; + if (AppendSyncXnNotify( + route.preSyncRemoteNotifyCke, + route.preSyncChannelId, + shared.preSyncLocalAddrXn, + route.preSyncRemoteAddrXn, + PreSyncSignalMask(route), + "mesh PreSync output", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (AppendSyncXnNotify( + route.preSyncRemoteTokenNotifyCke, + route.preSyncTokenChannelId, + shared.preSyncLocalTokenXn, + route.preSyncRemoteTokenXn, + PreSyncTokenMask(route), + "mesh PreSync token", + program, + report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; +} + +int AppendLocalCopyBlock( + const TileXRCcuAllToAllMeshProgramSpec& spec, + uint64_t offset, + uint64_t bytesPerBlock, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuInstr instr; + if (TileXRCcuEncodeLoadImdToGsa(spec.selfSourceGsa, spec.localSendAddr + offset, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self source address"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfSourceXn, spec.localSendToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self source token"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToGsa(spec.selfDestinationGsa, spec.localRecvAddr + offset, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self destination address"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfDestinationXn, spec.localRecvToken, 1U, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self destination token"); + } + program->push_back(instr); + if (TileXRCcuEncodeLoadImdToXn(spec.selfLengthXn, bytesPerBlock, 0, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to load direct CCU alltoall mesh self length"); + } + program->push_back(instr); + + TileXRCcuLocalMsTransferSpec transfer; + transfer.localGsa = spec.selfSourceGsa; + transfer.localXn = spec.selfSourceXn; + transfer.localMs = 0; + transfer.lengthXn = spec.selfLengthXn; + transfer.channelId = 0; + transfer.setCkeId = spec.selfCompletionCke; + transfer.setCkeMask = 1U; + if (TileXRCcuEncodeTransLocMemToLocMs(transfer, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer to local MS"); + } + program->push_back(instr); + + TileXRCcuCkeSpec wait; + wait.waitCkeId = spec.selfCompletionCke; + wait.waitMask = 1U; + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to wait direct CCU alltoall mesh self transfer"); + } + program->push_back(instr); + + transfer.localGsa = spec.selfDestinationGsa; + transfer.localXn = spec.selfDestinationXn; + if (TileXRCcuEncodeTransLocMsToLocMem(transfer, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh self transfer from local MS"); + } + program->push_back(instr); + if (TileXRCcuEncodeClearCke(wait, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to wait direct CCU alltoall mesh self transfer from local MS"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +int AppendMeshPostNotify( + const TileXRCcuAllToAll2RankProgramSpec& route, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + TileXRCcuSyncCkeSpec post; + post.remoteCke = route.postSyncRemoteNotifyCke; + post.localCke = route.sourceCke; + post.localCkeMask = route.ckeMask; + post.channelId = route.postSyncChannelId; + post.clearWait = true; + TileXRCcuInstr instr; + if (TileXRCcuEncodeSyncCke(post, &instr) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to notify direct CCU alltoall mesh completion"); + } + program->push_back(instr); + return TILEXR_SUCCESS; +} + +uint16_t InstructionSlot(const TileXRCcuInstr& instr, uint32_t slot) +{ + return static_cast( + (instr.words[slot / 4U] >> ((slot % 4U) * 16U)) & 0xffffU); +} + +int FailBindingValidation(TileXRCcuAllToAllProgramReport* report, const std::string& message) +{ + if (report != nullptr) { + report->message = "direct CCU alltoall encoded binding validation failed: " + message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; +} + +bool MatchesSyncXn( + const TileXRCcuInstr& instr, + uint16_t remoteXn, + uint16_t localXn, + uint16_t channelId, + uint16_t notifyCke, + uint16_t notifyMask) +{ + return InstructionSlot(instr, 0) == TILEXR_CCU_TRACE_SYNC_XN_HEADER && + InstructionSlot(instr, 1) == remoteXn && + InstructionSlot(instr, 2) == localXn && + InstructionSlot(instr, 4) == channelId && + InstructionSlot(instr, 5) == notifyCke && + InstructionSlot(instr, 6) == notifyMask; +} + +bool MatchesWait( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t waitCke, + uint16_t waitMask) +{ + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, 4) == waitCke && + InstructionSlot(instr, 5) == waitMask; +} + +bool MatchesTransfer( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t remoteGsa, + uint16_t remoteXn, + uint16_t localGsa, + uint16_t localXn, + uint16_t lengthXn, + uint16_t channelId, + uint16_t completionCke, + uint16_t completionMask) +{ + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, 1) == remoteGsa && + InstructionSlot(instr, 2) == remoteXn && + InstructionSlot(instr, 3) == localGsa && + InstructionSlot(instr, 4) == localXn && + InstructionSlot(instr, 5) == lengthXn && + InstructionSlot(instr, 6) == channelId && + InstructionSlot(instr, 12) == completionCke && + InstructionSlot(instr, 13) == completionMask; +} + +bool MatchesLocalMsTransfer( + const TileXRCcuInstr& instr, + uint16_t header, + uint16_t localGsa, + uint16_t localXn, + uint16_t localMs, + uint16_t lengthXn, + uint16_t channelId, + uint16_t completionCke, + uint16_t completionMask) +{ + const bool memToMs = header == TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER; + return InstructionSlot(instr, 0) == header && + InstructionSlot(instr, memToMs ? 1U : 3U) == localMs && + InstructionSlot(instr, memToMs ? 2U : 1U) == localGsa && + InstructionSlot(instr, memToMs ? 3U : 2U) == localXn && + InstructionSlot(instr, 4) == lengthXn && + InstructionSlot(instr, 5) == channelId && + InstructionSlot(instr, 12) == completionCke && + InstructionSlot(instr, 13) == completionMask; +} + +int ValidateMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + auto peers = spec.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuAllToAllMeshPeerSpec& lhs, + const TileXRCcuAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + const uint32_t blocksPerChunk = static_cast( + spec.chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES); + const size_t expectedSize = 12U + static_cast(blocksPerChunk) * 28U + 6U + 1U; + if (peers.size() != 3U || blocksPerChunk == 0 || program.size() != expectedSize) { + return FailBindingValidation(report, "unexpected mesh program shape"); + } + + const auto& sharedRoute = peers.front().route; + if (InstructionSlot(program[0], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[0], 1) != sharedRoute.preSyncLocalAddrXn || + InstructionSlot(program[1], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[1], 1) != sharedRoute.preSyncLocalTokenXn || + InstructionSlot(program[2], 0) != TILEXR_CCU_TRACE_SET_CKE_HEADER || + InstructionSlot(program[2], 2) != sharedRoute.sourceCke || + InstructionSlot(program[2], 3) != TILEXR_CCU_ALLTOALL_SOURCE_CKE_INIT_MASK) { + return FailBindingValidation(report, "pre-sync variable loads do not match the shared mesh resources"); + } + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& route = peers[ordinal].route; + if (!MatchesSyncXn( + program[3U + ordinal * 2U], + route.preSyncRemoteAddrXn, + sharedRoute.preSyncLocalAddrXn, + route.preSyncChannelId, + route.preSyncRemoteNotifyCke, + PreSyncSignalMask(route))) { + return FailBindingValidation(report, "output SyncXn does not match its peer route"); + } + if (!MatchesSyncXn( + program[4U + ordinal * 2U], + route.preSyncRemoteTokenXn, + sharedRoute.preSyncLocalTokenXn, + route.preSyncTokenChannelId, + route.preSyncRemoteTokenNotifyCke, + PreSyncTokenMask(route))) { + return FailBindingValidation(report, "token SyncXn does not match its peer route"); + } + const size_t wait = 9U + ordinal; + if (!MatchesWait( + program[wait], + TILEXR_CCU_TRACE_SET_CKE_HEADER, + route.preSyncLocalWaitCke, + static_cast(PreSyncSignalMask(route) | PreSyncTokenMask(route)))) { + return FailBindingValidation(report, "pre-sync waits do not match their peer route"); + } + } + + size_t instruction = 12U; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& route = peers[ordinal].route; + if (!MatchesTransfer( + program[instruction + 5U], + TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER, + route.remoteGsa, + route.remoteXn, + route.localGsa, + route.localXn, + route.lengthXn, + route.copyChannelId, + spec.remoteCompletionCke, + static_cast(1U << ordinal))) { + return FailBindingValidation(report, "remote copy does not match its peer route"); + } + instruction += 6U; + } + if (!MatchesLocalMsTransfer( + program[instruction + 5U], + TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MS_HEADER, + spec.selfSourceGsa, + spec.selfSourceXn, + 0, + spec.selfLengthXn, + 0, + spec.selfCompletionCke, + 1U) || + !MatchesWait( + program[instruction + 6U], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.selfCompletionCke, + 1U) || + !MatchesLocalMsTransfer( + program[instruction + 7U], + TILEXR_CCU_TRACE_TRANS_LOC_MS_TO_LOC_MEM_HEADER, + spec.selfDestinationGsa, + spec.selfDestinationXn, + 0, + spec.selfLengthXn, + 0, + spec.selfCompletionCke, + 1U) || + !MatchesWait( + program[instruction + 8U], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.selfCompletionCke, + 1U)) { + return FailBindingValidation(report, "self copy does not match its local route"); + } + instruction += 9U; + if (!MatchesWait( + program[instruction], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + spec.remoteCompletionCke, + 0x7U)) { + return FailBindingValidation(report, "combined remote copy wait does not match the mesh completion CKE"); + } + ++instruction; + } + for (const auto& peer : peers) { + const auto& route = peer.route; + if (InstructionSlot(program[instruction], 0) != TILEXR_CCU_TRACE_SYNC_CKE_HEADER || + InstructionSlot(program[instruction], 1) != route.postSyncRemoteNotifyCke || + InstructionSlot(program[instruction], 2) != route.sourceCke || + InstructionSlot(program[instruction], 3) != route.ckeMask || + InstructionSlot(program[instruction], 4) != route.postSyncChannelId) { + return FailBindingValidation(report, "post-sync notify does not match its peer route"); + } + ++instruction; + } + for (const auto& peer : peers) { + if (!MatchesWait( + program[instruction], + TILEXR_CCU_TRACE_CLEAR_CKE_HEADER, + peer.route.postSyncLocalWaitCke, + peer.route.ckeMask)) { + return FailBindingValidation(report, "post-sync wait does not match its peer route"); + } + ++instruction; + } + if (instruction + 1U != program.size() || + InstructionSlot(program[instruction], 0) != TILEXR_CCU_TRACE_LOAD_IMD_TO_XN_HEADER || + InstructionSlot(program[instruction], 1) != spec.selfSourceXn) { + return FailBindingValidation(report, "finish instruction does not match the mesh program"); + } + return TILEXR_SUCCESS; +} + } // namespace +int TileXRCcuValidateAllToAllMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report) +{ + return ValidateMeshProgramBindings(spec, program, report); +} + int TileXRCcuBuildAllToAll2RankProgram( const TileXRCcuAllToAll2RankProgramSpec& spec, std::vector* program, @@ -468,4 +982,131 @@ int TileXRCcuBuildAllToAll2RankProgram( return TILEXR_SUCCESS; } +int TileXRCcuBuildAllToAllMeshProgram( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report) +{ + ResetReport(report); + if (program != nullptr) { + program->clear(); + } + int ret = ValidateMeshSpec(spec, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + auto peers = spec.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuAllToAllMeshPeerSpec& lhs, + const TileXRCcuAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + const uint64_t bytesPerBlock = TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint32_t blocksPerChunk = static_cast(spec.chunkBytes / bytesPerBlock); + program->reserve(12U + blocksPerChunk * 28U + 6U + 1U); + + ret = AppendMeshPeerPosts(peers, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + for (const auto& peer : peers) { + ret = AppendNotifyWait( + peer.route.preSyncLocalWaitCke, + static_cast(PreSyncSignalMask(peer.route) | PreSyncTokenMask(peer.route)), + "mesh PreSync", + false, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + const uint64_t selfBaseOffset = static_cast(spec.localRank) * spec.chunkBytes; + for (uint32_t block = 0; block < blocksPerChunk; ++block) { + for (size_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const auto& peer = peers[ordinal]; + TileXRCcuAllToAll2RankProgramSpec route = peer.route; + route.localSendAddr = spec.localSendAddr + static_cast(peer.peerRank) * spec.chunkBytes; + route.remoteRecvAddr += static_cast(spec.localRank) * spec.chunkBytes; + ret = AppendMeshRemoteCopyBlock( + route, + static_cast(block) * bytesPerBlock, + bytesPerBlock, + spec.remoteCompletionCke, + static_cast(1U << ordinal), + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + ret = AppendLocalCopyBlock( + spec, + selfBaseOffset + static_cast(block) * bytesPerBlock, + bytesPerBlock, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + ret = AppendNotifyWait( + spec.remoteCompletionCke, + 0x7U, + "mesh Copy", + true, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + for (const auto& peer : peers) { + ret = AppendMeshPostNotify(peer.route, program, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + for (const auto& peer : peers) { + ret = AppendNotifyWait( + peer.route.postSyncLocalWaitCke, + peer.route.ckeMask, + "mesh PostSync", + true, + program, + report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + } + + TileXRCcuInstr finish; + if (TileXRCcuEncodeLoadImdToXn(spec.selfSourceXn, 0, 0, &finish) != TILEXR_SUCCESS) { + return Fail(program, report, "failed to encode direct CCU alltoall mesh finish"); + } + program->push_back(finish); + + if (TileXRCcuValidateAllToAllMeshProgramBindings(spec, *program, report) != TILEXR_SUCCESS) { + program->clear(); + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + if (report != nullptr) { + report->preSyncInstructionCount = 12U; + report->blockCount = blocksPerChunk; + report->bytesPerBlock = static_cast(bytesPerBlock); + report->copyInstructionCount = blocksPerChunk * 28U; + report->postSyncInstructionCount = 6U; + report->finishInstructionCount = 1U; + report->totalInstructionCount = static_cast(program->size()); + report->peerCount = 3U; + report->syncResourceCount = 3U; + report->remoteBlockCount = 3U * blocksPerChunk; + report->selfBlockCount = blocksPerChunk; + report->message = "ok"; + } + return TILEXR_SUCCESS; +} + } // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_alltoall_program.h b/src/comm/ccu/tilexr_ccu_alltoall_program.h index 484ff30a..93783ba2 100644 --- a/src/comm/ccu/tilexr_ccu_alltoall_program.h +++ b/src/comm/ccu/tilexr_ccu_alltoall_program.h @@ -22,6 +22,7 @@ constexpr uint16_t TILEXR_CCU_ALLTOALL_OUTPUT_XN_ID = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_TOKEN_XN_ID = 2U; constexpr uint16_t TILEXR_CCU_ALLTOALL_LOOP_MARKER_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_ID = 3U; +constexpr uint16_t TILEXR_CCU_ALLTOALL_POST_SYNC_MASK = 0x8U; constexpr uint16_t TILEXR_CCU_ALLTOALL_SIGNAL_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK0_SIGNAL_MASK = 1U; constexpr uint16_t TILEXR_CCU_ALLTOALL_RANK1_SIGNAL_MASK = 2U; @@ -50,6 +51,7 @@ struct TileXRCcuAllToAll2RankProgramSpec { uint16_t preSyncRemoteMarkerXn = 0; uint16_t preSyncMarkerArgIndex = 0; uint16_t channelId = 0; + uint16_t preSyncMarkerChannelId = 0; uint16_t preSyncChannelId = 0; uint16_t preSyncTokenChannelId = 0; uint16_t copyChannelId = 0; @@ -73,6 +75,30 @@ struct TileXRCcuAllToAll2RankProgramSpec { bool emitFinish = true; }; +struct TileXRCcuAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + TileXRCcuAllToAll2RankProgramSpec route; +}; + +struct TileXRCcuAllToAllMeshProgramSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 0; + uint16_t selfSourceGsa = 0; + uint16_t selfDestinationGsa = 0; + uint16_t selfSourceXn = 0; + uint16_t selfDestinationXn = 0; + uint16_t selfLengthXn = 0; + uint16_t selfChannelId = 0; + uint16_t selfCompletionCke = 0; + uint16_t remoteCompletionCke = 0; + std::vector peers; +}; + struct TileXRCcuAllToAllProgramReport { uint32_t preSyncInstructionCount = 0; uint32_t blockCount = 0; @@ -81,6 +107,10 @@ struct TileXRCcuAllToAllProgramReport { uint32_t postSyncInstructionCount = 0; uint32_t finishInstructionCount = 0; uint32_t totalInstructionCount = 0; + uint32_t peerCount = 0; + uint32_t syncResourceCount = 0; + uint32_t remoteBlockCount = 0; + uint32_t selfBlockCount = 0; std::string message; }; @@ -89,6 +119,16 @@ int TileXRCcuBuildAllToAll2RankProgram( std::vector* program, TileXRCcuAllToAllProgramReport* report); +int TileXRCcuBuildAllToAllMeshProgram( + const TileXRCcuAllToAllMeshProgramSpec& spec, + std::vector* program, + TileXRCcuAllToAllProgramReport* report); + +int TileXRCcuValidateAllToAllMeshProgramBindings( + const TileXRCcuAllToAllMeshProgramSpec& spec, + const std::vector& program, + TileXRCcuAllToAllProgramReport* report); + } // namespace TileXR #endif // TILEXR_CCU_ALLTOALL_PROGRAM_H diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.cpp b/src/comm/ccu/tilexr_ccu_collective_planner.cpp index f4659250..ad537e2e 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.cpp +++ b/src/comm/ccu/tilexr_ccu_collective_planner.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #ifdef TILEXR_CCU_TESTING #include "runtime/dev.h" @@ -27,7 +28,7 @@ namespace TileXR { constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT = 7U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = 7U + 64U * 7U; -constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; #endif constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; @@ -553,13 +554,29 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } const size_t peerRouteCount = static_cast(rankSize - 1); const size_t syncRouteCount = allocation.remoteXn.num; + size_t routedPeerCount = peerRouteCount; + int selectedDiagnosticPeer = -1; +#ifdef TILEXR_CCU_TESTING + for (const auto &override : directCcuRemoteRouteMemoryOverrides_) { + if (!override.allRoutes && override.syncRouteIndex == 0U && + override.buffer.peerRank < static_cast(rankSize) && + override.buffer.peerRank != static_cast(rank)) { + selectedDiagnosticPeer = static_cast(override.buffer.peerRank); + if (syncRouteCount == 1U) { + routedPeerCount = 1U; + } + break; + } + } +#endif + const size_t routesPerPeer = syncRouteCount / routedPeerCount; if (allocation.localXn.num == 0 || allocation.localWaitCke.num == 0 || allocation.remoteNotifyCke.num == 0 || - allocation.remoteXn.num < static_cast(rankSize - 1) || + allocation.remoteXn.num < routedPeerCount || allocation.localWaitCke.num < allocation.remoteXn.num || allocation.remoteNotifyCke.num < allocation.remoteXn.num || - allocation.channels.num == 0 || + allocation.channels.num == 0 || routesPerPeer == 0 || syncRouteCount % routedPeerCount != 0 || remoteCcuBuffers->size() != peerRouteCount) { if (report != nullptr) { report->message = "invalid direct CCU peer XN/CKE exchange shape"; @@ -602,31 +619,57 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( std::vector peerRanks; peerRanks.reserve(peerRouteCount); + if (selectedDiagnosticPeer >= 0) { + peerRanks.push_back(selectedDiagnosticPeer); + } for (int peer = 0; peer < rankSize; ++peer) { - if (peer != rank) { + if (peer != rank && peer != selectedDiagnosticPeer && routedPeerCount > 1U) { peerRanks.push_back(peer); } } - if (peerRanks.size() != peerRouteCount) { + if (peerRanks.size() != routedPeerCount) { if (report != nullptr) { report->message = "invalid direct CCU peer XN/CKE exchange shape"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } - std::vector peerCcuBuffers = *remoteCcuBuffers; + const std::vector peerCcuBuffers = *remoteCcuBuffers; + std::vector peerCcuBuffersByRank( + static_cast(rankSize), nullptr); + for (const auto &peerCcuBuffer : peerCcuBuffers) { + if (peerCcuBuffer.peerRank >= static_cast(rankSize) || + peerCcuBuffer.peerRank == static_cast(rank) || + peerCcuBuffersByRank[peerCcuBuffer.peerRank] != nullptr) { + if (report != nullptr) { + report->message = "invalid direct CCU peer buffer rank mapping"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + peerCcuBuffersByRank[peerCcuBuffer.peerRank] = &peerCcuBuffer; + } + for (const int peer : peerRanks) { + if (peerCcuBuffersByRank[static_cast(peer)] == nullptr) { + if (report != nullptr) { + report->message = "incomplete direct CCU peer buffer rank mapping"; + } + return TILEXR_ERROR_NOT_FOUND; + } + } remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{}); size_t routeIndex = 0; for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex) { - const size_t peerBufferIndex = syncIndex % peerRouteCount; + const size_t peerBufferIndex = syncIndex / routesPerPeer; + const size_t routeWithinPeer = syncIndex % routesPerPeer; const int peer = peerRanks[peerBufferIndex]; const PeerResourceExchange &peerResources = all[peer]; - const size_t peerLocalIndex = static_cast(rank < peer ? rank : rank - 1); - const uint32_t peerLocalXnOffset = - SelectDirectCcuPeerLocalXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t selectedRemoteXnOffset = - SelectDirectCcuChannelBoundRemoteXnOffset(peerLocalIndex, syncIndex, peerRouteCount); - const uint32_t peerLocalWaitCkeOffset = routeIndex; + const size_t peerLocalIndex = selectedDiagnosticPeer >= 0 ? + 0U : static_cast(rank < peer ? rank : rank - 1); + const uint32_t peerLocalResourceOffset = static_cast( + peerLocalIndex * routesPerPeer + routeWithinPeer); + const uint32_t peerLocalXnOffset = peerLocalResourceOffset; + const uint32_t selectedRemoteXnOffset = peerLocalResourceOffset; + const uint32_t peerLocalWaitCkeOffset = peerLocalResourceOffset; if (peerResources.localXnCount == 0 || peerResources.remoteXnCount == 0 || peerResources.localWaitCkeCount == 0 || @@ -634,7 +677,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( peerResources.channelCount == 0 || peerLocalXnOffset >= peerResources.localXnCount || selectedRemoteXnOffset >= peerResources.remoteXnCount || - peerLocalIndex >= peerResources.channelCount || + peerLocalResourceOffset >= peerResources.channelCount || peerLocalWaitCkeOffset >= peerResources.localWaitCkeCount || peerLocalWaitCkeOffset >= peerResources.remoteNotifyCkeCount) { if (report != nullptr) { @@ -642,17 +685,14 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( } return TILEXR_ERROR_NOT_FOUND; } - uint16_t channelBoundRemoteXnId = SelectDirectCcuChannelBoundRemoteXnId( - peerResources.remoteXnStartId, - peerLocalIndex, - syncIndex, - peerRouteCount); const uint16_t peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset); + const uint16_t channelBoundRemoteXnId = static_cast( + static_cast(peerResources.remoteXnStartId) + selectedRemoteXnOffset); uint16_t remoteNotifyCke = static_cast(static_cast(peerResources.localWaitCkeStartId) + peerLocalWaitCkeOffset); - (*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]; + (*remoteCcuBuffers)[routeIndex] = *peerCcuBuffersByRank[static_cast(peer)]; (*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId; (*remoteCcuBuffers)[routeIndex].remoteNotifyCke = remoteNotifyCke; const bool peerLocalXnOwnerVerified = @@ -672,7 +712,7 @@ int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke( static_cast(peerResources.remoteXnStartId) + peerResources.remoteXnCount && routeIndex < allocation.channels.num && peerResources.channelStartId != 0 && - peerLocalIndex < peerResources.channelCount; + peerLocalResourceOffset < peerResources.channelCount; const bool transportResourceExchangeVerified = notifyCkeOwnerVerified && allocation.localWaitCke.num != 0 && @@ -693,14 +733,18 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride( uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) { - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - 0, - peerRank, - remoteCcuVa, - memoryTokenId, - rawMemoryTokenId, - memoryTokenValue); - directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; + directCcuRemoteRouteMemoryOverrides_.clear(); + DirectCcuRemoteRouteMemoryOverride override; + override.allRoutes = true; + override.applyMemory = true; + override.buffer.peerRank = peerRank; + override.buffer.remoteCcuVa = remoteCcuVa; + override.buffer.memoryTokenId = memoryTokenId; + override.buffer.rawMemoryTokenId = rawMemoryTokenId; + override.buffer.memoryTokenValue = memoryTokenValue; + if (remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0) { + directCcuRemoteRouteMemoryOverrides_.push_back(override); + } } void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( @@ -711,44 +755,50 @@ void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverrideForSyncRou uint32_t rawMemoryTokenId, uint32_t memoryTokenValue) { - directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; - directCcuRemoteRouteMemoryOverride_.peerRank = peerRank; - directCcuRemoteRouteMemoryOverride_.remoteCcuVa = remoteCcuVa; - directCcuRemoteRouteMemoryOverride_.memoryTokenId = memoryTokenId; - directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId = rawMemoryTokenId; - directCcuRemoteRouteMemoryOverride_.memoryTokenValue = memoryTokenValue; - directCcuRemoteRouteMemoryOverrideValid_ = - remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; - directCcuRemoteRouteMemoryOverrideAllRoutes_ = false; - directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = syncRouteIndex; + DirectCcuRemoteRouteMemoryOverride override; + override.syncRouteIndex = syncRouteIndex; + override.buffer.peerRank = peerRank; + override.buffer.remoteCcuVa = remoteCcuVa; + override.buffer.memoryTokenId = memoryTokenId; + override.buffer.rawMemoryTokenId = rawMemoryTokenId; + override.buffer.memoryTokenValue = memoryTokenValue; + override.applyMemory = remoteCcuVa != 0 && memoryTokenId != 0 && memoryTokenValue != 0; + for (auto &existing : directCcuRemoteRouteMemoryOverrides_) { + if (!existing.allRoutes && existing.syncRouteIndex == syncRouteIndex) { + existing = override; + return; + } + } + directCcuRemoteRouteMemoryOverrides_.push_back(override); } void TileXRCcuCollectivePlanner::ClearDirectCcuRemoteRouteMemoryOverride() { - directCcuRemoteRouteMemoryOverride_ = TileXRCcuRemoteCcuBufferInfo {}; - directCcuRemoteRouteMemoryOverrideValid_ = false; - directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; - directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; + directCcuRemoteRouteMemoryOverrides_.clear(); } void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride( std::vector *remoteCcuBuffers) const { - if (!directCcuRemoteRouteMemoryOverrideValid_ || remoteCcuBuffers == nullptr) { + if (directCcuRemoteRouteMemoryOverrides_.empty() || remoteCcuBuffers == nullptr) { return; } uint32_t routeIndex = 0; for (auto &remoteCcuBuffer : *remoteCcuBuffers) { - if (remoteCcuBuffer.peerRank != directCcuRemoteRouteMemoryOverride_.peerRank || - (!directCcuRemoteRouteMemoryOverrideAllRoutes_ && - routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_)) { - ++routeIndex; - continue; + for (const auto &override : directCcuRemoteRouteMemoryOverrides_) { + if (remoteCcuBuffer.peerRank != override.buffer.peerRank || + (!override.allRoutes && override.syncRouteIndex != routeIndex)) { + continue; + } + if (!override.applyMemory) { + break; + } + remoteCcuBuffer.remoteCcuVa = override.buffer.remoteCcuVa; + remoteCcuBuffer.memoryTokenId = override.buffer.memoryTokenId; + remoteCcuBuffer.rawMemoryTokenId = override.buffer.rawMemoryTokenId; + remoteCcuBuffer.memoryTokenValue = override.buffer.memoryTokenValue; + break; } - remoteCcuBuffer.remoteCcuVa = directCcuRemoteRouteMemoryOverride_.remoteCcuVa; - remoteCcuBuffer.memoryTokenId = directCcuRemoteRouteMemoryOverride_.memoryTokenId; - remoteCcuBuffer.rawMemoryTokenId = directCcuRemoteRouteMemoryOverride_.rawMemoryTokenId; - remoteCcuBuffer.memoryTokenValue = directCcuRemoteRouteMemoryOverride_.memoryTokenValue; ++routeIndex; } } @@ -1380,6 +1430,169 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAll2RankInstallAttempt( return ret; } +int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t chunkBytes, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report) +{ + if (!session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "TileXRCcuBackend is not initialized for direct CCU alltoall mesh install attempt"; + } + return TILEXR_ERROR_NOT_INITIALIZED; + } + const int rank = session.Rank(); + const int rankSize = session.RankSize(); + if (rankSize != 4 || rank < 0 || rank >= rankSize || localSourceAddr == 0 || localDestinationAddr == 0 || + chunkBytes == 0 || chunkBytes > std::numeric_limits::max() / 4ULL) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall mesh endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const uint64_t bufferBytes = chunkBytes * 4ULL; + const std::string unavailableMessage = session.DirectCcuRuntimeUnavailableMessage(); + if (!unavailableMessage.empty()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = unavailableMessage; + } + return TILEXR_ERROR_NOT_FOUND; + } + + const uint8_t installDieId = SelectDirectCcuInstallDieId(); + const TileXRCcuBasicInfo *basicInfo = session.GetDirectCcuBasicInfo(); + if (basicInfo == nullptr || basicInfo->dieId != installDieId) { + const int refreshRet = session.RefreshDirectCcuBasicInfo(installDieId); + if (refreshRet != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = session.GetDirectCcuBasicInfoReport().message; + } + return refreshRet; + } + basicInfo = session.GetDirectCcuBasicInfo(); + } + if (basicInfo == nullptr || !session.Available()) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "direct CCU runtime is unavailable for alltoall mesh install attempt"; + } + return TILEXR_ERROR_NOT_FOUND; + } + + int ret = session.RegisterCcuResourceRmaBuffer(basicInfo->resourceAddr); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to register direct CCU resource window before alltoall mesh buffers"; + } + return ret; + } + DirectCcuMemoryCopyEndpoint localEndpoint; + ret = BuildDirectCcuLocalMemoryCopyEndpoint( + session, + static_cast(rank), + localSourceAddr, + localDestinationAddr, + bufferBytes, + &localEndpoint); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to query direct CCU alltoall mesh local buffer token"; + } + return ret; + } + + std::vector allEndpoints(static_cast(rankSize)); + ret = session.AllGather(&localEndpoint, sizeof(localEndpoint), allEndpoints.data()); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to exchange direct CCU alltoall mesh endpoints"; + } + return ret; + } + + TileXRCcuDirectAllToAllMeshSpec alltoall; + alltoall.rankSize = static_cast(rankSize); + alltoall.localRank = static_cast(rank); + alltoall.localSendAddr = localEndpoint.sourceAddr; + alltoall.localSendToken = localEndpoint.sourceToken; + alltoall.localRecvAddr = localEndpoint.destinationAddr; + alltoall.localRecvToken = localEndpoint.destinationToken; + alltoall.chunkBytes = chunkBytes; + for (uint32_t peerRank = 0; peerRank < static_cast(rankSize); ++peerRank) { + if (peerRank == static_cast(rank)) { + continue; + } + const DirectCcuMemoryCopyEndpoint &endpoint = allEndpoints[peerRank]; + if (endpoint.valid == 0 || endpoint.rank != peerRank || endpoint.bytes != bufferBytes || + !endpoint.destinationRemoteImport.valid) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "invalid direct CCU alltoall mesh peer endpoint"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + const TileXRCcuRemoteMemoryBufferImportRequest &remoteImport = endpoint.destinationRemoteImport; + TileXRCcuImportedRemoteMemoryBufferInfo imported; + ret = session.ImportRemoteMemoryBuffer(remoteImport, &imported); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = "failed to import direct CCU alltoall mesh remote destination"; + } + return ret; + } + TileXRCcuDirectAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + peer.remoteRecvAddr = remoteImport.addr; + peer.remoteRecvToken = TileXRCcuPackMemoryToken( + remoteImport.tokenId, remoteImport.tokenValue, true); + alltoall.peers.push_back(peer); + } + + TileXRCcuDriverAdapter adapter; + TileXRCcuDriverAdapterReport adapterReport; + ret = session.CreateDriverAdapter(&adapter, &adapterReport); + if (ret != TILEXR_SUCCESS) { + if (report != nullptr) { + *report = TileXRCcuDirectInstallReport {}; + report->message = adapterReport.message; + } + return ret; + } + LowerLayerPlanCallbackContext callbackContext {this, &session}; + TileXRCcuDirectInstallOptions next = options; + next.basicInfo = basicInfo; + next.offlineOnly = false; + next.driverAdapter = &adapter; + next.repositoryMemoryOps = TileXRCcuMakeRepositoryDeviceMemoryOps(next.repositoryMemoryAllocMode); + next.repositoryMemoryUserData = nullptr; + next.lowerLayerPlan = nullptr; + next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; + next.lowerLayerPlanUserData = &callbackContext; + next.sqeArgCount = 0; + next.syncResourceCount = 3U; + next.bindingsPerSyncResource = next.bindingsPerSyncResource == 0 ? 1 : next.bindingsPerSyncResource; + if (next.provider.empty()) { + next.provider = "tilexr-comm-direct-ccu-alltoall-mesh"; + } + + ClearDirectCcuRemoteRouteMemoryOverride(); + ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt(next, alltoall, attempt, report); + ClearDirectCcuRemoteRouteMemoryOverride(); + return ret; +} + int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, @@ -1399,7 +1612,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( } const int rank = session.Rank(); const int rankSize = session.RankSize(); - if (rankSize != 2 || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || + if ((rankSize != 2 && rankSize != 4) || localSourceAddr == 0 || localDestinationAddr == 0 || bytes == 0 || peerRank >= static_cast(rankSize) || peerRank == static_cast(rank)) { if (report != nullptr) { *report = TileXRCcuDirectInstallReport {}; @@ -1482,17 +1695,6 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( return TILEXR_ERROR_PARA_CHECK_FAIL; } - TileXRCcuImportedRemoteMemoryBufferInfo importedRemoteBuffer; - TileXRCcuRemoteMemoryBufferImportRequest remoteImportRequest = peerEndpoint.destinationRemoteImport; - ret = session.ImportRemoteMemoryBuffer(remoteImportRequest, &importedRemoteBuffer); - if (ret != TILEXR_SUCCESS) { - if (report != nullptr) { - *report = TileXRCcuDirectInstallReport {}; - report->message = "failed to import direct CCU SyncXn ping remote endpoint buffer"; - } - return ret; - } - TileXRCcuDriverAdapter adapter; TileXRCcuDriverAdapterReport adapterReport; ret = session.CreateDriverAdapter(&adapter, &adapterReport); @@ -1515,7 +1717,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( next.prepareLowerLayerPlan = &TileXRCcuCollectivePlanner::PrepareDirectCcuLowerLayerPlanCallback; next.lowerLayerPlanUserData = &callbackContext; next.sqeArgCount = 0; - next.syncResourceCount = 1; + next.syncResourceCount = rankSize == 4 ? 3U : 1U; next.syncInstructionCount = std::max( next.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT); @@ -1533,13 +1735,7 @@ int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt( syncXnPing.localWaitMask = SelectSyncXnPingMask("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK"); - SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute( - 0U, - peerRank, - importedRemoteBuffer.targetSegVa, - remoteImportRequest.tokenId, - remoteImportRequest.rawTokenId, - remoteImportRequest.tokenValue); + SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(0U, peerRank, 0, 0, 0, 0); ret = TileXRCcuRunDirectSyncXnPingInstallAttempt(next, syncXnPing, attempt, report); ClearDirectCcuRemoteRouteMemoryOverride(); return ret; diff --git a/src/comm/ccu/tilexr_ccu_collective_planner.h b/src/comm/ccu/tilexr_ccu_collective_planner.h index f09489ca..97f841ed 100644 --- a/src/comm/ccu/tilexr_ccu_collective_planner.h +++ b/src/comm/ccu/tilexr_ccu_collective_planner.h @@ -72,6 +72,14 @@ class TileXRCcuCollectivePlanner { uint32_t peerRank, TileXRCcuDirectInstallAttempt *attempt, TileXRCcuDirectInstallReport *report); + int PrepareDirectCcuAllToAllMeshInstallAttempt( + TileXRCcuRuntimeSession &session, + const TileXRCcuDirectInstallOptions &options, + uint64_t localSourceAddr, + uint64_t localDestinationAddr, + uint64_t chunkBytes, + TileXRCcuDirectInstallAttempt *attempt, + TileXRCcuDirectInstallReport *report); int PrepareDirectCcuSyncXnPingInstallAttempt( TileXRCcuRuntimeSession &session, const TileXRCcuDirectInstallOptions &options, @@ -140,10 +148,13 @@ class TileXRCcuCollectivePlanner { TileXRCcuLowerLayerTransportRoute directCcuLocalVerifiedEndpointRoute_ = {}; bool directCcuLocalVerifiedEndpointRouteValid_ = false; #ifdef TILEXR_CCU_TESTING - TileXRCcuRemoteCcuBufferInfo directCcuRemoteRouteMemoryOverride_ = {}; - bool directCcuRemoteRouteMemoryOverrideValid_ = false; - bool directCcuRemoteRouteMemoryOverrideAllRoutes_ = true; - uint32_t directCcuRemoteRouteMemoryOverrideSyncRouteIndex_ = 0; + struct DirectCcuRemoteRouteMemoryOverride { + uint32_t syncRouteIndex = 0; + bool allRoutes = false; + bool applyMemory = false; + TileXRCcuRemoteCcuBufferInfo buffer; + }; + std::vector directCcuRemoteRouteMemoryOverrides_; #endif }; diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp index d13fc152..1df19674 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.cpp @@ -27,6 +27,7 @@ constexpr uint16_t TILEXR_CCU_TRACE_SET_CKE_HEADER = 0x0802U; constexpr uint16_t TILEXR_CCU_TRACE_CLEAR_CKE_HEADER = 0x0804U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint16_t TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; constexpr uint16_t TILEXR_CCU_TRACE_SYNC_CKE_HEADER = 0x100bU; constexpr uint16_t TILEXR_CCU_TRACE_SYNC_XN_HEADER = 0x100dU; constexpr uint64_t TILEXR_CCU_PACKED_TOKEN_VALID_SHIFT = 52ULL; @@ -39,10 +40,70 @@ constexpr uint32_t TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT = 2U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT = 7U + 64U * 7U; +constexpr uint32_t TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT = 3U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_WAIT_INSTRUCTION_COUNT = 5U; constexpr uint32_t TILEXR_CCU_DIRECT_SIGNAL_WAIT_INSTRUCTION_COUNT = 6U; -constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U; +constexpr uint32_t TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U; + +uint32_t SyncXnPingAllocationInstructionCount(uint32_t syncResourceCount) +{ + if (syncResourceCount > std::numeric_limits::max() / 2U) { + return std::numeric_limits::max(); + } + return syncResourceCount * 2U; +} + +uint32_t DirectAllToAllMeshInstructionCount(uint64_t chunkBytes) +{ + if (chunkBytes == 0 || chunkBytes % TILEXR_CCU_ALLTOALL_BLOCK_BYTES != 0) { + return 0; + } + const uint64_t blocks = chunkBytes / TILEXR_CCU_ALLTOALL_BLOCK_BYTES; + const uint64_t instructions = 11ULL + blocks * 28ULL + 7ULL + 1ULL; + return instructions > std::numeric_limits::max() ? 0U : static_cast(instructions); +} + +bool DirectAllToAllMeshCapacityFits( + const TileXRCcuResourceSpec& resources, + uint32_t instructionCount, + std::string* message) +{ + const auto require = [message](const char* resource, uint32_t requested, uint32_t available) { + if (requested <= available) { + return true; + } + if (message != nullptr) { + std::ostringstream stream; + stream << "insufficient alltoall mesh " << resource + << " resources requested=" << requested + << " available=" << available; + *message = stream.str(); + } + return false; + }; + const uint32_t missionInstructionStart = resources.missionInstructionStartId == 0 ? + resources.instructionStartId : resources.missionInstructionStartId; + const uint32_t repositoryPrefix = missionInstructionStart - resources.instructionStartId; + const uint32_t localWaitCkeCount = resources.localWaitCkeCount == 0 ? + resources.ckeCount : resources.localWaitCkeCount; + const uint32_t remoteNotifyCkeCount = resources.remoteNotifyCkeCount == 0 ? + resources.ckeCount : resources.remoteNotifyCkeCount; + return require("mission", 1U, resources.missionCount) && + require("instruction", repositoryPrefix + instructionCount, resources.instructionCount) && + require("GSA", TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT, resources.gsaCount) && + require("local XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.xnCount) && + require("remote XN", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + resources.remoteXnCount == 0 ? + (resources.xnCount > TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT ? + resources.xnCount - TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : 0U) : + resources.remoteXnCount) && + require("local CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT + 2U, + localWaitCkeCount) && + require("remote CKE", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, + remoteNotifyCkeCount) && + require("channel", TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT, resources.channelCount); +} void ResetReport(TileXRCcuDirectInstallReport* report) { @@ -395,6 +456,27 @@ void TraceDecodedInstr(const char* label, size_t index, const TileXRCcuInstr& in << " waitCkeMask=" << TraceSlot(instr.words[3], 3); break; } + case TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER: { + const uint16_t control = TraceSlot(instr.words[1], 3); + const uint16_t flags = TraceSlot(instr.words[2], 3); + std::cerr << "decoded=TransLocMemToLocMem" + << " sourceGsa=" << TraceSlot(instr.words[0], 3) + << " sourceXn=" << TraceSlot(instr.words[1], 0) + << " destinationGsa=" << TraceSlot(instr.words[0], 1) + << " destinationXn=" << TraceSlot(instr.words[0], 2) + << " lengthXn=" << TraceSlot(instr.words[1], 1) + << " channelId=" << TraceSlot(instr.words[1], 2) + << " clearType=" << (flags & 0x1U) + << " lengthEn=" << ((flags >> 1U) & 0x1U) + << " reduceEn=" << ((flags >> 2U) & 0x1U) + << " reduceDataType=" << ((control >> 8U) & 0xfU) + << " reduceOpCode=" << ((control >> 12U) & 0xfU) + << " setCkeId=" << TraceSlot(instr.words[3], 0) + << " setCkeMask=" << TraceSlot(instr.words[3], 1) + << " waitCkeId=" << TraceSlot(instr.words[3], 2) + << " waitCkeMask=" << TraceSlot(instr.words[3], 3); + break; + } case TILEXR_CCU_TRACE_SYNC_XN_HEADER: std::cerr << "decoded=SyncXn" << " remoteXn=" << TraceSlot(instr.words[0], 1) @@ -804,6 +886,48 @@ int ConfigureDirectAllToAll2RankResources( return TILEXR_SUCCESS; } +int ConfigureDirectAllToAllMeshResources( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + const uint32_t instructionCount = DirectAllToAllMeshInstructionCount(alltoall.chunkBytes); + if (attempt == nullptr || instructionCount == 0 || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.taskWindows.size() != 1U) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU plan requires three peer sync resources and one task"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (options.gsaStartId == 0 || attempt->resourceSpec.gsaCount < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU requires two kernel-local GSA resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (attempt->allocation.localXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.remoteXn.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.localWaitCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.remoteNotifyCke.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.channels.num < TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->allocation.sourceCke.num < 2U) { + if (report != nullptr) { + report->message = "alltoall mesh direct CCU allocation is missing XN/CKE/channel resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + attempt->allocation.localGsa = MakeRange( + attempt->specInfo.dieId, + options.gsaStartId, + static_cast(TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT)); + attempt->plan.kernelLocalGsa = attempt->allocation.localGsa; + attempt->plan.taskWindows[0].instCnt = static_cast(instructionCount); + return TILEXR_SUCCESS; +} + int BuildDirectMemoryCopyLaunchPackage( const TileXRCcuDirectMemoryCopySpec& memoryCopy, TileXRCcuDirectInstallAttempt* attempt, @@ -962,6 +1086,7 @@ int BuildDirectAllToAll2RankLaunchPackage( alltoallSpec.channelId = copyResource.channelId; alltoallSpec.preSyncChannelId = preSyncOnCopyRoute ? copyResource.channelId : preResource.channelId; + alltoallSpec.preSyncMarkerChannelId = alltoallSpec.preSyncChannelId; alltoallSpec.preSyncTokenChannelId = preResource.channelId; alltoallSpec.copyChannelId = copyResource.channelId; alltoallSpec.postSyncChannelId = postResource.channelId; @@ -1071,6 +1196,172 @@ int BuildDirectAllToAll2RankLaunchPackage( return TILEXR_SUCCESS; } +int ValidateDirectAllToAllMeshRouteResources( + const TileXRCcuAllToAllMeshProgramSpec& mesh, + const TileXRCcuProducerPlan& plan, + TileXRCcuDirectInstallReport* report) +{ + if (mesh.peers.size() != 3U || + plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT) { + if (report != nullptr) { + report->message = "alltoall mesh route binding validation has an invalid shape"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + for (size_t ordinal = 0; ordinal < mesh.peers.size(); ++ordinal) { + const auto& resource = plan.syncResources[ordinal]; + const auto& route = mesh.peers[ordinal].route; + if (route.preSyncMarkerEnabled || + route.preSyncChannelId != resource.channelId || + route.preSyncTokenChannelId != resource.channelId || + route.copyChannelId != resource.channelId || + route.postSyncChannelId != resource.channelId || + route.preSyncLocalWaitCke != resource.localWaitCke || + route.preSyncTokenLocalWaitCke != resource.localWaitCke || + route.postSyncLocalWaitCke != resource.localWaitCke || + route.preSyncRemoteNotifyCke != resource.notifyCke || + route.preSyncRemoteTokenNotifyCke != resource.notifyCke || + route.postSyncRemoteNotifyCke != resource.notifyCke || + route.copyCompletionCke != mesh.remoteCompletionCke) { + if (report != nullptr) { + std::ostringstream stream; + stream << "alltoall mesh route binding mismatch peerRank=" << mesh.peers[ordinal].peerRank + << " ordinal=" << ordinal; + report->message = stream.str(); + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + } + return TILEXR_SUCCESS; +} + +int BuildDirectAllToAllMeshLaunchPackage( + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + if (attempt == nullptr || + attempt->plan.syncResources.size() != TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT || + attempt->plan.kernelLocalGsa.num < TILEXR_CCU_DIRECT_MEMORY_COPY_LOCAL_GSA_COUNT) { + if (report != nullptr) { + report->message = "missing direct CCU alltoall mesh producer resources"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + auto peers = alltoall.peers; + std::sort(peers.begin(), peers.end(), [](const TileXRCcuDirectAllToAllMeshPeerSpec& lhs, + const TileXRCcuDirectAllToAllMeshPeerSpec& rhs) { + return lhs.peerRank < rhs.peerRank; + }); + TileXRCcuAllToAllMeshProgramSpec mesh; + mesh.rankSize = alltoall.rankSize; + mesh.localRank = alltoall.localRank; + mesh.localSendAddr = alltoall.localSendAddr; + mesh.localSendToken = alltoall.localSendToken; + mesh.localRecvAddr = alltoall.localRecvAddr; + mesh.localRecvToken = alltoall.localRecvToken; + mesh.chunkBytes = alltoall.chunkBytes; + mesh.selfSourceGsa = attempt->plan.kernelLocalGsa.startId; + mesh.selfDestinationGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + + const uint16_t localXnStart = attempt->allocation.localXn.startId; + const uint16_t remoteXnStart = attempt->allocation.remoteXn.startId; + mesh.remoteCompletionCke = static_cast(attempt->allocation.sourceCke.startId + 1U); + for (uint32_t ordinal = 0; ordinal < peers.size(); ++ordinal) { + const TileXRCcuSyncResource& resource = attempt->plan.syncResources[ordinal]; + TileXRCcuAllToAllMeshPeerSpec peer; + peer.peerRank = peers[ordinal].peerRank; + auto& route = peer.route; + route.localRank = alltoall.localRank; + route.localSendAddr = alltoall.localSendAddr; + route.localSendToken = alltoall.localSendToken; + route.localRecvAddr = alltoall.localRecvAddr; + route.localRecvToken = alltoall.localRecvToken; + route.remoteRecvAddr = peers[ordinal].remoteRecvAddr; + route.remoteRecvToken = peers[ordinal].remoteRecvToken; + route.bytes = alltoall.chunkBytes; + route.localGsa = attempt->plan.kernelLocalGsa.startId; + route.remoteGsa = static_cast(attempt->plan.kernelLocalGsa.startId + 1U); + route.localXn = localXnStart; + route.remoteXn = static_cast(remoteXnStart + 2U); + route.lengthXn = static_cast(localXnStart + 2U); + route.preSyncLocalAddrXn = localXnStart; + route.preSyncLocalTokenXn = static_cast(localXnStart + 1U); + route.preSyncRemoteAddrXn = remoteXnStart; + route.preSyncRemoteTokenXn = static_cast(remoteXnStart + 1U); + route.preSyncMarkerEnabled = false; + route.preSyncChannelId = resource.channelId; + route.preSyncTokenChannelId = resource.channelId; + route.copyChannelId = resource.channelId; + route.postSyncChannelId = resource.channelId; + route.copyCompletionCke = mesh.remoteCompletionCke; + route.preSyncLocalWaitCke = resource.localWaitCke; + route.preSyncRemoteNotifyCke = resource.notifyCke; + route.preSyncTokenLocalWaitCke = resource.localWaitCke; + route.preSyncRemoteTokenNotifyCke = resource.notifyCke; + route.postSyncLocalWaitCke = resource.localWaitCke; + route.postSyncRemoteNotifyCke = resource.notifyCke; + route.sourceCke = attempt->allocation.sourceCke.startId; + route.ckeMask = static_cast(1U << TILEXR_CCU_ALLTOALL_POST_SYNC_ID); + route.preSyncNotify = true; + route.preSyncWait = true; + route.postSyncNotify = true; + route.postSyncWait = true; + route.emitFinish = false; + mesh.peers.push_back(peer); + } + mesh.selfSourceXn = localXnStart; + mesh.selfDestinationXn = static_cast(localXnStart + 1U); + mesh.selfLengthXn = static_cast(localXnStart + 2U); + mesh.selfChannelId = 0; + mesh.selfCompletionCke = attempt->plan.syncResources[0].localWaitCke; + + if (ValidateDirectAllToAllMeshRouteResources(mesh, attempt->plan, report) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + TileXRCcuProgram program; + TileXRCcuAllToAllProgramReport alltoallReport; + if (TileXRCcuBuildAllToAllMeshProgram(mesh, &program.sync, &alltoallReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = alltoallReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (program.sync.empty() || program.sync.size() > std::numeric_limits::max()) { + if (report != nullptr) { + report->message = "invalid direct CCU alltoall mesh instruction count"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); + + TileXRCcuRepositoryImage repository; + TileXRCcuRepositoryReport repositoryReport; + if (TileXRCcuBuildRepositoryImage(attempt->plan, program, &repository, &repositoryReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = repositoryReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector tasks; + TileXRCcuProducerPlanReport planReport; + if (TileXRCcuBuildTasks(attempt->plan, &tasks, &planReport) != TILEXR_SUCCESS) { + if (report != nullptr) { + report->message = planReport.message; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + attempt->package.plan = attempt->plan; + attempt->package.program = program; + attempt->package.repository = repository; + attempt->package.tasks = tasks; + attempt->package.installScope = TileXRCcuLaunchInstallScope {}; + attempt->package.requiresHardwareInstall = true; + return TILEXR_SUCCESS; +} + uint32_t SignalWaitInstructionCount(TileXRCcuSignalWaitProgramRole role) { if (role == TileXRCcuSignalWaitProgramRole::Wait) { @@ -1223,29 +1514,25 @@ int BuildDirectSyncXnPingLaunchPackage( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - if (attempt == nullptr || attempt->plan.syncResources.size() != 1 || attempt->plan.taskWindows.size() != 1) { + if (attempt == nullptr || attempt->plan.syncResources.empty() || attempt->plan.taskWindows.size() != 1) { if (report != nullptr) { report->message = "missing direct CCU SyncXn ping producer resources"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } - if (syncXnPing.localRank > 1U || syncXnPing.peerRank > 1U || syncXnPing.localRank == syncXnPing.peerRank) { + if (syncXnPing.localRank > 3U || syncXnPing.peerRank > 3U || syncXnPing.localRank == syncXnPing.peerRank) { if (report != nullptr) { - report->message = "direct CCU SyncXn ping requires two distinct rank ids"; + report->message = "direct CCU SyncXn ping requires distinct rank ids in the range [0, 3]"; } return TILEXR_ERROR_PARA_CHECK_FAIL; } const TileXRCcuSyncResource& resource = attempt->plan.syncResources[0]; - const uint16_t localWaitCke = resource.localWaitCke == 0 ? resource.notifyCke : resource.localWaitCke; const uint16_t defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank); - const uint16_t defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank); const uint16_t remoteNotifyMask = syncXnPing.remoteNotifyMask == 0 ? defaultRemoteNotifyMask : syncXnPing.remoteNotifyMask; - const uint16_t localWaitMask = - syncXnPing.localWaitMask == 0 ? defaultLocalWaitMask : syncXnPing.localWaitMask; if (resource.localXn == 0 || resource.remoteXn == 0 || resource.channelId == 0 || - resource.notifyCke == 0 || localWaitCke == 0 || resource.sourceCke == 0) { + resource.notifyCke == 0) { if (report != nullptr) { report->message = "missing direct CCU SyncXn ping XN/CKE/channel resource"; } @@ -1277,44 +1564,6 @@ int BuildDirectSyncXnPingLaunchPackage( return TILEXR_ERROR_PARA_CHECK_FAIL; } program.sync.push_back(instr); - - TileXRCcuCkeSpec source; - source.ckeId = resource.sourceCke; - source.mask = remoteNotifyMask; - source.clearWait = true; - if (TileXRCcuEncodeSetCke(source, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping source CKE set"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); - - TileXRCcuSyncCkeSpec syncCke; - syncCke.remoteCke = resource.notifyCke; - syncCke.localCke = resource.sourceCke; - syncCke.localCkeMask = remoteNotifyMask; - syncCke.channelId = resource.channelId; - syncCke.clearWait = true; - if (TileXRCcuEncodeSyncCke(syncCke, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping SyncCke notify"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); - - TileXRCcuCkeSpec wait; - wait.waitCkeId = localWaitCke; - wait.waitMask = localWaitMask; - wait.clearWait = true; - if (TileXRCcuEncodeSetCke(wait, &instr) != TILEXR_SUCCESS) { - if (report != nullptr) { - report->message = "failed to encode direct CCU SyncXn ping wait"; - } - return TILEXR_ERROR_PARA_CHECK_FAIL; - } - program.sync.push_back(instr); attempt->plan.taskWindows[0].instCnt = static_cast(program.sync.size()); TileXRCcuRepositoryImage repository; @@ -1520,6 +1769,7 @@ int RunDirectInstallAttemptImpl( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectMemoryCopySpec* memoryCopy, const TileXRCcuDirectAllToAll2RankSpec* alltoall, + const TileXRCcuDirectAllToAllMeshSpec* alltoallMesh, const TileXRCcuDirectSignalWaitSpec* signalWait, const TileXRCcuDirectSyncXnPingSpec* syncXnPing, TileXRCcuDirectInstallAttempt* attempt, @@ -1565,27 +1815,45 @@ int RunDirectInstallAttemptImpl( attempt->resourceSpec.missionInstructionStartId = options.missionInstructionStartId; ApplyRemoteXnOptions(options, &attempt->resourceSpec); ApplySplitCkeOptions(options, &attempt->resourceSpec); + if (alltoallMesh != nullptr) { + std::string capacityMessage; + if (!DirectAllToAllMeshCapacityFits( + attempt->resourceSpec, + DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes), + &capacityMessage)) { + return Fail(attempt, report, capacityMessage); + } + } - const bool customProgram = - memoryCopy != nullptr || alltoall != nullptr || signalWait != nullptr || syncXnPing != nullptr; + const bool customProgram = memoryCopy != nullptr || alltoall != nullptr || alltoallMesh != nullptr || + signalWait != nullptr || syncXnPing != nullptr; attempt->resourceRequest.sqeArgCount = customProgram ? 0U : options.sqeArgCount; attempt->resourceRequest.syncResourceCount = + alltoallMesh != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_MESH_SYNC_RESOURCE_COUNT : alltoall != nullptr ? TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT : + syncXnPing != nullptr ? options.syncResourceCount : customProgram ? 1U : options.syncResourceCount; attempt->resourceRequest.syncInstructionCount = memoryCopy != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_MEMORY_COPY_INSTRUCTION_COUNT) : alltoall != nullptr ? std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT) : + alltoallMesh != nullptr ? + std::max(options.syncInstructionCount, + DirectAllToAllMeshInstructionCount(alltoallMesh->chunkBytes)) : signalWait != nullptr ? std::max(options.syncInstructionCount, SignalWaitInstructionCount(*signalWait)) : syncXnPing != nullptr ? - std::max(options.syncInstructionCount, TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT) : + std::max( + options.syncInstructionCount, + SyncXnPingAllocationInstructionCount(options.syncResourceCount)) : options.syncInstructionCount; attempt->resourceRequest.bindingsPerSyncResource = options.bindingsPerSyncResource; - attempt->resourceRequest.barrierMode = + attempt->resourceRequest.sourceCkeCount = alltoallMesh != nullptr ? 2U : 1U; + attempt->resourceRequest.barrierMode = + alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : - syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn : signalWait == nullptr ? options.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); TileXRCcuResourceAllocator allocator; @@ -1623,10 +1891,21 @@ int RunDirectInstallAttemptImpl( "failed to configure direct CCU alltoall resources" : report->message); } + } else if (alltoallMesh != nullptr) { + ret = ConfigureDirectAllToAllMeshResources(options, *alltoallMesh, attempt, report); + if (ret != TILEXR_SUCCESS) { + return Fail( + attempt, + report, + report == nullptr || report->message.empty() ? + "failed to configure direct CCU alltoall mesh resources" : + report->message); + } } attempt->plan.barrierMode = + alltoallMesh != nullptr ? TileXRCcuBarrierMode::SyncCke : alltoall != nullptr ? TileXRCcuBarrierMode::SyncXn : - syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncCke : + syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn : signalWait == nullptr ? attempt->plan.barrierMode : EffectiveSignalWaitBarrierMode(*signalWait); ret = PrepareLowerLayerPlanIfNeeded(options, attempt, report); @@ -1658,6 +1937,8 @@ int RunDirectInstallAttemptImpl( TileXRCcuLaunchPackageReport packageReport; ret = memoryCopy != nullptr ? BuildDirectMemoryCopyLaunchPackage(*memoryCopy, attempt, report) : + alltoallMesh != nullptr ? + BuildDirectAllToAllMeshLaunchPackage(*alltoallMesh, attempt, report) : alltoall != nullptr ? BuildDirectAllToAll2RankLaunchPackage(*alltoall, attempt, report) : signalWait != nullptr ? @@ -1673,6 +1954,8 @@ int RunDirectInstallAttemptImpl( (report == nullptr || report->message.empty() ? (memoryCopy != nullptr ? "failed to build direct CCU memory copy launch package" : + alltoallMesh != nullptr ? + "failed to build direct CCU alltoall mesh launch package" : alltoall != nullptr ? "failed to build direct CCU alltoall launch package" : syncXnPing != nullptr ? @@ -1740,7 +2023,7 @@ int TileXRCcuRunDirectInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectMemoryCopyInstallAttempt( @@ -1759,7 +2042,7 @@ int TileXRCcuRunDirectMemoryCopyInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, &memoryCopy, nullptr, nullptr, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectAllToAll2RankInstallAttempt( @@ -1783,7 +2066,37 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( } return TILEXR_ERROR_PARA_CHECK_FAIL; } - return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, &alltoall, nullptr, nullptr, nullptr, attempt, report); +} + +int TileXRCcuRunDirectAllToAllMeshInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report) +{ + bool peerRanks[4] = {}; + bool valid = alltoall.rankSize == 4U && alltoall.localRank < alltoall.rankSize && + alltoall.localSendAddr != 0 && alltoall.localSendToken != 0 && + alltoall.localRecvAddr != 0 && alltoall.localRecvToken != 0 && + DirectAllToAllMeshInstructionCount(alltoall.chunkBytes) != 0 && alltoall.peers.size() == 3U; + for (const auto& peer : alltoall.peers) { + if (peer.peerRank >= alltoall.rankSize || peer.peerRank == alltoall.localRank || + peerRanks[peer.peerRank] || peer.remoteRecvAddr == 0 || peer.remoteRecvToken == 0) { + valid = false; + break; + } + peerRanks[peer.peerRank] = true; + } + if (!valid) { + ResetReport(report); + ClearAttempt(attempt); + if (report != nullptr) { + report->message = "invalid direct CCU alltoall mesh address/token/rank inputs"; + } + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &alltoall, nullptr, nullptr, attempt, report); } int TileXRCcuRunDirectSignalWaitInstallAttempt( @@ -1792,7 +2105,7 @@ int TileXRCcuRunDirectSignalWaitInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, &signalWait, nullptr, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &signalWait, nullptr, attempt, report); } int TileXRCcuRunDirectSyncXnPingInstallAttempt( @@ -1801,7 +2114,7 @@ int TileXRCcuRunDirectSyncXnPingInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report) { - return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); + return RunDirectInstallAttemptImpl(options, nullptr, nullptr, nullptr, nullptr, &syncXnPing, attempt, report); } int TileXRCcuReleaseDirectInstallAttemptResources(TileXRCcuDirectInstallAttempt& attempt) diff --git a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h index dd5a2ee5..a98b945f 100644 --- a/src/comm/ccu/tilexr_ccu_direct_orchestrator.h +++ b/src/comm/ccu/tilexr_ccu_direct_orchestrator.h @@ -85,6 +85,23 @@ struct TileXRCcuDirectAllToAll2RankSpec { uint32_t memSlicePerBlock = TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_BLOCK; }; +struct TileXRCcuDirectAllToAllMeshPeerSpec { + uint32_t peerRank = 0; + uint64_t remoteRecvAddr = 0; + uint64_t remoteRecvToken = 0; +}; + +struct TileXRCcuDirectAllToAllMeshSpec { + uint32_t rankSize = 4; + uint32_t localRank = 0; + uint64_t localSendAddr = 0; + uint64_t localSendToken = 0; + uint64_t localRecvAddr = 0; + uint64_t localRecvToken = 0; + uint64_t chunkBytes = 0; + std::vector peers; +}; + struct TileXRCcuDirectSignalWaitSpec { TileXRCcuSignalWaitProgramRole role = TileXRCcuSignalWaitProgramRole::Signal; bool overrideBarrierMode = false; @@ -161,6 +178,12 @@ int TileXRCcuRunDirectAllToAll2RankInstallAttempt( TileXRCcuDirectInstallAttempt* attempt, TileXRCcuDirectInstallReport* report); +int TileXRCcuRunDirectAllToAllMeshInstallAttempt( + const TileXRCcuDirectInstallOptions& options, + const TileXRCcuDirectAllToAllMeshSpec& alltoall, + TileXRCcuDirectInstallAttempt* attempt, + TileXRCcuDirectInstallReport* report); + int TileXRCcuRunDirectSignalWaitInstallAttempt( const TileXRCcuDirectInstallOptions& options, const TileXRCcuDirectSignalWaitSpec& signalWait, diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp index 26b91e71..1183e2a2 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.cpp +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.cpp @@ -5,6 +5,8 @@ #include "ccu/tilexr_ccu_direct_runtime.h" +#include "ccu/tilexr_ccu_topology.h" + #include #include #include @@ -30,6 +32,7 @@ constexpr uint32_t TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH = 64; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_MAX_POLLS = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_ASYNC_SLEEP_US = 1000; constexpr uint32_t TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_MAX_ATTEMPTS = 8; +constexpr uint8_t TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT = 16; constexpr int TILEXR_CCU_DIRECT_MAX_RANK_SIZE = 128; constexpr int TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL = 2; constexpr int TILEXR_CCU_HCCP_ASYNC_EAGAIN = 128301; @@ -44,6 +47,8 @@ constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE"; constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV = "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX"; +constexpr const char* TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV = + "TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID"; constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_COLLECTION_MODE"; constexpr const char* TILEXR_CCU_DIRECT_TRACE_ENDPOINT_ROUTE_ENV = @@ -52,11 +57,19 @@ constexpr const char* TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE_ENV = "TILEXR_CCU_DIRECT_TRUST_SYNTHETIC_ENDPOINT_ROUTE"; constexpr const char* TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV = "TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE"; +constexpr const char* TILEXR_CCU_HCCL_ROOT_INFO_PATH = "/etc/hccl_rootinfo.json"; constexpr const char* TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET_ENV = "TILEXR_CCU_DIRECT_REMOTE_CCU_VA_OFFSET"; +constexpr const char* TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE_ENV = + "TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE"; constexpr uint8_t TILEXR_CCU_DIRECT_DEFAULT_DIE_ID = 0; constexpr uint64_t TILEXR_CCU_UB_MEM_PAGE_SIZE = 4096ULL; constexpr uint32_t TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT = 8; +constexpr uint32_t TILEXR_CCU_TP_HANDLE_REQUEST_NUM = 8; +constexpr uint32_t TILEXR_CCU_TP_ATTR_BITMAP_SL = 1U << 10U; +constexpr uint32_t TILEXR_CCU_TP_ATTR_BITMAP_SL_AVAILABLE = 1U << 17U; +constexpr uint32_t TILEXR_CCU_DEFAULT_HCCL_QOS = 4; +constexpr uint32_t TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT = 19U; struct TileXRCcuEndpointTpHandleExchange { uint64_t tpHandles[TILEXR_CCU_DIRECT_MAX_RANK_SIZE] = {}; @@ -109,12 +122,52 @@ bool HasCompleteEndpointRoute(const TileXRCcuLowerLayerTransportRoute& route) route.sqDepth != 0; } +struct TileXRCcuPeerEndpointOffer { + uint64_t resourceAddr = 0; + uint32_t resourceTokenId = 0; + uint32_t resourceRawTokenId = 0; + uint32_t resourceTokenValue = 0; + uint32_t jettyTokenValue = 0; + std::array eid {}; + TileXRCcuHccpQpKey qpKey {}; + uint32_t psn = 0; + uint32_t funcId = 0; + bool funcIdValid = false; + bool valid = false; +}; + +bool SameEid( + const std::array& expected, + const TileXRCcuHccpEid& actual) +{ + return std::memcmp(expected.data(), actual.raw, expected.size()) == 0; +} + bool UseImportedPeerEndpointRoute() { const char* mode = std::getenv(TILEXR_CCU_DIRECT_ENDPOINT_ROUTE_EXCHANGE_MODE_ENV); return mode == nullptr || mode[0] == '\0' || std::strcmp(mode, "imported_peer") == 0; } +uint8_t SelectDirectCcuCleanupDieId() +{ + const char* value = std::getenv("TILEXR_CCU_DIRECT_INSTALL_DIE_ID"); + if (value == nullptr || value[0] == '\0') { + return TILEXR_CCU_DIRECT_DEFAULT_DIE_ID; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value, &end, 10); + return end != value && *end == '\0' && parsed <= UINT8_MAX ? + static_cast(parsed) : + TILEXR_CCU_DIRECT_DEFAULT_DIE_ID; +} + +bool RecoverTaskKillState() +{ + const char* value = std::getenv(TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE_ENV); + return value != nullptr && value[0] != '\0' && value[0] != '0'; +} + uint64_t SelectResourceWindowBytes(const TileXRCcuBasicInfo& basicInfo) { (void)basicInfo; @@ -213,12 +266,16 @@ void TraceEndpointRouteStep(const std::string& message) } } -void TraceTaskKillCleanup(uint8_t dieId, int ret, const TileXRCcuDriverAdapterReport& report) +void TraceTaskKillStep( + const char* step, + uint8_t dieId, + int ret, + const TileXRCcuDriverAdapterReport& report) { if (!TraceEndpointRoute()) { return; } - std::cerr << "TileXRDirectCcuTrace taskKillCleanup" + std::cerr << "TileXRDirectCcuTrace taskKill" << step << " dieId=" << static_cast(dieId) << " ret=" << ret << " opcode=" << report.opcode @@ -237,37 +294,63 @@ void TraceRaCtxEidInfos(const std::vector& eidInfos) std::cerr << "TileXRDirectCcuTrace endpointRoute raCtxEidInfo" << " ordinal=" << i << " eidIndex=" << eidInfos[i].eidIndex + << " dieId=" << eidInfos[i].dieId << " funcId=" << eidInfos[i].funcId + << " devFeature=0x" << std::hex << eidInfos[i].resv << std::dec << " eid=" << FormatEndpointEid(CopyRawEid(eidInfos[i].eid)) << std::endl; } } -bool SelectRaCtxResourceWindowEidInfo( +bool ParseEndpointEid(const char* value, std::array* eid); + +bool BuildRaCtxResourceWindowEidCandidates( int rank, + uint8_t dieId, const std::vector& eidInfos, - TileXRCcuHccpDevEidInfo* selectedEid) + std::vector* candidates) { - if (eidInfos.empty() || selectedEid == nullptr) { + if (eidInfos.empty() || candidates == nullptr) { return false; } + candidates->clear(); TraceRaCtxEidInfos(eidInfos); - const char* configured = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, rank); - if (configured == nullptr) { - *selectedEid = eidInfos[0]; - return true; + const char* configuredEid = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV, rank); + if (configuredEid != nullptr) { + std::array expected {}; + if (!ParseEndpointEid(configuredEid, &expected)) { + return false; + } + for (const auto& eidInfo : eidInfos) { + if (CopyRawEid(eidInfo.eid) == expected) { + candidates->push_back(eidInfo); + return true; + } + } + return false; } - uint64_t configuredIndex = 0; - if (!ParseUnsignedEnv(configured, &configuredIndex) || configuredIndex > 0xffffffffULL) { + const char* configured = SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, rank); + if (configured != nullptr) { + uint64_t configuredIndex = 0; + if (!ParseUnsignedEnv(configured, &configuredIndex) || configuredIndex > 0xffffffffULL) { + return false; + } + for (const auto& eidInfo : eidInfos) { + if (eidInfo.eidIndex == static_cast(configuredIndex)) { + candidates->push_back(eidInfo); + return true; + } + } return false; } - for (const auto& eidInfo : eidInfos) { - if (eidInfo.eidIndex == static_cast(configuredIndex)) { - *selectedEid = eidInfo; - return true; + for (auto it = eidInfos.rbegin(); it != eidInfos.rend(); ++it) { + const bool uboeOnly = + ((it->resv >> TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT) & 1U) != 0U; + if (it->dieId == dieId && !uboeOnly) { + candidates->push_back(*it); } } - return false; + return !candidates->empty(); } bool IsRaCtxLoopEndpointRouteCollectionMode() @@ -316,6 +399,55 @@ bool HasRaCtxEndpointRouteSymbols(const TileXRCcuHccpLoader& loader) loader.RaGetAsyncReqResult != nullptr; } +uint32_t CountAvailableSl(uint16_t mask) +{ + uint32_t count = 0; + for (uint32_t bit = 0; bit < 16U; ++bit) { + count += (mask & (1U << bit)) != 0U ? 1U : 0U; + } + return count; +} + +uint8_t SlAtRank(uint16_t mask, uint32_t rank) +{ + uint32_t seen = 0; + for (uint8_t bit = 0; bit < 16U; ++bit) { + if ((mask & (1U << bit)) != 0U && seen++ == rank) { + return bit; + } + } + return 0; +} + +bool MapQosToTpAndSl( + uint32_t qos, + uint32_t tpCount, + uint16_t slMask, + uint32_t* tpIndex, + uint8_t* mappedSl) +{ + if (tpIndex == nullptr || mappedSl == nullptr || tpCount == 0) { + return false; + } + const uint32_t slCount = CountAvailableSl(slMask); + const uint32_t k = std::min(tpCount, slCount); + if (k == 0) { + return false; + } + const uint32_t groupCount = std::min(8U, k); + const uint32_t q = qos & 7U; + const uint32_t group = k == 3U ? (q < 3U ? 0U : (q < 5U ? 1U : 2U)) : + (q * groupCount) / 8U; + const uint32_t slot = (group * k) / groupCount; + if (slot >= k || slot >= tpCount) { + return false; + } + const uint32_t slRank = (slCount - 1U) - slot; + *tpIndex = (k - 1U) - slot; + *mappedSl = SlAtRank(slMask, slRank); + return true; +} + uint32_t SelectEndpointRouteSqDepth() { return TILEXR_CCU_DEFAULT_DIRECT_SQ_DEPTH; @@ -586,11 +718,25 @@ int TileXRCcuDirectRuntime::Init( initialized_ = true; TileXRCcuDriverAdapter adapter; TileXRCcuDriverAdapterReport adapterReport; + const uint8_t cleanupDieId = SelectDirectCcuCleanupDieId(); int cleanupRet = CreateDriverAdapter(&adapter, &adapterReport); + if (cleanupRet == TILEXR_SUCCESS && RecoverTaskKillState()) { + cleanupRet = adapter.SetTaskKill(cleanupDieId, &adapterReport); + TraceTaskKillStep("Set", cleanupDieId, cleanupRet, adapterReport); + // hcomm treats SET_TASKKILL as a best-effort trigger and gates recovery + // on the following CLEAN_TASKKILL_STATE result. + cleanupRet = TILEXR_SUCCESS; + } if (cleanupRet == TILEXR_SUCCESS) { - cleanupRet = adapter.CleanTaskKillState(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, &adapterReport); + cleanupRet = adapter.CleanTaskKillState(cleanupDieId, &adapterReport); + } + TraceTaskKillStep("Cleanup", cleanupDieId, cleanupRet, adapterReport); + if (RecoverTaskKillState() && cleanupRet != TILEXR_SUCCESS) { + const std::string message = "failed to clean direct CCU task-kill state after explicit recovery: " + + adapterReport.message; + Shutdown(); + return Fail(report, message, cleanupRet); } - TraceTaskKillCleanup(TILEXR_CCU_DIRECT_DEFAULT_DIE_ID, cleanupRet, adapterReport); if (report != nullptr) { report->initialized = true; report->raInitialized = true; @@ -899,27 +1045,64 @@ int TileXRCcuDirectRuntime::RegisterCcuResourceRmaBufferWithRaCtx( if (ret != 0 || queriedEidNum == 0) { return ret == 0 ? TILEXR_ERROR_NOT_FOUND : TILEXR_ERROR_MKIRT; } + eidInfos.resize(queriedEidNum); void* ctxHandle = nullptr; void* tokenIdHandle = nullptr; void* lmemHandle = nullptr; - TileXRCcuHccpDevEidInfo selectedEid {}; - if (!SelectRaCtxResourceWindowEidInfo(options_.rank, eidInfos, &selectedEid)) { + std::vector eidCandidates; + if (!BuildRaCtxResourceWindowEidCandidates( + options_.rank, + SelectDirectCcuCleanupDieId(), + eidInfos, + &eidCandidates)) { return TILEXR_ERROR_PARA_CHECK_FAIL; } TileXRCcuHccpCtxInitCfg ctxCfg {}; ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; ctxCfg.rdma.disabledLiteThread = false; - TileXRCcuHccpCtxInitAttr ctxAttr {}; - ctxAttr.phyId = devicePhyId_; - ctxAttr.ub.eidIndex = selectedEid.eidIndex; - ctxAttr.ub.eid = selectedEid.eid; - - ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); - if (ret != 0 || ctxHandle == nullptr) { - return TILEXR_ERROR_MKIRT; + TileXRCcuHccpDevEidInfo selectedEid {}; + const bool explicitEid = + HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_ENV, options_.rank) || + HasRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV, options_.rank); + const bool canProbeLoopTp = !explicitEid && + loader_.RaGetTpInfoListAsync != nullptr && loader_.RaGetAsyncReqResult != nullptr; + for (const auto& candidate : eidCandidates) { + TileXRCcuHccpCtxInitAttr ctxAttr {}; + ctxAttr.phyId = devicePhyId_; + ctxAttr.ub.eidIndex = candidate.eidIndex; + ctxAttr.ub.eid = candidate.eid; + ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &ctxHandle); + if (ret != 0 || ctxHandle == nullptr) { + ctxHandle = nullptr; + continue; + } + uint64_t loopTpHandle = 0; + const auto candidateEid = CopyRawEid(candidate.eid); + const int probeRet = canProbeLoopTp ? + QueryTpHandleForPeer(ctxHandle, candidateEid, candidateEid, &loopTpHandle) : + TILEXR_SUCCESS; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute loopEidCandidate" + << " eidIndex=" << candidate.eidIndex + << " dieId=" << candidate.dieId + << " funcId=" << candidate.funcId + << " eid=" << FormatEndpointEid(candidateEid) + << " probeRet=" << probeRet + << " tpHandle=0x" << std::hex << loopTpHandle << std::dec + << std::endl; + } + if (probeRet == TILEXR_SUCCESS) { + selectedEid = candidate; + break; + } + (void)loader_.RaCtxDeinit(ctxHandle); + ctxHandle = nullptr; + } + if (ctxHandle == nullptr) { + return TILEXR_ERROR_NOT_FOUND; } TileXRCcuHccpTokenId allocatedToken {}; @@ -1060,8 +1243,51 @@ void TileXRCcuDirectRuntime::ReleasePeerEndpointImports() endpointPeerRemoteQpHandles_.clear(); } +void TileXRCcuDirectRuntime::ReleasePeerEndpointState(TileXRCcuPeerEndpointState* state) +{ + if (state == nullptr) { + return; + } + if (state->remoteQpHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxQpUnimport != nullptr) { + (void)loader_.RaCtxQpUnimport(state->resourceWindow.raCtxHandle, state->remoteQpHandle); + } + if (state->qpHandle != nullptr && loader_.RaCtxQpDestroy != nullptr) { + (void)loader_.RaCtxQpDestroy(state->qpHandle); + } + if (state->cqHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxCqDestroy != nullptr) { + (void)loader_.RaCtxCqDestroy(state->resourceWindow.raCtxHandle, state->cqHandle); + } + if (state->resourceWindow.lmemHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxLmemUnregister != nullptr) { + (void)loader_.RaCtxLmemUnregister( + state->resourceWindow.raCtxHandle, + state->resourceWindow.lmemHandle); + } + if (state->resourceWindow.tokenIdHandle != nullptr && state->resourceWindow.raCtxHandle != nullptr && + loader_.RaCtxTokenIdFree != nullptr) { + (void)loader_.RaCtxTokenIdFree( + state->resourceWindow.raCtxHandle, + state->resourceWindow.tokenIdHandle); + } + if (state->resourceWindow.raCtxHandle != nullptr && loader_.RaCtxDeinit != nullptr) { + (void)loader_.RaCtxDeinit(state->resourceWindow.raCtxHandle); + } + *state = TileXRCcuPeerEndpointState {}; +} + +void TileXRCcuDirectRuntime::ReleasePeerEndpointRoutes() +{ + for (auto it = peerEndpointStates_.rbegin(); it != peerEndpointStates_.rend(); ++it) { + ReleasePeerEndpointState(&*it); + } + peerEndpointStates_.clear(); +} + void TileXRCcuDirectRuntime::ReleaseLocalEndpointRoute() { + ReleasePeerEndpointRoutes(); ReleasePeerEndpointImports(); if (endpointRouteBound_ && endpointQpHandle_ != nullptr && loader_.RaCtxQpUnbind != nullptr) { (void)loader_.RaCtxQpUnbind(endpointQpHandle_); @@ -1189,6 +1415,7 @@ int TileXRCcuDirectRuntime::CollectLocalEndpointRouteWithRaCtxOnce( qpAttr.ub.jfsFlag.bs.errorSuspend = 1; qpAttr.ub.priority = 2; qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; + qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; qpAttr.ub.extMode.cstmFlag.value = 0; qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; qpAttr.ub.extMode.sq.buffVa = sqVa; @@ -1322,6 +1549,9 @@ int TileXRCcuDirectRuntime::RefreshLocalVerifiedEndpointRoute(TileXRCcuDirectRun return Fail(report, "direct CCU resource window is not registered for endpoint route collection", TILEXR_ERROR_NOT_INITIALIZED); } + if (options_.rankSize > 2) { + return PreparePeerEndpointRoutes(report); + } TileXRCcuLowerLayerTransportRoute route; int ret = TILEXR_ERROR_NOT_FOUND; @@ -1413,12 +1643,498 @@ int TileXRCcuDirectRuntime::ExportLocalCcuRmaBuffer(TileXRCcuLocalResourceWindow return TILEXR_SUCCESS; } +int TileXRCcuDirectRuntime::CreatePeerEndpointState( + uint32_t peerRank, + uint32_t peerDevicePhyId, + const std::array& localEid, + const std::array& peerEid, + uint32_t peerOrdinal, + TileXRCcuPeerEndpointState* state) +{ + if (state == nullptr || !resourceWindowRegistered_ || localResourceWindow_.addr == 0 || + !HasRaCtxResourceWindowSymbols(loader_) || !HasRaCtxEndpointRouteSymbols(loader_)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *state = TileXRCcuPeerEndpointState {}; + state->peerRank = peerRank; + state->peerDevicePhyId = peerDevicePhyId; + + TileXRCcuRaInfo raInfo {}; + raInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + raInfo.phyId = devicePhyId_; + uint32_t eidNum = 0; + int ret = loader_.RaGetDevEidInfoNum(raInfo, &eidNum); + if (ret != 0 || eidNum == 0) { + return TILEXR_ERROR_MKIRT; + } + std::vector eidInfos(eidNum); + uint32_t queriedEidNum = eidNum; + ret = loader_.RaGetDevEidInfoList(raInfo, eidInfos.data(), &queriedEidNum); + if (ret != 0 || queriedEidNum == 0) { + return TILEXR_ERROR_MKIRT; + } + const auto eidIt = std::find_if( + eidInfos.begin(), + eidInfos.begin() + queriedEidNum, + [&localEid](const TileXRCcuHccpDevEidInfo& info) { + return SameEid(localEid, info.eid); + }); + if (eidIt == eidInfos.begin() + queriedEidNum) { + return TILEXR_ERROR_NOT_FOUND; + } + state->eidInfo = *eidIt; + + TileXRCcuHccpCtxInitCfg ctxCfg {}; + ctxCfg.mode = TILEXR_CCU_NETWORK_OFFLINE; + ctxCfg.rdma.disabledLiteThread = false; + TileXRCcuHccpCtxInitAttr ctxAttr {}; + ctxAttr.phyId = devicePhyId_; + ctxAttr.ub.eidIndex = state->eidInfo.eidIndex; + ctxAttr.ub.eid = state->eidInfo.eid; + ret = loader_.RaCtxInit(&ctxCfg, &ctxAttr, &state->resourceWindow.raCtxHandle); + if (ret != 0 || state->resourceWindow.raCtxHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + TileXRCcuHccpTokenId allocatedToken {}; + ret = loader_.RaCtxTokenIdAlloc( + state->resourceWindow.raCtxHandle, + &allocatedToken, + &state->resourceWindow.tokenIdHandle); + if (ret != 0 || state->resourceWindow.tokenIdHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + TileXRCcuRaInfo randomInfo {}; + randomInfo.mode = TILEXR_CCU_NETWORK_OFFLINE; + randomInfo.phyId = devicePhyId_; + ret = loader_.RaGetSecRandom(&randomInfo, &state->resourceWindow.tokenValue); + if (ret != 0) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + const uint64_t alignedAddr = AlignResourceWindowAddr(localResourceWindow_.addr); + TileXRCcuHccpMrRegInfo mr {}; + mr.in.mem.addr = alignedAddr; + mr.in.mem.size = localResourceWindow_.bytes + (localResourceWindow_.addr - alignedAddr); + mr.in.ub.flags.value = 0; + mr.in.ub.flags.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + mr.in.ub.flags.bs.tokenIdValid = 1; + mr.in.ub.flags.bs.access = TILEXR_CCU_HCCP_MEM_SEG_ACCESS_DEFAULT; + mr.in.ub.flags.bs.nonPin = 1; + mr.in.ub.tokenValue = state->resourceWindow.tokenValue; + mr.in.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; + ret = loader_.RaCtxLmemRegister( + state->resourceWindow.raCtxHandle, + &mr, + &state->resourceWindow.lmemHandle); + if (ret != 0 || state->resourceWindow.lmemHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + const uint32_t rawTokenId = mr.out.ub.tokenId != 0 ? mr.out.ub.tokenId : allocatedToken.tokenId; + state->resourceWindow.addr = localResourceWindow_.addr; + state->resourceWindow.bytes = localResourceWindow_.bytes; + state->resourceWindow.rawTokenId = rawTokenId; + state->resourceWindow.tokenId = rawTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + state->resourceWindow.targetSegHandle = mr.out.ub.targetSegHandle; + state->resourceWindow.eid = localEid; + state->resourceWindow.eidIndex = state->eidInfo.eidIndex; + state->resourceWindow.funcId = state->eidInfo.funcId; + state->resourceWindow.funcIdValid = true; + state->resourceWindow.raCtxRegistered = true; + + ret = SelectTpRouteForPeer( + state->resourceWindow.raCtxHandle, + localEid, + peerEid, + &state->localTpHandle, + &state->mappedJettyPriority); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointState(state); + return ret; + } + + TileXRCcuHccpCqInfo cqInfo {}; + cqInfo.in.chanHandle = nullptr; + cqInfo.in.depth = TILEXR_CCU_DIRECT_CCU_POLL_CQ_DEPTH; + cqInfo.in.ub.userCtx = 0; + cqInfo.in.ub.mode = TILEXR_CCU_HCCP_JFC_MODE_CCU_POLL; + cqInfo.in.ub.ceqn = 0; + cqInfo.in.ub.flag.value = 0; + ret = loader_.RaCtxCqCreate(state->resourceWindow.raCtxHandle, &cqInfo, &state->cqHandle); + if (ret != 0 || state->cqHandle == nullptr) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + + const uint32_t sqDepth = SelectEndpointRouteSqDepth(); + TileXRCcuHccpQpCreateAttr qpAttr {}; + qpAttr.scqHandle = state->cqHandle; + qpAttr.rcqHandle = state->cqHandle; + qpAttr.srqHandle = state->cqHandle; + qpAttr.sqDepth = sqDepth; + qpAttr.rqDepth = TILEXR_CCU_HCCP_RQ_DEPTH_DEFAULT; + qpAttr.transportMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + qpAttr.ub.mode = static_cast(TILEXR_CCU_HCCP_JETTY_MODE_CCU); + qpAttr.ub.jettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + peerOrdinal); + qpAttr.ub.tokenIdHandle = state->resourceWindow.tokenIdHandle; + qpAttr.ub.tokenValue = state->resourceWindow.tokenValue; + qpAttr.ub.flag.value = 0; + qpAttr.ub.flag.bs.shareJfr = 1; + qpAttr.ub.jfsFlag.bs.errorSuspend = 1; + qpAttr.ub.priority = state->mappedJettyPriority; + qpAttr.ub.rnrRetry = TILEXR_CCU_HCCP_RNR_RETRY_DEFAULT; + qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT; + qpAttr.ub.extMode.cstmFlag.value = 0; + qpAttr.ub.extMode.cstmFlag.bs.sqCstm = 1; + qpAttr.ub.extMode.sq.buffVa = SelectEndpointRouteSqVa(localResourceWindow_) + + static_cast(peerOrdinal) * TILEXR_CCU_DIRECT_SQ_BUFFER_BYTES; + qpAttr.ub.extMode.sq.buffSize = SelectEndpointRouteSqBytes(sqDepth); + qpAttr.ub.extMode.sqebbNum = sqDepth; + ret = loader_.RaCtxQpCreate( + state->resourceWindow.raCtxHandle, + &qpAttr, + &state->qpInfo, + &state->qpHandle); + if (ret != 0 || state->qpHandle == nullptr || state->qpInfo.key.size == 0) { + ReleasePeerEndpointState(state); + return TILEXR_ERROR_MKIRT; + } + state->psn = endpointPsn_++; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace peerEndpoint created" + << " rank=" << options_.rank + << " peerRank=" << peerRank + << " peerDevice=" << peerDevicePhyId + << " localEid=" << FormatEndpointEid(localEid) + << " eidIndex=" << state->eidInfo.eidIndex + << " funcId=" << state->eidInfo.funcId + << " tpHandle=0x" << std::hex << state->localTpHandle << std::dec + << " priority=" << static_cast(state->mappedJettyPriority) + << " qpId=" << state->qpInfo.ub.id + << " psn=" << state->psn + << std::endl; + } + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeReport* report) +{ + ReleasePeerEndpointRoutes(); + std::vector allDevicePhyIds(static_cast(options_.rankSize), 0); + int ret = options_.allGather( + &devicePhyId_, + sizeof(devicePhyId_), + allDevicePhyIds.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "failed to exchange physical device ids for CCU endpoint routes", ret); + } + std::vector peerDevicePhyIds; + std::vector peerRanks; + for (int peer = 0; peer < options_.rankSize; ++peer) { + if (peer != options_.rank) { + peerRanks.push_back(static_cast(peer)); + peerDevicePhyIds.push_back(allDevicePhyIds[static_cast(peer)]); + } + } + std::vector topologyRoutes; + std::string topologyMessage; + ret = TileXRCcuResolvePeerEidRoutes( + TILEXR_CCU_HCCL_ROOT_INFO_PATH, + devicePhyId_, + peerDevicePhyIds, + &topologyRoutes, + &topologyMessage); + if (ret != TILEXR_SUCCESS || topologyRoutes.size() != peerRanks.size()) { + return Fail(report, topologyMessage.empty() ? + "failed to resolve peer-specific CCU EIDs" : topologyMessage, ret); + } + + std::vector> localEidsByPeer( + static_cast(options_.rankSize)); + for (uint32_t ordinal = 0; ordinal < peerRanks.size(); ++ordinal) { + localEidsByPeer[peerRanks[ordinal]] = topologyRoutes[ordinal].localEid; + } + std::vector> allLocalEidsByPeer( + static_cast(options_.rankSize) * static_cast(options_.rankSize)); + ret = options_.allGather( + localEidsByPeer.data(), + localEidsByPeer.size() * sizeof(localEidsByPeer.front()), + allLocalEidsByPeer.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + return Fail(report, "failed to exchange peer-specific CCU topology EIDs", ret); + } + + peerEndpointStates_.reserve(peerRanks.size()); + for (uint32_t ordinal = 0; ordinal < peerRanks.size(); ++ordinal) { + const auto& peerEid = allLocalEidsByPeer[ + static_cast(peerRanks[ordinal]) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + if (IsEmptyEid(peerEid)) { + ReleasePeerEndpointRoutes(); + return Fail(report, "missing reciprocal peer-specific CCU topology EID", TILEXR_ERROR_NOT_FOUND); + } + TileXRCcuPeerEndpointState state; + ret = CreatePeerEndpointState( + peerRanks[ordinal], + peerDevicePhyIds[ordinal], + topologyRoutes[ordinal].localEid, + peerEid, + ordinal, + &state); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to create peer-specific CCU endpoint", ret); + } + peerEndpointStates_.push_back(state); + } + + std::vector localOffers(static_cast(options_.rankSize)); + for (const auto& state : peerEndpointStates_) { + auto& offer = localOffers[state.peerRank]; + offer.resourceAddr = state.resourceWindow.addr; + offer.resourceTokenId = state.resourceWindow.tokenId; + offer.resourceRawTokenId = state.resourceWindow.rawTokenId; + offer.resourceTokenValue = state.resourceWindow.tokenValue; + offer.jettyTokenValue = state.resourceWindow.tokenValue; + offer.eid = state.resourceWindow.eid; + offer.qpKey = state.qpInfo.key; + if (offer.qpKey.size == 0 || offer.qpKey.size > TILEXR_CCU_HCCP_QP_KEY_BYTES) { + ReleasePeerEndpointRoutes(); + return Fail(report, "peer-specific CCU QP key has an invalid size", TILEXR_ERROR_MKIRT); + } + offer.psn = state.psn; + offer.funcId = state.resourceWindow.funcId; + offer.funcIdValid = state.resourceWindow.funcIdValid; + offer.valid = true; + } + std::vector allOffers( + static_cast(options_.rankSize) * static_cast(options_.rankSize)); + ret = options_.allGather( + localOffers.data(), + localOffers.size() * sizeof(TileXRCcuPeerEndpointOffer), + allOffers.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to exchange peer-specific CCU endpoint offers", ret); + } + + std::vector localTpHandles(static_cast(options_.rankSize), 0); + for (auto& state : peerEndpointStates_) { + const auto& peerOffer = allOffers[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + if (!peerOffer.valid || peerOffer.qpKey.size == 0) { + ReleasePeerEndpointRoutes(); + return Fail(report, "missing reciprocal peer-specific CCU endpoint offer", TILEXR_ERROR_NOT_FOUND); + } + localTpHandles[state.peerRank] = state.localTpHandle; + } + std::vector allTpHandles( + static_cast(options_.rankSize) * static_cast(options_.rankSize), 0); + ret = options_.allGather( + localTpHandles.data(), + localTpHandles.size() * sizeof(uint64_t), + allTpHandles.data(), + options_.allGatherUserData); + if (ret != TILEXR_SUCCESS) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to exchange peer-specific CCU TP handles", ret); + } + + for (uint32_t ordinal = 0; ordinal < peerEndpointStates_.size(); ++ordinal) { + auto& state = peerEndpointStates_[ordinal]; + const auto& peerOffer = allOffers[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + const uint64_t localTpHandle = localTpHandles[state.peerRank]; + const uint64_t peerTpHandle = allTpHandles[ + static_cast(state.peerRank) * static_cast(options_.rankSize) + + static_cast(options_.rank)]; + TileXRCcuHccpQpImportInfo importInfo {}; + importInfo.in.key = peerOffer.qpKey; + if (importInfo.in.key.size == 0 || importInfo.in.key.size > TILEXR_CCU_HCCP_QP_KEY_BYTES) { + ReleasePeerEndpointRoutes(); + return Fail(report, "peer-specific remote CCU QP key has an invalid size", TILEXR_ERROR_MKIRT); + } + importInfo.in.ub.mode = TILEXR_CCU_HCCP_JETTY_IMPORT_MODE_EXP; + importInfo.in.ub.tokenValue = peerOffer.jettyTokenValue; + importInfo.in.ub.policy = TILEXR_CCU_HCCP_JETTY_GRP_POLICY_RR; + importInfo.in.ub.type = TILEXR_CCU_HCCP_TARGET_TYPE_JETTY; + importInfo.in.ub.flag.value = 0; + importInfo.in.ub.flag.bs.tokenPolicy = TILEXR_CCU_HCCP_TOKEN_POLICY_PLAIN_TEXT; + importInfo.in.ub.expImportCfg.tpHandle = localTpHandle; + importInfo.in.ub.expImportCfg.peerTpHandle = peerTpHandle; + importInfo.in.ub.expImportCfg.txPsn = state.psn; + importInfo.in.ub.expImportCfg.rxPsn = peerOffer.psn; + importInfo.in.ub.tpType = TILEXR_CCU_HCCP_TP_TYPE_RTP; + ret = loader_.RaCtxQpImport( + state.resourceWindow.raCtxHandle, + &importInfo, + &state.remoteQpHandle); + if (ret != 0 || state.remoteQpHandle == nullptr) { + ReleasePeerEndpointRoutes(); + return Fail(report, "failed to import peer-specific CCU QP", TILEXR_ERROR_MKIRT); + } + state.route.remoteEid = ReverseEndpointEid(peerOffer.eid); + state.route.tpn = importInfo.out.ub.tpn; + state.route.doorbellVa = state.qpInfo.ub.dbAddr; + state.route.doorbellTokenId = + state.qpInfo.ub.dbTokenId >> TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + state.route.doorbellTokenValue = state.resourceWindow.tokenValue; + state.route.sqDepth = SelectEndpointRouteSqDepth(); + state.route.startJettyId = static_cast(state.qpInfo.ub.id); + state.route.remoteCcuVa = peerOffer.resourceAddr; + state.route.memoryTokenId = peerOffer.resourceTokenId; + state.route.memoryTokenValue = peerOffer.resourceTokenValue; + state.route.endpointRouteVerified = true; + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace peerEndpoint imported" + << " rank=" << options_.rank + << " peerRank=" << state.peerRank + << " localEid=" << FormatEndpointEid(state.resourceWindow.eid) + << " peerEid=" << FormatEndpointEid(peerOffer.eid) + << " localTpHandle=0x" << std::hex << localTpHandle + << " peerTpHandle=0x" << peerTpHandle + << std::dec + << " localMemoryTokenId=0x" << std::hex << state.resourceWindow.tokenId + << " peerMemoryTokenId=0x" << peerOffer.resourceTokenId + << " localCcuResourceTokenId=0x" << state.resourceWindow.tokenId + << std::dec + << " localPsn=" << state.psn + << " peerPsn=" << peerOffer.psn + << " tpn=0x" << std::hex << state.route.tpn + << " doorbellVa=0x" << state.route.doorbellVa + << " remoteCcuVa=0x" << state.route.remoteCcuVa + << std::dec + << " taJettyId=" << state.route.startJettyId + << std::endl; + } + } + localVerifiedEndpointRoute_ = peerEndpointStates_.front().route; + localVerifiedEndpointRouteValid_ = true; + if (report != nullptr) { + report->initialized = initialized_; + report->message = "peer-specific direct CCU endpoint routes prepared"; + } + return TILEXR_SUCCESS; +} + int TileXRCcuDirectRuntime::QueryTpHandleForPeer( const std::array& peerEid, uint64_t* tpHandle) { - if (tpHandle == nullptr || IsEmptyEid(localResourceWindow_.eid) || IsEmptyEid(peerEid) || - localResourceWindow_.raCtxHandle == nullptr || loader_.RaGetTpInfoListAsync == nullptr || + return QueryTpHandleForPeer( + localResourceWindow_.raCtxHandle, + localResourceWindow_.eid, + peerEid, + tpHandle); +} + +int TileXRCcuDirectRuntime::SelectTpRouteForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle, + uint8_t* mappedJettyPriority) +{ + if (ctxHandle == nullptr || tpHandle == nullptr || mappedJettyPriority == nullptr || + IsEmptyEid(localEid) || IsEmptyEid(peerEid) || + loader_.RaGetTpInfoListAsync == nullptr || loader_.RaGetTpAttrAsync == nullptr || + loader_.RaSetTpAttrAsync == nullptr || loader_.RaGetAsyncReqResult == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + *tpHandle = 0; + *mappedJettyPriority = 0; + + TileXRCcuHccpGetTpCfg tpCfg {}; + tpCfg.flag.bs.rtp = 1; + tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; + std::copy(localEid.begin(), localEid.end(), tpCfg.localEid.raw); + std::copy(peerEid.begin(), peerEid.end(), tpCfg.peerEid.raw); + + std::array tpInfos {}; + uint32_t tpInfoNum = static_cast(tpInfos.size()); + void* reqHandle = nullptr; + int ret = loader_.RaGetTpInfoListAsync( + ctxHandle, &tpCfg, tpInfos.data(), &tpInfoNum, &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS || tpInfoNum == 0 || tpInfoNum > tpInfos.size()) { + return ret == TILEXR_SUCCESS ? TILEXR_ERROR_NOT_FOUND : ret; + } + + TileXRCcuHccpTpAttr attr {}; + uint32_t attrBitmap = TILEXR_CCU_TP_ATTR_BITMAP_SL_AVAILABLE | + TILEXR_CCU_TP_ATTR_BITMAP_SL; + reqHandle = nullptr; + ret = loader_.RaGetTpAttrAsync( + ctxHandle, tpInfos[0].tpHandle, &attrBitmap, &attr, &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + uint32_t tpIndex = 0; + uint8_t mappedSl = 0; + if (!MapQosToTpAndSl( + TILEXR_CCU_DEFAULT_HCCL_QOS, tpInfoNum, attr.slBitmap, &tpIndex, &mappedSl) || + tpInfos[tpIndex].tpHandle == 0) { + return TILEXR_ERROR_NOT_FOUND; + } + + TileXRCcuHccpTpAttr setAttr {}; + setAttr.sl = mappedSl; + reqHandle = nullptr; + ret = loader_.RaSetTpAttrAsync( + ctxHandle, + tpInfos[tpIndex].tpHandle, + TILEXR_CCU_TP_ATTR_BITMAP_SL, + &setAttr, + &reqHandle); + if (ret != 0 || reqHandle == nullptr) { + return TILEXR_ERROR_MKIRT; + } + ret = WaitRaCtxAsyncRequest(loader_, reqHandle); + if (ret != TILEXR_SUCCESS) { + return ret; + } + + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute selectedTp" + << " localEid=" << FormatEndpointEid(localEid) + << " peerEid=" << FormatEndpointEid(peerEid) + << " tpCount=" << tpInfoNum + << " slBitmap=0x" << std::hex << attr.slBitmap + << " tpIndex=" << std::dec << tpIndex + << " tpHandle=0x" << std::hex << tpInfos[tpIndex].tpHandle + << std::dec << " mappedSl=" << static_cast(mappedSl) + << std::endl; + } + *tpHandle = tpInfos[tpIndex].tpHandle; + *mappedJettyPriority = mappedSl; + return TILEXR_SUCCESS; +} + +int TileXRCcuDirectRuntime::QueryTpHandleForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle) +{ + if (tpHandle == nullptr || IsEmptyEid(localEid) || IsEmptyEid(peerEid) || + ctxHandle == nullptr || loader_.RaGetTpInfoListAsync == nullptr || loader_.RaGetAsyncReqResult == nullptr) { return TILEXR_ERROR_PARA_CHECK_FAIL; } @@ -1429,7 +2145,7 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( tpCfg.flag.bs.rtp = 1; tpCfg.transMode = TILEXR_CCU_HCCP_TRANSPORT_MODE_RM; for (uint32_t i = 0; i < TILEXR_CCU_EID_BYTES; ++i) { - tpCfg.localEid.raw[i] = localResourceWindow_.eid[i]; + tpCfg.localEid.raw[i] = localEid[i]; tpCfg.peerEid.raw[i] = peerEid[i]; } @@ -1437,7 +2153,7 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( uint32_t tpInfoNum = 1; void* reqHandle = nullptr; const int ret = loader_.RaGetTpInfoListAsync( - localResourceWindow_.raCtxHandle, + ctxHandle, &tpCfg, &tpInfo, &tpInfoNum, @@ -1460,6 +2176,13 @@ int TileXRCcuDirectRuntime::QueryTpHandleForPeer( if (tpInfo.tpHandle == 0) { return TILEXR_ERROR_NOT_FOUND; } + if (TraceEndpointRoute()) { + std::cerr << "TileXRDirectCcuTrace endpointRoute peerTpInfoReady" + << " localEid=" << FormatEndpointEid(localEid) + << " peerEid=" << FormatEndpointEid(peerEid) + << " tpHandle=0x" << std::hex << tpInfo.tpHandle + << std::dec << std::endl; + } *tpHandle = tpInfo.tpHandle; return TILEXR_SUCCESS; } @@ -1534,6 +2257,33 @@ int TileXRCcuDirectRuntime::ExportRemoteCcuRmaBuffers(std::vectorreserve(peerEndpointStates_.size()); + for (const auto& state : peerEndpointStates_) { + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = state.route.remoteCcuVa; + remote.peerRank = state.peerRank; + remote.memoryTokenId = state.route.memoryTokenId; + remote.rawMemoryTokenId = state.route.memoryTokenId << TILEXR_CCU_URMA_TOKEN_ID_RIGHT_SHIFT; + remote.memoryTokenValue = state.route.memoryTokenValue; + remote.localPfeId = state.resourceWindow.funcId; + remote.localPfeIdValid = state.resourceWindow.funcIdValid; + remote.remoteEid = state.route.remoteEid; + remote.tpn = state.route.tpn; + remote.doorbellVa = state.route.doorbellVa; + remote.doorbellTokenId = state.route.doorbellTokenId; + remote.doorbellTokenValue = state.route.doorbellTokenValue; + remote.sqDepth = state.route.sqDepth; + remote.startJettyId = state.route.startJettyId; + remote.localDoorbellVa = state.route.doorbellVa; + remote.localDoorbellTokenId = state.route.doorbellTokenId; + remote.localDoorbellTokenValue = state.route.doorbellTokenValue; + remote.localSqDepth = state.route.sqDepth; + remote.endpointRouteVerified = true; + buffers->push_back(remote); + } + return TILEXR_SUCCESS; + } if (options_.rankSize <= 1) { return TILEXR_SUCCESS; } diff --git a/src/comm/ccu/tilexr_ccu_direct_runtime.h b/src/comm/ccu/tilexr_ccu_direct_runtime.h index 83f73dac..f664a89e 100644 --- a/src/comm/ccu/tilexr_ccu_direct_runtime.h +++ b/src/comm/ccu/tilexr_ccu_direct_runtime.h @@ -127,6 +127,21 @@ struct TileXRCcuDirectRuntimeReport { std::string message; }; +struct TileXRCcuPeerEndpointState { + uint32_t peerRank = 0; + uint32_t peerDevicePhyId = 0; + TileXRCcuLocalResourceWindowInfo resourceWindow; + TileXRCcuHccpDevEidInfo eidInfo {}; + void* cqHandle = nullptr; + void* qpHandle = nullptr; + void* remoteQpHandle = nullptr; + TileXRCcuHccpQpCreateInfo qpInfo {}; + TileXRCcuLowerLayerTransportRoute route; + uint32_t psn = 0; + uint64_t localTpHandle = 0; + uint8_t mappedJettyPriority = 0; +}; + class TileXRCcuDirectRuntime { public: int Init(const TileXRCcuDirectRuntimeOptions& options, TileXRCcuDirectRuntimeReport* report); @@ -155,7 +170,26 @@ class TileXRCcuDirectRuntime { int CollectLocalEndpointRouteWithRaCtxOnce( TileXRCcuLowerLayerTransportRoute* route, bool* asyncWaitFailed); + int PreparePeerEndpointRoutes(TileXRCcuDirectRuntimeReport* report); + int CreatePeerEndpointState( + uint32_t peerRank, + uint32_t peerDevicePhyId, + const std::array& localEid, + const std::array& peerEid, + uint32_t peerOrdinal, + TileXRCcuPeerEndpointState* state); + int SelectTpRouteForPeer( + void* ctxHandle, + const std::array& localEid, + const std::array& peerEid, + uint64_t* tpHandle, + uint8_t* mappedJettyPriority); + int QueryTpHandleForPeer( + const std::array& peerEid, + uint64_t* tpHandle); int QueryTpHandleForPeer( + void* ctxHandle, + const std::array& localEid, const std::array& peerEid, uint64_t* tpHandle); int ImportPeerEndpointRoute( @@ -170,6 +204,8 @@ class TileXRCcuDirectRuntime { void ReleaseRegisteredMemoryBuffers(); void ReleaseRegisteredResourceWindow(); void ReleaseLocalEndpointRoute(); + void ReleasePeerEndpointState(TileXRCcuPeerEndpointState* state); + void ReleasePeerEndpointRoutes(); TileXRCcuDirectRuntimeOptions options_; TileXRCcuHccpLoader loader_; @@ -185,6 +221,7 @@ class TileXRCcuDirectRuntime { void* endpointQpHandle_ = nullptr; void* endpointRemoteQpHandle_ = nullptr; std::vector endpointPeerRemoteQpHandles_; + std::vector peerEndpointStates_; std::vector registeredMemoryBuffers_; std::vector importedRemoteMemoryBuffers_; TileXRCcuHccpQpKey endpointQpKey_ = {}; diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp index 71945955..9d507b64 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.cpp +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.cpp @@ -480,6 +480,12 @@ int TileXRCcuDriverAdapter::InstallMsidToken( return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_MSID_TOKEN, in, &out, report); } +int TileXRCcuDriverAdapter::SetTaskKill(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const +{ + TileXRCcuCustomChannelOut out; + return Call(dieId, TILEXR_CCU_U_OP_SET_TASKKILL, &out, report); +} + int TileXRCcuDriverAdapter::CleanTaskKillState(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const { TileXRCcuCustomChannelOut out; @@ -515,21 +521,34 @@ int TileXRCcuDriverAdapter::InstallJettyCtx( if (ctxs == nullptr) { return Fail(report, "missing CCU local jetty context payloads"); } - if (count == 0 || count > TILEXR_CCU_MAX_DATA_ARRAY_SIZE) { + if (count == 0) { return Fail(report, "invalid CCU local jetty context count"); } - TileXRCcuCustomChannelIn in; - InitRequest(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, &in); - in.offsetStartIdx = startJettyCtxId; - in.data.dataInfo.dataArraySize = count; - in.data.dataInfo.dataLen = count * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; - for (uint32_t i = 0; i < count; ++i) { - CopyPayloadToSlot(ctxs[i], &in.data.dataInfo.dataArray[i]); - } + uint32_t remaining = count; + uint32_t offset = startJettyCtxId; + uint32_t inputOffset = 0; + while (remaining > 0) { + const uint32_t batch = std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE); + TileXRCcuCustomChannelIn in; + InitRequest(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, &in); + in.offsetStartIdx = offset; + in.data.dataInfo.dataArraySize = batch; + in.data.dataInfo.dataLen = batch * TILEXR_CCU_LOCAL_JETTY_CTX_BYTES; + for (uint32_t i = 0; i < batch; ++i) { + CopyPayloadToSlot(ctxs[inputOffset + i], &in.data.dataInfo.dataArray[i]); + } - TileXRCcuCustomChannelOut out; - return CallPrepared(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, in, &out, report); + TileXRCcuCustomChannelOut out; + const int ret = CallPrepared(dieId, TILEXR_CCU_U_OP_SET_JETTY_CTX, in, &out, report); + if (ret != TILEXR_SUCCESS) { + return ret; + } + remaining -= batch; + offset += batch; + inputOffset += batch; + } + return TILEXR_SUCCESS; } int TileXRCcuDriverAdapter::InstallChannelCtxV1( diff --git a/src/comm/ccu/tilexr_ccu_driver_adapter.h b/src/comm/ccu/tilexr_ccu_driver_adapter.h index ce6ce16a..b2c9a9e5 100644 --- a/src/comm/ccu/tilexr_ccu_driver_adapter.h +++ b/src/comm/ccu/tilexr_ccu_driver_adapter.h @@ -122,6 +122,7 @@ class TileXRCcuDriverAdapter { uint32_t tokenId, uint32_t tokenValue, TileXRCcuDriverAdapterReport* report) const; + int SetTaskKill(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const; int CleanTaskKillState(uint8_t dieId, TileXRCcuDriverAdapterReport* report) const; int InstallPfeCtx( uint8_t dieId, diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.cpp b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp index fcf96e0d..6d788ce1 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_loader.cpp +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.cpp @@ -155,6 +155,8 @@ int TileXRCcuHccpLoader::Load(TileXRCcuHccpLoaderReport* report) LoadOptionalSymbol(raHandle_, RaCtxQpBind, "RaCtxQpBind", "ra_ctx_qp_bind"); LoadOptionalSymbol(raHandle_, RaCtxQpUnbind, "RaCtxQpUnbind", "ra_ctx_qp_unbind"); LoadOptionalSymbol(raHandle_, RaGetTpInfoListAsync, "RaGetTpInfoListAsync", "ra_get_tp_info_list_async"); + LoadOptionalSymbol(raHandle_, RaGetTpAttrAsync, "RaGetTpAttrAsync", "ra_get_tp_attr_async"); + LoadOptionalSymbol(raHandle_, RaSetTpAttrAsync, "RaSetTpAttrAsync", "ra_set_tp_attr_async"); LoadOptionalSymbol(raHandle_, RaGetAsyncReqResult, "RaGetAsyncReqResult", "ra_get_async_req_result"); runtimeHandle_ = dlopen("libruntime.so", RTLD_NOW); @@ -246,6 +248,8 @@ void TileXRCcuHccpLoader::Unload() RaCtxQpBind = nullptr; RaCtxQpUnbind = nullptr; RaGetTpInfoListAsync = nullptr; + RaGetTpAttrAsync = nullptr; + RaSetTpAttrAsync = nullptr; RaGetAsyncReqResult = nullptr; CollectLocalEndpointRoute = nullptr; loaded_ = false; diff --git a/src/comm/ccu/tilexr_ccu_hccp_loader.h b/src/comm/ccu/tilexr_ccu_hccp_loader.h index ef0f30ef..42cb5f57 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_loader.h +++ b/src/comm/ccu/tilexr_ccu_hccp_loader.h @@ -87,6 +87,8 @@ class TileXRCcuHccpLoader { TileXRCcuRaCtxQpBindFunc RaCtxQpBind = nullptr; TileXRCcuRaCtxQpUnbindFunc RaCtxQpUnbind = nullptr; TileXRCcuRaGetTpInfoListAsyncFunc RaGetTpInfoListAsync = nullptr; + TileXRCcuRaGetTpAttrAsyncFunc RaGetTpAttrAsync = nullptr; + TileXRCcuRaSetTpAttrAsyncFunc RaSetTpAttrAsync = nullptr; TileXRCcuRaGetAsyncReqResultFunc RaGetAsyncReqResult = nullptr; TileXRCcuEndpointRouteProviderFunc CollectLocalEndpointRoute = nullptr; diff --git a/src/comm/ccu/tilexr_ccu_hccp_types.h b/src/comm/ccu/tilexr_ccu_hccp_types.h index 58460d30..9cb9b918 100644 --- a/src/comm/ccu/tilexr_ccu_hccp_types.h +++ b/src/comm/ccu/tilexr_ccu_hccp_types.h @@ -493,6 +493,33 @@ struct TileXRCcuHccpTpInfo { uint32_t resv; }; +#pragma pack(push, 1) +struct TileXRCcuHccpTpAttr { + uint8_t retryTimesInit : 3; + uint8_t at : 5; + uint8_t sip[16]; + uint8_t dip[16]; + uint8_t sma[6]; + uint8_t dma[6]; + uint16_t vlanId : 12; + uint8_t vlanEn : 1; + uint8_t dscp : 6; + uint8_t atTimes : 5; + uint8_t sl : 4; + uint8_t ttl; + uint16_t ackUdpSrcport; + uint16_t dataUdpSrcport; + uint8_t udpSrcportRange : 4; + uint8_t sprayEn : 1; + uint8_t udpGlobalEn : 1; + uint8_t reserve0 : 2; + uint16_t slBitmap; + uint8_t dscpConfigMode : 1; + uint8_t reserve1 : 7; + uint8_t reserved[70]; +}; +#pragma pack(pop) + struct TileXRCcuRaInitConfig { uint32_t phyId; uint32_t nicPosition; @@ -608,6 +635,18 @@ using TileXRCcuRaGetTpInfoListAsyncFunc = int (*)( TileXRCcuHccpTpInfo infoList[], uint32_t* num, void** reqHandle); +using TileXRCcuRaGetTpAttrAsyncFunc = int (*)( + void* ctx, + uint64_t tpHandle, + uint32_t* attrBitmap, + TileXRCcuHccpTpAttr* attr, + void** reqHandle); +using TileXRCcuRaSetTpAttrAsyncFunc = int (*)( + void* ctx, + uint64_t tpHandle, + uint32_t attrBitmap, + TileXRCcuHccpTpAttr* attr, + void** reqHandle); using TileXRCcuRaGetAsyncReqResultFunc = int (*)(void* reqHandle, int* reqResult); using TileXRCcuEndpointRouteProviderFunc = int (*)( diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp index fe0825af..39515270 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace TileXR { namespace { @@ -19,6 +20,7 @@ constexpr uint16_t TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE = 4; constexpr uint16_t TILEXR_CCU_HCOMM_WQE_BASIC_BLOCKS_PER_ROUTE = 256; constexpr uint32_t TILEXR_CCU_HCOMM_PER_DIE_PFE_RESERVED_NUM = 16; constexpr uint16_t TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM = 23; +constexpr uint16_t TILEXR_CCU_HCOMM_PER_DIE_JETTY_NUM = 128; constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_START_JETTY_CTX_ID = 92; constexpr uint16_t TILEXR_CCU_HCOMM_OUTER_FE_JETTY_NUM = 36; constexpr uint32_t TILEXR_CCU_HCOMM_MAX_INNER_FE_ID = 7; @@ -140,7 +142,11 @@ void NormalizeVerifiedEndpointRouteJettyWindow(TileXRCcuLowerLayerTransportSnaps ++explicitStartCount; } - if (minExplicitStart != 0) { + const bool configuredWindowContainsExplicitJettys = + snapshot->startJettyId != 0 && snapshot->pfeJettyCount != 0 && minExplicitStart != 0 && + minExplicitStart >= snapshot->startJettyId && + explicitEnd <= static_cast(snapshot->startJettyId) + snapshot->pfeJettyCount; + if (minExplicitStart != 0 && !configuredWindowContainsExplicitJettys) { snapshot->startJettyId = minExplicitStart; } @@ -193,7 +199,7 @@ void ApplyHcommOrderedPfePartition(TileXRCcuLowerLayerTransportSnapshot* snapsho return; } snapshot->startLocalJettyCtxId = 0; - snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_INNER_FE_JETTY_NUM; + snapshot->pfeJettyCount = TILEXR_CCU_HCOMM_PER_DIE_JETTY_NUM; snapshot->startJettyId = TILEXR_CCU_DEFAULT_START_JETTY_ID; } @@ -326,6 +332,14 @@ void AppendRemoteXnClears( if (plan == nullptr) { return; } + if (snapshot.remoteXnStartId != 0 && snapshot.remoteXnCount != 0) { + plan->xnClears.push_back({ + snapshot.dieId, + snapshot.remoteXnStartId, + snapshot.remoteXnCount, + }); + return; + } std::vector remoteXns; remoteXns.reserve(snapshot.routes.size()); for (const auto& route : snapshot.routes) { @@ -403,7 +417,9 @@ int TileXRCcuBuildLowerLayerInstallPlan( TileXRCcuJettyInstall jettyInstall; jettyInstall.dieId = spec.pfe.dieId; - jettyInstall.startJettyCtxId = spec.pfe.startLocalJettyCtxId; + jettyInstall.startJettyCtxId = spec.jettys.front().startJettyCtxId == 0 ? + spec.pfe.startLocalJettyCtxId : + spec.jettys.front().startJettyCtxId; for (const auto& jettySpec : spec.jettys) { if (jettySpec.startJettyCtxId != 0 && jettySpec.startJettyCtxId != jettyInstall.startJettyCtxId + jettyInstall.ctxs.size()) { @@ -512,13 +528,15 @@ int TileXRCcuBuildLowerLayerTransportTemplate( ApplyLowerLayerPfePartition(result.pfeId, &result); result.xnStartId = allocation.localXn.startId; result.xnCount = allocation.localXn.num; + result.remoteXnStartId = allocation.remoteXn.startId; + result.remoteXnCount = allocation.remoteXn.num; result.ckeStartId = localWaitCke.startId; result.ckeCount = localWaitCke.num; result.routes.reserve(remoteCcuBuffers.size()); const uint16_t wqeBasicBlockStride = SelectLowerLayerWqeBasicBlockStride(); - uint16_t verifiedStartJettyId = 0; - uint32_t verifiedJettyEnd = 0; + std::map wqeStartByJettyId; + uint32_t nextVerifiedWqeStartId = 0; for (uint32_t i = 0; i < remoteCcuBuffers.size(); ++i) { const auto& remoteCcuBuffer = remoteCcuBuffers[i]; if (remoteCcuBuffer.remoteCcuVa == 0) { @@ -538,7 +556,6 @@ int TileXRCcuBuildLowerLayerTransportTemplate( if (i > std::numeric_limits::max() / wqeBasicBlockStride) { return Fail(nullptr, report, "lower-layer CCU WQE basic block start overflows"); } - route.wqeBasicBlockStartId = static_cast(i * wqeBasicBlockStride); route.remoteCcuVa = remoteCcuBuffer.remoteCcuVa; route.memoryTokenId = remoteCcuBuffer.memoryTokenId; route.memoryTokenValue = remoteCcuBuffer.memoryTokenValue; @@ -561,22 +578,42 @@ int TileXRCcuBuildLowerLayerTransportTemplate( route.localSqDepth = remoteCcuBuffer.localSqDepth; route.startJettyId = remoteCcuBuffer.startJettyId; route.endpointRouteVerified = true; - if (route.startJettyId != 0) { - verifiedStartJettyId = verifiedStartJettyId == 0 ? - route.startJettyId : - std::min(verifiedStartJettyId, route.startJettyId); - verifiedJettyEnd = std::max( - verifiedJettyEnd, - static_cast(route.startJettyId) + 1U); + } + uint32_t wqeOrdinal = i; + if (route.startJettyId != 0) { + const uint32_t sqDepth = route.localSqDepth == 0 ? route.sqDepth : route.localSqDepth; + const uint32_t wqeBasicBlockCount = sqDepth * TILEXR_CCU_WQE_BASIC_BLOCKS_PER_ROUTE; + if (wqeBasicBlockCount == 0 || nextVerifiedWqeStartId > std::numeric_limits::max()) { + return Fail(nullptr, report, "invalid verified endpoint WQE basic block window"); + } + uint32_t wqeBasicBlockStartId = nextVerifiedWqeStartId; + const bool pfeWindowContainsJetty = result.pfeJettyCount != 0 && + route.startJettyId >= result.startJettyId && + static_cast(route.startJettyId) < + static_cast(result.startJettyId) + result.pfeJettyCount; + if (pfeWindowContainsJetty) { + const uint32_t localJettyOffset = + static_cast(route.startJettyId) - result.startJettyId; + wqeBasicBlockStartId = localJettyOffset * wqeBasicBlockCount; + } + if (wqeBasicBlockStartId > std::numeric_limits::max()) { + return Fail(nullptr, report, "verified endpoint WQE basic block start overflows"); + } + const auto inserted = wqeStartByJettyId.emplace( + route.startJettyId, + static_cast(wqeBasicBlockStartId)); + route.wqeBasicBlockStartId = inserted.first->second; + if (inserted.second && !pfeWindowContainsJetty) { + nextVerifiedWqeStartId += wqeBasicBlockCount; + } + } else { + if (wqeOrdinal > std::numeric_limits::max() / wqeBasicBlockStride) { + return Fail(nullptr, report, "lower-layer CCU WQE basic block start overflows"); } + route.wqeBasicBlockStartId = static_cast(wqeOrdinal * wqeBasicBlockStride); } result.routes.push_back(route); } - if (verifiedStartJettyId != 0) { - result.startJettyId = verifiedStartJettyId; - const uint32_t requiredJettyCount = verifiedJettyEnd - verifiedStartJettyId; - result.pfeJettyCount = CheckedU16(std::max(result.pfeJettyCount, requiredJettyCount)); - } NormalizeVerifiedEndpointRouteJettyWindow(&result); *snapshot = result; @@ -670,12 +707,28 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( spec.ckeClear.count = normalized.ckeCount; spec.ckeClear.valid = normalized.ckeCount != 0; - uint32_t routeIndex = 0; - for (const auto& route : normalized.routes) { + std::map jettyRoutes; + for (uint32_t i = 0; i < normalized.routes.size(); ++i) { + const auto& route = normalized.routes[i]; + const uint16_t jettyId = route.startJettyId == 0 ? + static_cast(normalized.startJettyId + i) : + route.startJettyId; + jettyRoutes.emplace(jettyId, &route); + } + for (const auto& entry : jettyRoutes) { + if (entry.first < normalized.startJettyId) { + return Fail(plan, report, "lower-layer CCU endpoint jetty ID precedes the PFE jetty window"); + } + const uint32_t localJettyOffset = static_cast(entry.first) - normalized.startJettyId; + if (localJettyOffset >= normalized.pfeJettyCount || + static_cast(normalized.startLocalJettyCtxId) + localJettyOffset >= 128U) { + return Fail(plan, report, "lower-layer CCU endpoint jetty ID is outside the PFE jetty window"); + } + const auto& route = *entry.second; TileXRCcuLowerLayerJettySpec jetty; jetty.dieId = normalized.dieId; jetty.pfeId = normalized.pfeId; - jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + routeIndex); + jetty.startJettyCtxId = static_cast(normalized.startLocalJettyCtxId + localJettyOffset); jetty.doorbellVa = route.localDoorbellVa == 0 ? route.doorbellVa : route.localDoorbellVa; jetty.doorbellTokenId = route.localDoorbellTokenId == 0 ? route.doorbellTokenId : @@ -686,7 +739,10 @@ int TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( jetty.sqDepth = route.localSqDepth == 0 ? route.sqDepth : route.localSqDepth; jetty.wqeBasicBlockStartId = route.wqeBasicBlockStartId; spec.jettys.push_back(jetty); + } + uint32_t routeIndex = 0; + for (const auto& route : normalized.routes) { TileXRCcuLowerLayerChannelSpec channel; channel.dieId = normalized.dieId; channel.channelId = route.channelId; diff --git a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h index c37f908a..5a71bce9 100644 --- a/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h +++ b/src/comm/ccu/tilexr_ccu_lower_layer_plan_builder.h @@ -153,6 +153,8 @@ struct TileXRCcuLowerLayerTransportSnapshot { uint16_t startLocalJettyCtxId = 0; uint32_t xnStartId = 0; uint32_t xnCount = 0; + uint32_t remoteXnStartId = 0; + uint32_t remoteXnCount = 0; uint32_t ckeStartId = 0; uint32_t ckeCount = 0; std::vector routes; diff --git a/src/comm/ccu/tilexr_ccu_microcode.cpp b/src/comm/ccu/tilexr_ccu_microcode.cpp index 44abfdbe..1ba5e4e3 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.cpp +++ b/src/comm/ccu/tilexr_ccu_microcode.cpp @@ -13,8 +13,11 @@ constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_GSA_HEADER = 0x0002U; constexpr uint64_t TILEXR_CCU_LOAD_IMD_TO_XN_HEADER = 0x0003U; constexpr uint64_t TILEXR_CCU_SET_CKE_HEADER = 0x0802U; constexpr uint64_t TILEXR_CCU_CLEAR_CKE_HEADER = 0x0804U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MS_HEADER = 0x1000U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MS_TO_LOC_MEM_HEADER = 0x1002U; constexpr uint64_t TILEXR_CCU_TRANS_RMT_MEM_TO_LOC_MEM_HEADER = 0x1008U; constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_RMT_MEM_HEADER = 0x1009U; +constexpr uint64_t TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER = 0x100aU; constexpr uint64_t TILEXR_CCU_SYNC_CKE_HEADER = 0x100bU; constexpr uint64_t TILEXR_CCU_SYNC_XN_HEADER = 0x100dU; constexpr uint64_t TILEXR_CCU_SYNC_XN_TRACE_FLAG = 0x0001000000000000ULL; @@ -77,6 +80,23 @@ int ValidateTransferSpec(const TileXRCcuMemTransferSpec& spec) return TILEXR_SUCCESS; } +int ValidateLocalMsTransferSpec(const TileXRCcuLocalMsTransferSpec& spec) +{ + if (spec.localGsa == 0 || spec.localXn == 0 || spec.lengthXn == 0) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + if ((spec.setCkeId == 0) != (spec.setCkeMask == 0) || + (spec.waitCkeId == 0) != (spec.waitCkeMask == 0)) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + return TILEXR_SUCCESS; +} + +uint16_t LocalMsTransferFlagSlot(const TileXRCcuLocalMsTransferSpec& spec) +{ + return static_cast((spec.clearWait ? 1U : 0U) | (spec.lengthFromXn ? 2U : 0U)); +} + void WriteLe16(uint8_t* bytes, size_t offset, uint16_t value) { bytes[offset] = static_cast(value & 0xffU); @@ -255,6 +275,59 @@ int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, Til return TILEXR_SUCCESS; } +int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || ValidateTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MEM_HEADER), + spec.remoteGsa, + spec.remoteXn, + spec.localGsa); + instr->words[1] = PackSlots(spec.localXn, spec.lengthXn, spec.channelId, TransferControlSlot(spec)); + instr->words[2] = PackSlots(0, 0, 0, TransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeTransLocMemToLocMs(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || + ValidateLocalMsTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MEM_TO_LOC_MS_HEADER), + spec.localMs, + spec.localGsa, + spec.localXn); + instr->words[1] = PackSlots(spec.lengthXn, spec.channelId, 0, 0); + instr->words[2] = PackSlots(0, 0, 0, LocalMsTransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + +int TileXRCcuEncodeTransLocMsToLocMem(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr) +{ + if (ValidateInstrOutput(instr) != TILEXR_SUCCESS || + ValidateLocalMsTransferSpec(spec) != TILEXR_SUCCESS) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + + instr->words[0] = PackSlots( + static_cast(TILEXR_CCU_TRANS_LOC_MS_TO_LOC_MEM_HEADER), + spec.localGsa, + spec.localXn, + spec.localMs); + instr->words[1] = PackSlots(spec.lengthXn, spec.channelId, 0, 0); + instr->words[2] = PackSlots(0, 0, 0, LocalMsTransferFlagSlot(spec)); + instr->words[3] = PackSlots(spec.setCkeId, spec.setCkeMask, spec.waitCkeId, spec.waitCkeMask); + return TILEXR_SUCCESS; +} + int TileXRCcuBuildSqeLoadProgram(uint16_t firstXnId, uint32_t argCount, std::vector* program) { if (program == nullptr || firstXnId == 0 || argCount == 0 || argCount > TILEXR_CCU_SQE_ARGS_LEN) { diff --git a/src/comm/ccu/tilexr_ccu_microcode.h b/src/comm/ccu/tilexr_ccu_microcode.h index fe8c0328..fbe59783 100644 --- a/src/comm/ccu/tilexr_ccu_microcode.h +++ b/src/comm/ccu/tilexr_ccu_microcode.h @@ -68,6 +68,20 @@ struct TileXRCcuMemTransferSpec { bool reduceEnabled = false; }; +struct TileXRCcuLocalMsTransferSpec { + uint16_t localGsa = 0; + uint16_t localXn = 0; + uint16_t localMs = 0; + uint16_t lengthXn = 0; + uint16_t channelId = 0; + uint16_t setCkeId = 0; + uint16_t setCkeMask = 0; + uint16_t waitCkeId = 0; + uint16_t waitCkeMask = 0; + bool clearWait = true; + bool lengthFromXn = true; +}; + int TileXRCcuEncodeLoadSqeArgsToX(uint16_t xnId, uint32_t sqeArgId, TileXRCcuInstr* instr); int TileXRCcuEncodeLoadImdToXn(uint16_t xnId, uint64_t immediate, uint16_t secFlag, TileXRCcuInstr* instr); @@ -86,6 +100,12 @@ int TileXRCcuEncodeTransRmtMemToLocMem(const TileXRCcuMemTransferSpec& spec, Til int TileXRCcuEncodeTransLocMemToRmtMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); +int TileXRCcuEncodeTransLocMemToLocMem(const TileXRCcuMemTransferSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeTransLocMemToLocMs(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr); + +int TileXRCcuEncodeTransLocMsToLocMem(const TileXRCcuLocalMsTransferSpec& spec, TileXRCcuInstr* instr); + int TileXRCcuBuildSqeLoadProgram( uint16_t firstXnId, uint32_t argCount, diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp index 282d9fe2..5b4840f7 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.cpp +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.cpp @@ -196,7 +196,10 @@ int TileXRCcuResourceAllocator::Allocate( const uint32_t requiredPostWaitInstructionCount = postOnly ? request.syncResourceCount : request.syncResourceCount * 2U; const uint32_t sourceCkeInitCount = syncCkeMode ? 1U : 0U; - const uint32_t sourceCkeResourceCount = syncCkeMode ? 1U : 0U; + const uint32_t sourceCkeResourceCount = syncCkeMode ? request.sourceCkeCount : 0U; + if (syncCkeMode && sourceCkeResourceCount == 0) { + return Fail(report, "invalid CCU source CKE resource request"); + } const uint32_t task1PreludeInstructionCount = hcommStyleTask1Prelude ? TILEXR_CCU_HCOMM_TASK1_PRELUDE_INSTRUCTION_COUNT : 0U; const uint32_t requiredBarrierInstructionCount = diff --git a/src/comm/ccu/tilexr_ccu_resource_allocator.h b/src/comm/ccu/tilexr_ccu_resource_allocator.h index 706ff2b4..959e712d 100644 --- a/src/comm/ccu/tilexr_ccu_resource_allocator.h +++ b/src/comm/ccu/tilexr_ccu_resource_allocator.h @@ -43,6 +43,7 @@ struct TileXRCcuResourceRequest { uint32_t syncResourceCount = 0; uint32_t syncInstructionCount = 0; uint32_t bindingsPerSyncResource = 1; + uint32_t sourceCkeCount = 1; TileXRCcuBarrierMode barrierMode = TileXRCcuBarrierMode::SyncXn; }; diff --git a/src/comm/ccu/tilexr_ccu_topology.cpp b/src/comm/ccu/tilexr_ccu_topology.cpp new file mode 100644 index 00000000..fc20242a --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_topology.cpp @@ -0,0 +1,293 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#include "ccu/tilexr_ccu_topology.h" + +#include "tilexr_types.h" + +#include +#include +#include +#include +#include +#include + +namespace TileXR { +namespace { + +struct RootInfo { + std::string topoPath; + std::unordered_map deviceToLocalId; + std::unordered_map>> portToEidByLocalId; +}; + +struct TopoEdge { + uint32_t localA = 0; + uint32_t localB = 0; + std::vector localAPorts; + std::vector localBPorts; +}; + +std::string ReadTextFile(const std::string& path) +{ + std::ifstream input(path); + if (!input.is_open()) { + return {}; + } + return std::string(std::istreambuf_iterator(input), std::istreambuf_iterator()); +} + +bool ParseUint(const std::string& value, uint32_t* out) +{ + if (value.empty() || out == nullptr) { + return false; + } + char* end = nullptr; + const unsigned long parsed = std::strtoul(value.c_str(), &end, 10); + if (end == value.c_str() || *end != '\0' || parsed > UINT32_MAX) { + return false; + } + *out = static_cast(parsed); + return true; +} + +std::string JsonStringField(const std::string& object, const std::string& field) +{ + const std::regex pattern("\"" + field + "\"\\s*:\\s*\"([^\"]*)\""); + std::smatch match; + return std::regex_search(object, match, pattern) ? match[1].str() : std::string(); +} + +bool JsonUintField(const std::string& object, const std::string& field, uint32_t* out) +{ + const std::regex quoted("\"" + field + "\"\\s*:\\s*\"([0-9]+)\""); + const std::regex plain("\"" + field + "\"\\s*:\\s*([0-9]+)"); + std::smatch match; + if (std::regex_search(object, match, quoted) || std::regex_search(object, match, plain)) { + return ParseUint(match[1].str(), out); + } + return false; +} + +bool ParseEidHex(const std::string& text, std::array* eid) +{ + if (eid == nullptr || text.size() != eid->size() * 2U) { + return false; + } + for (size_t i = 0; i < eid->size(); ++i) { + const char hi = text[i * 2U]; + const char lo = text[i * 2U + 1U]; + if (!std::isxdigit(static_cast(hi)) || + !std::isxdigit(static_cast(lo))) { + return false; + } + (*eid)[i] = static_cast( + std::strtoul(text.substr(i * 2U, 2U).c_str(), nullptr, 16)); + } + return true; +} + +std::vector JsonStringArrayField(const std::string& object, const std::string& field) +{ + const std::regex arrayPattern("\"" + field + "\"\\s*:\\s*\\[([^\\]]*)\\]"); + std::smatch arrayMatch; + if (!std::regex_search(object, arrayMatch, arrayPattern)) { + return {}; + } + const std::string body = arrayMatch[1].str(); + std::vector values; + const std::regex valuePattern("\"([^\"]*)\""); + for (auto it = std::sregex_iterator(body.begin(), body.end(), valuePattern); + it != std::sregex_iterator(); ++it) { + values.push_back((*it)[1].str()); + } + return values; +} + +std::vector ExtractObjectsWithKey(const std::string& text, const std::string& key) +{ + std::vector objects; + const std::string needle = "\"" + key + "\""; + size_t pos = 0; + while ((pos = text.find(needle, pos)) != std::string::npos) { + const size_t begin = text.rfind('{', pos); + if (begin == std::string::npos) { + ++pos; + continue; + } + int depth = 0; + bool inString = false; + bool escaped = false; + for (size_t i = begin; i < text.size(); ++i) { + const char ch = text[i]; + if (inString) { + escaped = !escaped && ch == '\\'; + if (ch == '"' && !escaped) { + inString = false; + } else if (ch != '\\') { + escaped = false; + } + continue; + } + if (ch == '"') { + inString = true; + } else if (ch == '{') { + ++depth; + } else if (ch == '}') { + --depth; + if (depth == 0) { + objects.emplace_back(text.substr(begin, i - begin + 1U)); + pos = i + 1U; + break; + } + } + } + if (depth != 0) { + break; + } + } + return objects; +} + +bool ParseRootInfo(const std::string& path, RootInfo* root) +{ + if (root == nullptr) { + return false; + } + const std::string content = ReadTextFile(path); + root->topoPath = JsonStringField(content, "topo_file_path"); + if (content.empty() || root->topoPath.empty()) { + return false; + } + for (const auto& rankObject : ExtractObjectsWithKey(content, "device_id")) { + uint32_t deviceId = 0; + uint32_t localId = 0; + if (!JsonUintField(rankObject, "device_id", &deviceId) || + !JsonUintField(rankObject, "local_id", &localId)) { + continue; + } + root->deviceToLocalId[deviceId] = localId; + for (const auto& addressObject : ExtractObjectsWithKey(rankObject, "addr")) { + std::array eid {}; + if (!ParseEidHex(JsonStringField(addressObject, "addr"), &eid)) { + continue; + } + for (const auto& port : JsonStringArrayField(addressObject, "ports")) { + root->portToEidByLocalId[localId][port] = eid; + } + } + } + return !root->deviceToLocalId.empty(); +} + +std::vector ParseTopoInfo(const std::string& path) +{ + const std::string content = ReadTextFile(path); + std::vector edges; + for (const auto& edgeObject : ExtractObjectsWithKey(content, "local_a")) { + TopoEdge edge; + if (!JsonUintField(edgeObject, "local_a", &edge.localA) || + !JsonUintField(edgeObject, "local_b", &edge.localB)) { + continue; + } + edge.localAPorts = JsonStringArrayField(edgeObject, "local_a_ports"); + edge.localBPorts = JsonStringArrayField(edgeObject, "local_b_ports"); + if (!edge.localAPorts.empty() && !edge.localBPorts.empty()) { + edges.push_back(edge); + } + } + return edges; +} + +bool ResolveLocalPort( + const std::vector& edges, + uint32_t localId, + uint32_t peerLocalId, + std::string* localPort) +{ + if (localPort == nullptr) { + return false; + } + for (const auto& edge : edges) { + if (edge.localA == localId && edge.localB == peerLocalId) { + *localPort = edge.localAPorts.front(); + return true; + } + if (edge.localB == localId && edge.localA == peerLocalId) { + *localPort = edge.localBPorts.front(); + return true; + } + } + return false; +} + +} // namespace + +int TileXRCcuResolvePeerEidRoutes( + const std::string& rootInfoPath, + uint32_t localDevicePhyId, + const std::vector& peerDevicePhyIds, + std::vector* routes, + std::string* message) +{ + if (routes == nullptr) { + return TILEXR_ERROR_PARA_CHECK_FAIL; + } + routes->clear(); + RootInfo root; + if (!ParseRootInfo(rootInfoPath, &root)) { + if (message != nullptr) { + *message = "failed to parse HCCL root info"; + } + return TILEXR_ERROR_NOT_FOUND; + } + const auto localIdIt = root.deviceToLocalId.find(localDevicePhyId); + if (localIdIt == root.deviceToLocalId.end()) { + if (message != nullptr) { + *message = "local physical device is absent from HCCL root info"; + } + return TILEXR_ERROR_NOT_FOUND; + } + const auto edges = ParseTopoInfo(root.topoPath); + const auto eidMapIt = root.portToEidByLocalId.find(localIdIt->second); + if (edges.empty() || eidMapIt == root.portToEidByLocalId.end()) { + if (message != nullptr) { + *message = "HCCL topology has no local EID routes"; + } + return TILEXR_ERROR_NOT_FOUND; + } + for (const uint32_t peerDevicePhyId : peerDevicePhyIds) { + const auto peerIdIt = root.deviceToLocalId.find(peerDevicePhyId); + std::string localPort; + if (peerIdIt == root.deviceToLocalId.end() || + !ResolveLocalPort(edges, localIdIt->second, peerIdIt->second, &localPort)) { + if (message != nullptr) { + *message = "HCCL topology has no device-pair edge"; + } + routes->clear(); + return TILEXR_ERROR_NOT_FOUND; + } + const auto eidIt = eidMapIt->second.find(localPort); + if (eidIt == eidMapIt->second.end()) { + if (message != nullptr) { + *message = "HCCL root info has no EID for the selected local port"; + } + routes->clear(); + return TILEXR_ERROR_NOT_FOUND; + } + TileXRCcuPeerEidRoute route; + route.peerDevicePhyId = peerDevicePhyId; + route.localEid = eidIt->second; + route.localPort = localPort; + routes->push_back(route); + } + if (message != nullptr) { + *message = "ok"; + } + return TILEXR_SUCCESS; +} + +} // namespace TileXR diff --git a/src/comm/ccu/tilexr_ccu_topology.h b/src/comm/ccu/tilexr_ccu_topology.h new file mode 100644 index 00000000..cbb68d96 --- /dev/null +++ b/src/comm/ccu/tilexr_ccu_topology.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2026 TileXR Project + * Licensed under the Apache License, Version 2.0 + */ + +#ifndef TILEXR_CCU_TOPOLOGY_H +#define TILEXR_CCU_TOPOLOGY_H + +#include "ccu/tilexr_ccu_hccp_types.h" +#include "tilexr_types.h" + +#include +#include +#include +#include + +namespace TileXR { + +struct TileXRCcuPeerEidRoute { + uint32_t peerDevicePhyId = 0; + std::array localEid {}; + std::string localPort; +}; + +int TileXRCcuResolvePeerEidRoutes( + const std::string& rootInfoPath, + uint32_t localDevicePhyId, + const std::vector& peerDevicePhyIds, + std::vector* routes, + std::string* message); + +} // namespace TileXR + +#endif // TILEXR_CCU_TOPOLOGY_H diff --git a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp index fd4e9b8b..288b04c1 100644 --- a/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp +++ b/tests/ccu/ccu_tilexr_direct_smoke_probe.cpp @@ -79,12 +79,14 @@ constexpr const char* kP2pCcuCopyDirectionEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU constexpr const char* kP2pCcuCopyResourceWindowEnv = "TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY_RESOURCE_WINDOW"; constexpr const char* kAllToAllEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL"; constexpr const char* kAllToAllLongMissionEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION"; +constexpr const char* kAllToAllMeshEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH"; constexpr const char* kAllToAllSingleRouteBidirectionalEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_SINGLE_ROUTE_BIDIRECTIONAL"; constexpr const char* kAllToAllBytesEnv = "TILEXR_CCU_ALLTOALL_BYTES"; constexpr const char* kAllToAllMemSlicePerLoopEnv = "TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP"; constexpr const char* kAllToAllLoopCountEnv = "TILEXR_CCU_ALLTOALL_LOOP_COUNT"; constexpr const char* kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"; +constexpr const char* kSyncXnPingPeerXorEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR"; constexpr const char* kSignalWaitEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT"; constexpr const char* kSignalWaitSignalRankEnv = "TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK"; constexpr const char* kSignalWaitBarrierEnv = "TILEXR_CCU_DIRECT_SMOKE_BARRIER"; @@ -141,6 +143,8 @@ struct AllToAllState { std::vector expected; std::vector observed; size_t bytes = 0; + size_t chunkBytes = 0; + int rankSize = 0; int initRet = ACL_SUCCESS; int readRet = ACL_SUCCESS; uint32_t mismatchCount = 0; @@ -179,6 +183,11 @@ bool AllToAllLongMissionEnabled() return EnvFlag(kAllToAllLongMissionEnv); } +bool AllToAllMeshSmokeEnabled() +{ + return EnvFlag(kAllToAllMeshEnv); +} + bool AllToAllSingleRouteBidirectionalEnabled() { return EnvFlag(kAllToAllSingleRouteBidirectionalEnv); @@ -608,6 +617,99 @@ std::vector BuildAllToAllLoopPattern(int rank, int loopIndex, size_t by return pattern; } +uint8_t BuildAllToAllMeshByte( + uint32_t sourceRank, + uint32_t targetRank, + uint32_t loopIndex, + size_t chunkOffset) +{ + return static_cast( + ((sourceRank + 1U) * 67U + (targetRank + 1U) * 29U + + (loopIndex + 1U) * 17U + chunkOffset * 13U) & 0xffU); +} + +int InitAllToAllMeshState(int rank, int rankSize, AllToAllState* state) +{ + if (state == nullptr || rank < 0 || rank >= rankSize || rankSize != 4) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + state->chunkBytes = AllToAllBytesFromEnv(); + state->rankSize = rankSize; + state->bytes = static_cast(rankSize) * state->chunkBytes; + const bool supportedChunkBytes = + state->chunkBytes == 128U * 1024U || state->chunkBytes == 2U * 1024U * 1024U; + if (!supportedChunkBytes || + state->bytes / state->chunkBytes != static_cast(rankSize) || + AllToAllMemSlicePerLoopFromEnv() != 8) { + state->initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + return state->initRet; + } + state->expected.assign(state->bytes, 0); + state->observed.assign(state->bytes, 0); + std::vector initial(state->bytes, 0xa5U); + int ret = state->source.Allocate(state->bytes); + if (ret == ACL_SUCCESS) { + ret = state->destination.Allocate(state->bytes); + } + if (ret == ACL_SUCCESS) { + ret = aclrtMemcpy( + state->source.ptr, state->bytes, initial.data(), initial.size(), ACL_MEMCPY_HOST_TO_DEVICE); + } + if (ret == ACL_SUCCESS) { + ret = aclrtMemcpy( + state->destination.ptr, state->bytes, initial.data(), initial.size(), ACL_MEMCPY_HOST_TO_DEVICE); + } + state->initRet = ret; + return ret; +} + +int ResetAllToAllMeshStateForLoop(int rank, int loopIndex, AllToAllState* state) +{ + if (state == nullptr || state->source.ptr == nullptr || state->destination.ptr == nullptr || + state->rankSize != 4 || state->bytes != static_cast(state->rankSize) * state->chunkBytes || + loopIndex < 0) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + std::vector source(state->bytes); + state->expected.assign(state->bytes, 0); + for (int targetRank = 0; targetRank < state->rankSize; ++targetRank) { + for (size_t chunkOffset = 0; chunkOffset < state->chunkBytes; ++chunkOffset) { + source[static_cast(targetRank) * state->chunkBytes + chunkOffset] = + BuildAllToAllMeshByte(rank, targetRank, loopIndex, chunkOffset); + } + } + for (int sourceRank = 0; sourceRank < state->rankSize; ++sourceRank) { + for (size_t chunkOffset = 0; chunkOffset < state->chunkBytes; ++chunkOffset) { + state->expected[static_cast(sourceRank) * state->chunkBytes + chunkOffset] = + BuildAllToAllMeshByte(sourceRank, rank, loopIndex, chunkOffset); + } + } + state->observed.assign(state->bytes, 0); + state->readRet = ACL_SUCCESS; + state->mismatchCount = 0; + state->firstMismatchOffset = 0; + state->lastMismatchOffset = 0; + state->firstMismatchObserved = 0; + state->firstMismatchExpected = 0; + state->mismatchedBlockCount = 0; + state->firstMismatchedBlock = 0; + state->lastMismatchedBlock = 0; + state->passed = false; + std::vector destination( + state->bytes, static_cast(0xa5U ^ static_cast(loopIndex))); + int ret = aclrtMemcpy( + state->source.ptr, state->bytes, source.data(), source.size(), ACL_MEMCPY_HOST_TO_DEVICE); + if (ret != ACL_SUCCESS) { + return ret; + } + return aclrtMemcpy( + state->destination.ptr, + state->bytes, + destination.data(), + destination.size(), + ACL_MEMCPY_HOST_TO_DEVICE); +} + int InitAllToAllState(int rank, int peer, AllToAllState* state) { if (state == nullptr) { @@ -1365,9 +1467,10 @@ void PrintCcuResourceState( DirectCcuSmokeContext* context, uint8_t dieId, const TileXRDirectCcuPrepareOptions& options, - const char* label) + const char* label, + uint32_t resourceCount = 3U) { - if (context == nullptr || label == nullptr) { + if (context == nullptr || label == nullptr || resourceCount == 0) { return; } TileXR::TileXRCcuDriverAdapter adapter; @@ -1379,38 +1482,49 @@ void PrintCcuResourceState( return; } - uint64_t localXn[3] {}; - uint64_t remoteXn[3] {}; - uint64_t localWaitCke[3] {}; - uint64_t remoteNotifyCke[3] {}; + std::vector localXn(resourceCount, 0); + std::vector remoteXn(resourceCount, 0); + std::vector localWaitCke(resourceCount, 0); + std::vector remoteNotifyCke(resourceCount, 0); const uint32_t localXnStartId = options.xnStartId; const uint32_t remoteXnStartId = options.remoteXnStartId; const uint32_t localWaitCkeStartId = options.localWaitCkeStartId; const uint32_t remoteNotifyCkeStartId = options.remoteNotifyCkeStartId; - const int localXnRet = adapter.ReadXnRange(dieId, localXnStartId, localXn, 3, &report); - const int remoteXnRet = adapter.ReadXnRange(dieId, remoteXnStartId, remoteXn, 3, &report); + const int localXnRet = adapter.ReadXnRange( + dieId, localXnStartId, localXn.data(), resourceCount, &report); + const int remoteXnRet = adapter.ReadXnRange( + dieId, remoteXnStartId, remoteXn.data(), resourceCount, &report); const int localCkeRet = adapter.ReadCkeRange( - dieId, localWaitCkeStartId, localWaitCke, 3, &report); + dieId, localWaitCkeStartId, localWaitCke.data(), resourceCount, &report); const int remoteCkeRet = adapter.ReadCkeRange( - dieId, remoteNotifyCkeStartId, remoteNotifyCke, 3, &report); + dieId, remoteNotifyCkeStartId, remoteNotifyCke.data(), resourceCount, &report); + + const auto values = [](const std::vector& data) { + std::ostringstream out; + for (size_t i = 0; i < data.size(); ++i) { + if (i != 0) { + out << ","; + } + out << "0x" << std::hex << data[i]; + } + return out.str(); + }; std::cerr << label << " resourceState" + << " resourceCount=" << resourceCount << " localXnStartId=" << localXnStartId << " localXnRet=" << localXnRet - << " localXn=0x" << std::hex << localXn[0] << ",0x" << localXn[1] << ",0x" << localXn[2] - << std::dec + << " localXn=" << values(localXn) << " remoteXnStartId=" << remoteXnStartId << " remoteXnRet=" << remoteXnRet - << " remoteXn=0x" << std::hex << remoteXn[0] << ",0x" << remoteXn[1] << ",0x" << remoteXn[2] - << std::dec + << " remoteXn=" << values(remoteXn) << " localWaitCkeStartId=" << localWaitCkeStartId << " localCkeRet=" << localCkeRet - << " localCke=0x" << std::hex << localWaitCke[0] << ",0x" << localWaitCke[1] << ",0x" << localWaitCke[2] - << std::dec + << " localCke=" << values(localWaitCke) << " remoteNotifyCkeStartId=" << remoteNotifyCkeStartId << " remoteCkeRet=" << remoteCkeRet - << " remoteCke=0x" << std::hex << remoteNotifyCke[0] << ",0x" << remoteNotifyCke[1] << ",0x" << remoteNotifyCke[2] - << std::dec << std::endl; + << " remoteCke=" << values(remoteNotifyCke) + << std::endl; } int ReadAndValidatePeerLoopMarker( @@ -1418,7 +1532,11 @@ int ReadAndValidatePeerLoopMarker( uint8_t dieId, uint32_t markerXnId, int rank, + int peerRank, int loopIndex, + uint32_t routeIndex, + uint32_t channelId, + uint32_t ckeId, uint64_t expectedPeerLoopMarker) { if (context == nullptr) { @@ -1435,7 +1553,11 @@ int ReadAndValidatePeerLoopMarker( peerLoopMarker == expectedPeerLoopMarker; std::cout << "tilexr_ccu_alltoall peerLoopMarker" << " rank=" << rank + << " peerRank=" << peerRank << " loopIndex=" << loopIndex + << " route=" << routeIndex + << " channel=" << channelId + << " cke=" << ckeId << " xnId=" << markerXnId << " readRet=" << readRet << " observed=0x" << std::hex << peerLoopMarker @@ -1688,7 +1810,7 @@ bool WaitForCollectiveSubmitDone(int rank, int rankSize, int localResult, int ph << " allRanksDone=1" << " allRanksSucceeded=" << (allSucceeded ? 1 : 0) << std::endl; - return true; + return allSucceeded; } const auto elapsedMs = std::chrono::duration_cast( std::chrono::steady_clock::now() - start).count(); @@ -1897,12 +2019,18 @@ void PrintAllToAllResult(int rank, int loopIndex, int finalRet, const AllToAllSt << " mismatches=" << alltoall.mismatchCount << std::endl; } else { + const size_t chunkBytes = alltoall.chunkBytes == 0 ? alltoall.bytes : alltoall.chunkBytes; + const size_t sourceRank = chunkBytes == 0 ? 0 : alltoall.firstMismatchOffset / chunkBytes; + const size_t chunkOffset = chunkBytes == 0 ? 0 : alltoall.firstMismatchOffset % chunkBytes; std::cout << "tilexr_ccu_alltoall result passed=0" << " rank=" << rank << " loopIndex=" << loopIndex << " ret=" << finalRet << " readRet=" << alltoall.readRet << " mismatches=" << alltoall.mismatchCount + << " sourceRank=" << sourceRank + << " chunkOffset=" << chunkOffset + << " globalOffset=" << alltoall.firstMismatchOffset << " firstMismatchOffset=" << alltoall.firstMismatchOffset << " lastMismatchOffset=" << alltoall.lastMismatchOffset << " firstMismatchObserved=0x" << std::hex << alltoall.firstMismatchObserved @@ -1928,11 +2056,207 @@ void MaybeFastExitAfterAllToAllRun(int finalRet) } } +bool MeshPreparedIdentityMatches( + const TileXR::TileXRCcuDirectInstallAttempt& attempt, + const TileXR::TileXRCcuTask& stableTask, + const std::vector& stableResources) +{ + if (attempt.submitTasks.size() != 1U || attempt.plan.syncResources.size() != stableResources.size()) { + return false; + } + const auto& task = attempt.submitTasks.front(); + if (task.dieId != stableTask.dieId || task.missionId != stableTask.missionId || + task.key != stableTask.key || task.instStartId != stableTask.instStartId || + task.instCnt != stableTask.instCnt || task.argSize != stableTask.argSize) { + return false; + } + for (size_t i = 0; i < stableResources.size(); ++i) { + const auto& current = attempt.plan.syncResources[i]; + const auto& stable = stableResources[i]; + if (current.localXn != stable.localXn || current.remoteXn != stable.remoteXn || + current.notifyCke != stable.notifyCke || current.localWaitCke != stable.localWaitCke || + current.sourceCke != stable.sourceCke || current.channelId != stable.channelId) { + return false; + } + } + return true; +} + +int RunAllToAllMeshLongMissionSmokeForRank( + DirectCcuSmokeContext* context, + int rank, + int rankSize, + int device) +{ + if (context == nullptr) { + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + if (rankSize != 4) { + std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize + << " reason=\"direct CCU alltoall mesh requires four ranks\"" << std::endl; + return 0; + } + + const int loopCount = AllToAllLoopCountFromEnv(); + AllToAllState alltoall; + alltoall.initRet = InitAllToAllMeshState(rank, rankSize, &alltoall); + if (loopCount == 0 && alltoall.initRet == ACL_SUCCESS) { + alltoall.initRet = TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); + options.syncResourceCount = 3U; + options.sqeArgCount = TILEXR_DIRECT_CCU_SQE_ARGS_LEN; + if (std::getenv("TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT") == nullptr) { + options.syncInstructionCount = 131U; + } + if (options.gsaStartId == 0) { + options.gsaStartId = 1; + } + std::cout << "tilexr_ccu_alltoall config" + << " rank=" << rank + << " rankSize=" << rankSize + << " chunkBytes=" << alltoall.chunkBytes + << " bytes=" << alltoall.bytes + << " loopCount=" << loopCount + << " resourceCount=3" + << " mesh=1" + << " longMission=1" + << std::endl; + PrintConfig(options, rankSize); + + TileXR::TileXRCcuDirectInstallAttempt attempt; + TileXRDirectCcuPreparedTasksPtr prepared = &attempt; + TileXRDirectCcuPrepareReport installReport; + const int prepareRet = alltoall.initRet != ACL_SUCCESS ? + alltoall.initRet : + context->planner.PrepareDirectCcuAllToAllMeshInstallAttempt( + context->session, + options, + reinterpret_cast(alltoall.source.ptr), + reinterpret_cast(alltoall.destination.ptr), + alltoall.chunkBytes, + prepared, + &installReport); + PrintInstallReport("tilexr_ccu_alltoall prepare", prepareRet, installReport); + PrintPreparedTasks(prepared, installReport.submitTaskCount); + PrintInstructionReadback(context, prepared, installReport.submitTaskCount); + + int finalRet = 0; + const bool submitRequested = EnvFlag(kSubmitEnv); + if (prepareRet != TileXR::TILEXR_SUCCESS) { + finalRet = 6; + } else if (attempt.submitTasks.size() != 1U || attempt.submitTasks.front().argSize != + TILEXR_DIRECT_CCU_SQE_ARGS_LEN || attempt.plan.syncResources.size() != 3U) { + std::cerr << "tilexr_ccu_alltoall invalidMeshPreparedTask" + << " rank=" << rank + << " taskCount=" << attempt.submitTasks.size() + << " resourceCount=" << attempt.plan.syncResources.size() + << std::endl; + finalRet = 6; + } else if (submitRequested && !installReport.submitReady) { + std::cout << "tilexr_ccu_alltoall submit skipped reason=\"prepare did not reach submitReady\"" + << std::endl; + } else if (submitRequested) { + const TileXR::TileXRCcuTask stableTask = attempt.submitTasks.front(); + const auto stableResources = attempt.plan.syncResources; + aclrtStream stream = nullptr; + const int streamRet = aclrtCreateStream(&stream); + if (streamRet != ACL_SUCCESS) { + std::cerr << "tilexr_ccu_alltoall aclrtCreateStream ret=" << streamRet << std::endl; + finalRet = 7; + } else { + bool skipStreamDestroy = false; + const int syncTimeoutMs = std::max(1, EnvInt("TILEXR_CCU_DIRECT_SUBMIT_TIMEOUT", 6000)); + for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex) { + const int resetRet = ResetAllToAllMeshStateForLoop(rank, loopIndex, &alltoall); + const bool ready = WaitForCollectiveSubmitReadiness( + rank, + rankSize, + resetRet == ACL_SUCCESS && installReport.submitReady, + loopIndex); + if (resetRet != ACL_SUCCESS) { + finalRet = 14; + } else if (!ready) { + finalRet = 13; + } + + if (finalRet == 0) { + TileXRDirectCcuSubmitReport submitReport; + const int submitRet = TileXRDirectCcuSubmitPrepared(prepared, stream, &submitReport); + PrintSubmitReport("tilexr_ccu_alltoall submit", submitRet, submitReport); + const int syncRet = aclrtSynchronizeStreamWithTimeout(stream, syncTimeoutMs); + std::cout << "tilexr_ccu_alltoall timing" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " mesh=1" + << " submitRet=" << submitRet + << " syncRet=" << syncRet + << " syncTimeoutMs=" << syncTimeoutMs + << std::endl; + if (submitRet != TileXR::TILEXR_SUCCESS) { + finalRet = 9; + } else if (syncRet != ACL_SUCCESS) { + finalRet = 8; + skipStreamDestroy = true; + } + } + + if (finalRet == 0 && CheckAllToAllState(&alltoall) != ACL_SUCCESS) { + finalRet = 14; + } + if (finalRet == 0 && !MeshPreparedIdentityMatches(attempt, stableTask, stableResources)) { + finalRet = 16; + } + if (!WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex) && finalRet == 0) { + finalRet = 13; + } + PrintAllToAllResult(rank, loopIndex, finalRet, alltoall); + if (finalRet != 0) { + std::cerr << "tilexr_ccu_alltoall loopFailure" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " ret=" << finalRet + << " resourceCount=3" + << " selfCopyCompletionCke=" << attempt.plan.syncResources[0].localWaitCke + << std::endl; + PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_alltoall"); + PrintCcuResourceState( + context, + attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_alltoall", + 3U); + break; + } + std::cout << "tilexr_ccu_alltoall stableResources=1" + << " rank=" << rank + << " loopIndex=" << loopIndex + << " missionId=" << static_cast(stableTask.missionId) + << " instStartId=" << stableTask.instStartId + << " instCnt=" << stableTask.instCnt + << std::endl; + } + if (skipStreamDestroy) { + std::cout << "tilexr_ccu_alltoall skipDestroyStream=1 rank=" << rank << std::endl; + } else { + aclrtDestroyStream(stream); + } + } + } + + MaybeFastExitAfterAllToAllRun(finalRet); + const int destroyRet = TileXRDirectCcuDestroyPrepared(prepared); + return destroyRet != TileXR::TILEXR_SUCCESS && finalRet == 0 ? 11 : finalRet; +} + int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSize, int device) { if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } + if (AllToAllMeshSmokeEnabled()) { + return RunAllToAllMeshLongMissionSmokeForRank(context, rank, rankSize, device); + } if (rankSize != 2) { std::cout << "tilexr_ccu_alltoall skipped rankSize=" << rankSize << " reason=\"direct CCU alltoall MVP requires two ranks\"" << std::endl; @@ -2067,9 +2391,13 @@ int RunAllToAllLongMissionSmokeForRank(DirectCcuSmokeContext* context, int rank, const int markerRet = ReadAndValidatePeerLoopMarker( context, attempt.submitTasks.front().dieId, - options.remoteXnStartId, + attempt.plan.syncResources[0].remoteXn, rank, + peer, loopIndex, + 0U, + attempt.plan.syncResources[0].channelId, + attempt.plan.syncResources[0].localWaitCke, BuildAllToAllLoopMarker(peer, loopIndex)); if (markerRet != TileXR::TILEXR_SUCCESS) { finalRet = 15; @@ -2124,7 +2452,7 @@ int RunAllToAllSmokeForRank(DirectCcuSmokeContext* context, int rank, int rankSi if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (AllToAllLongMissionEnabled()) { + if (AllToAllMeshSmokeEnabled() || AllToAllLongMissionEnabled()) { return RunAllToAllLongMissionSmokeForRank(context, rank, rankSize, device); } if (rankSize != 2) { @@ -2177,23 +2505,30 @@ int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank if (context == nullptr) { return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; } - if (rankSize != 2) { + if (rankSize != 2 && rankSize != 4) { std::cout << "tilexr_ccu_sync_xn_ping skipped rankSize=" << rankSize - << " reason=\"direct CCU SyncXn ping requires two ranks\"" << std::endl; + << " reason=\"direct CCU SyncXn ping requires two or four ranks\"" << std::endl; return 0; } - const int peer = 1 - rank; + const int peerXor = EnvInt(kSyncXnPingPeerXorEnv, 1); + if (peerXor < 1 || peerXor >= rankSize) { + std::cerr << "tilexr_ccu_sync_xn_ping invalid peerXor=" << peerXor + << " rankSize=" << rankSize << std::endl; + return TileXR::TILEXR_ERROR_PARA_CHECK_FAIL; + } + const int peer = rank ^ peerXor; AllToAllState routeState; routeState.initRet = InitAllToAllState(rank, peer, &routeState); TileXRDirectCcuPrepareOptions options = MakePrepareOptions(rank, rankSize, device); options.syncResourceCount = 1; options.sqeArgCount = 0; - options.syncInstructionCount = 3; + options.syncInstructionCount = 2; std::cout << "tilexr_ccu_sync_xn_ping config" << " rank=" << rank << " peer=" << peer + << " peerXor=" << peerXor << std::endl; PrintConfig(options, rankSize); @@ -2265,6 +2600,12 @@ int RunSyncXnPingSmokeForRank(DirectCcuSmokeContext* context, int rank, int rank if (!attempt.submitTasks.empty()) { PrintMissionContext(context, attempt.submitTasks.front(), "tilexr_ccu_sync_xn_ping"); } + PrintCcuResourceState( + context, + attempt.submitTasks.empty() ? 0 : attempt.submitTasks.front().dieId, + options, + "tilexr_ccu_sync_xn_ping", + 1U); finalRet = 8; } else if (submitRet != TileXR::TILEXR_SUCCESS) { finalRet = 9; diff --git a/tests/ccu/run_tilexr_ccu_direct_smoke.sh b/tests/ccu/run_tilexr_ccu_direct_smoke.sh index ef704493..806594ec 100644 --- a/tests/ccu/run_tilexr_ccu_direct_smoke.sh +++ b/tests/ccu/run_tilexr_ccu_direct_smoke.sh @@ -2,7 +2,7 @@ # # Copyright (c) 2026 TileXR Project # -# Two-rank runner for the private TileXR direct CCU smoke probe. +# Multi-rank runner for the private TileXR direct CCU smoke probe. # Default execution is safe and does not touch ACL/NPU runtime. set -euo pipefail @@ -37,6 +37,7 @@ endpoint_fields=( ) resource_window_token_fields=( + EID EID_INDEX TOKEN_ID RAW_TOKEN_ID @@ -92,6 +93,11 @@ alltoall_mode_enabled() [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0}" = "1" ] } +alltoall_mesh_mode_enabled() +{ + [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH:-0}" = "1" ] +} + alltoall_long_mission_enabled() { [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ] @@ -183,11 +189,17 @@ apply_sync_xn_ping_defaults() export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" - export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-2}" export TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW="${TILEXR_CCU_DIRECT_REPOSITORY_INSTALL_WINDOW:-full_repository}" export TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_DATA_LEN_MODE:-instruction_bytes}" export TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE="${TILEXR_CCU_DIRECT_REPOSITORY_MEMORY_ALLOC_MODE:-acl}" export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_REGISTRATION_MODE:-ra_ctx}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE:-hcomm_die}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION:-hcomm}" + export TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}" + export TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}" export TILEXR_CCU_PROBE_RANK0_XN_START="${TILEXR_CCU_PROBE_RANK0_XN_START:-1961}" export TILEXR_CCU_PROBE_RANK1_XN_START="${TILEXR_CCU_PROBE_RANK1_XN_START:-1961}" export TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START="${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-2361}" @@ -211,13 +223,31 @@ apply_alltoall_defaults() fi export TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}" - export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + if alltoall_mesh_mode_enabled; then + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-131072}" + else + export TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}" + fi export TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}" export TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" export TILEXR_CCU_PROBE_MISSION_START="${TILEXR_CCU_PROBE_MISSION_START:-6}" export TILEXR_CCU_PROBE_INSTRUCTION_START="${TILEXR_CCU_PROBE_INSTRUCTION_START:-475}" export TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START="${TILEXR_CCU_PROBE_MISSION_INSTRUCTION_START:-489}" - if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then + if alltoall_mesh_mode_enabled; then + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_OFFSET_SOURCE:-hcomm_die}" + export TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION="${TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION:-hcomm}" + export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-0}" + export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" + export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-131}" + export TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}" + export TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}" + export TILEXR_CCU_PROBE_REMOTE_XN_COUNT="${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-16}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}" + export TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT:-16}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}" + export TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT:-16}" + export TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX="${TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX:-3}" + elif [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0}" = "1" ]; then export TILEXR_CCU_PROBE_SQE_ARG_COUNT="${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" export TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-3}" export TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-453}" @@ -253,6 +283,30 @@ apply_signal_wait_defaults apply_sync_xn_ping_defaults apply_alltoall_defaults +rank_size="$(parse_int "${TILEXR_CCU_RANK_SIZE:-${TILEXR_CCU_PROBE_RANK_SIZE:-2}}" 2)" +if [ "${rank_size}" -lt 1 ]; then + echo "ERROR: rank size must be positive: ${rank_size}" >&2 + exit 2 +fi +devices="${TILEXR_CCU_SMOKE_DEVICES:-${TILEXR_TEST_DEVICES:-0,1}}" +IFS=',' read -r -a device_list <<< "${devices}" +if [ "${#device_list[@]}" -ne "${rank_size}" ]; then + echo "ERROR: device count ${#device_list[@]} does not match rank size ${rank_size}: ${devices}" >&2 + exit 2 +fi +declare -A seen_devices=() +for device in "${device_list[@]}"; do + if [ -z "${device}" ]; then + echo "ERROR: empty device in list: ${devices}" >&2 + exit 2 + fi + if [ "${seen_devices[${device}]+set}" = "set" ]; then + echo "ERROR: duplicate device ${device} in list: ${devices}" >&2 + exit 2 + fi + seen_devices["${device}"]=1 +done + if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "tilexr_ccu_direct_smoke_runner dryRun=1 workDir=${work_dir}" for diagnostic_var in \ @@ -268,6 +322,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then TILEXR_CCU_DIRECT_SMOKE_BARRIER \ TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING \ TILEXR_CCU_DIRECT_SMOKE_ALLTOALL \ + TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH \ TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION \ TILEXR_CCU_ALLTOALL_BYTES \ TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP \ @@ -279,6 +334,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then echo "dryRun ${diagnostic_var}=${diagnostic_value}" fi done + echo "dryRun TILEXR_CCU_PROBE_RANK_SIZE=${rank_size} devices=${devices}" sqe_arg_count="$(parse_int "${TILEXR_CCU_PROBE_SQE_ARG_COUNT:-13}" 13)" sync_resource_count="$(parse_int "${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}" 1)" default_sync_instruction_count_value="$(default_sync_instruction_count "${sync_resource_count}")" @@ -324,30 +380,24 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_DRY_RUN:-0}" = "1" ]; then for endpoint_field in "${endpoint_fields[@]}"; do endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" common_endpoint_value="${!endpoint_var:-}" - rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" - rank0_endpoint_value="${!rank0_endpoint_var:-${common_endpoint_value}}" - rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" - rank1_endpoint_value="${!rank1_endpoint_var:-${common_endpoint_value}}" - if [ "${rank0_endpoint_value}" != "" ]; then - echo "dryRun rank0 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}" - fi - if [ "${rank1_endpoint_value}" != "" ]; then - echo "dryRun rank1 TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}" - fi + for ((rank=0; rank/dev/null 2>&1; then npu_smi_rc=0 @@ -435,11 +485,13 @@ comm_domain="${TILEXR_CCU_PROBE_COMM_DOMAIN:-0}" timeout_s="${TILEXR_CCU_SMOKE_TIMEOUT:-180}" ready_dir="${work_dir}/submit_ready_${comm_port}" done_dir="${work_dir}/submit_done_${comm_port}" -rank0_log="${work_dir}/ccu_rank0.log" -rank1_log="${work_dir}/ccu_rank1.log" rm -rf "${ready_dir}" "${done_dir}" mkdir -p "${ready_dir}" "${done_dir}" -rm -f "${rank0_log}" "${rank1_log}" +rank_logs=() +for ((rank=0; rank&2 + exit 2 + fi common_env+=("TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE=1") fi if [ "${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-0}" = "1" ]; then @@ -539,9 +595,15 @@ fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING}") fi +if [ "${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR:-}" != "" ]; then + common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR}") +fi if [ "${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-}" != "" ]; then common_env+=("TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION}") fi @@ -612,90 +674,45 @@ for token_field in "${resource_window_token_fields[@]}"; do if [ "${token_value}" != "" ]; then common_env+=("${token_var}=${token_value}") fi - rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" - rank0_token_value="${!rank0_token_var:-}" - if [ "${rank0_token_value}" != "" ]; then - common_env+=("${rank0_token_var}=${rank0_token_value}") - fi - rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" - rank1_token_value="${!rank1_token_var:-}" - if [ "${rank1_token_value}" != "" ]; then - common_env+=("${rank1_token_var}=${rank1_token_value}") - fi done -rank0_env=() -rank1_env=() -if [ "${TILEXR_CCU_PROBE_RANK0_XN_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK0_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_XN_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_XN_START=${TILEXR_CCU_PROBE_RANK1_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_XN_COUNT}") -elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_XN_COUNT}") -elif [ "${TILEXR_CCU_PROBE_REMOTE_XN_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_XN_COUNT=${TILEXR_CCU_PROBE_REMOTE_XN_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_LOCAL_WAIT_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_LOCAL_WAIT_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_START}") -fi -if [ "${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then - rank0_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK0_REMOTE_NOTIFY_CKE_COUNT}") -fi -if [ "${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT:-}" != "" ]; then - rank1_env+=("TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_COUNT=${TILEXR_CCU_PROBE_RANK1_REMOTE_NOTIFY_CKE_COUNT}") -fi -for endpoint_field in "${endpoint_fields[@]}"; do - rank0_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK0" - rank0_endpoint_value="${!rank0_endpoint_var:-}" - if [ "${rank0_endpoint_value}" != "" ]; then - rank0_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank0_endpoint_value}") - fi - rank1_endpoint_var="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}_RANK1" - rank1_endpoint_value="${!rank1_endpoint_var:-}" - if [ "${rank1_endpoint_value}" != "" ]; then - rank1_env+=("TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}=${rank1_endpoint_value}") - fi -done -for token_field in "${resource_window_token_fields[@]}"; do - rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0" - rank0_token_value="${!rank0_token_var:-}" - if [ "${rank0_token_value}" != "" ]; then - rank0_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}") - fi - rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1" - rank1_token_value="${!rank1_token_var:-}" - if [ "${rank1_token_value}" != "" ]; then - rank1_env+=("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}") - fi -done +build_rank_env() +{ + local rank="$1" + rank_env=() + local mapping generic rank_var rank_value + for mapping in \ + XN_START \ + REMOTE_XN_START \ + REMOTE_XN_COUNT \ + LOCAL_WAIT_CKE_START \ + LOCAL_WAIT_CKE_COUNT \ + REMOTE_NOTIFY_CKE_START \ + REMOTE_NOTIFY_CKE_COUNT; do + generic="TILEXR_CCU_PROBE_${mapping}" + rank_var="TILEXR_CCU_PROBE_RANK${rank}_${mapping}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done + for endpoint_field in "${endpoint_fields[@]}"; do + generic="TILEXR_CCU_DIRECT_LOCAL_ENDPOINT_${endpoint_field}" + rank_var="${generic}_RANK${rank}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done + for token_field in "${resource_window_token_fields[@]}"; do + generic="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}" + rank_var="${generic}_RANK${rank}" + rank_value="${!rank_var:-${!generic:-}}" + if [ -n "${rank_value}" ]; then + rank_env+=("${generic}=${rank_value}") + fi + done +} echo "tilexr_ccu_direct_smoke_runner begin workDir=${work_dir} devices=${devices} commId=${comm_id} threadMode=${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0} submit=${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0} barrierMode=${TILEXR_CCU_DIRECT_BARRIER_MODE:-} p2pCcuCopy=${TILEXR_CCU_DIRECT_SMOKE_P2P_CCU_COPY:-0} syncXnPing=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0} alltoall=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL:-0} alltoallLongMission=${TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_LONG_MISSION:-0} signalWait=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_WAIT:-0} signalRank=${TILEXR_CCU_DIRECT_SMOKE_SIGNAL_RANK:-0} ccuBarrier=${TILEXR_CCU_DIRECT_SMOKE_BARRIER:-0} timeout=${timeout_s} npuSmiTimeout=${TILEXR_CCU_SMOKE_NPU_SMI_TIMEOUT:-20}" @@ -808,31 +825,43 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_THREAD_MODE:-0}" = "1" ]; then exit 0 fi -timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0 "${probe_bin}" > "${rank0_log}" 2>&1 & -rank0_pid=$! -sleep "${TILEXR_CCU_SMOKE_RANK1_DELAY:-1}" -timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1 "${probe_bin}" > "${rank1_log}" 2>&1 & -rank1_pid=$! - -rank0_status=0 -rank1_status=0 -wait "${rank0_pid}" || rank0_status=$? -wait "${rank1_pid}" || rank1_status=$? +rank_pids=() +rank_statuses=() +for ((rank=0; rank "${rank_logs[${rank}]}" 2>&1 & + rank_pids+=("$!") + if [ "${rank}" -eq 0 ] && [ "${rank_size}" -gt 1 ]; then + sleep "${TILEXR_CCU_SMOKE_RANK1_DELAY:-1}" + fi +done -cat "${rank0_log}" -cat "${rank1_log}" +any_rank_failed=0 +for ((rank=0; rank&2 - echo "rank0 log: ${rank0_log}" >&2 - echo "rank1 log: ${rank1_log}" >&2 +if [ "${any_rank_failed}" -ne 0 ]; then + echo "ERROR: direct CCU smoke rank process failed statuses=${rank_statuses[*]}" >&2 exit 4 fi if alltoall_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_alltoall prepare ret=0" "${log}"; then echo "ERROR: direct CCU alltoall prepare did not return success in ${log}" >&2 exit 5 @@ -843,7 +872,7 @@ if alltoall_mode_enabled; then fi done elif signal_wait_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_signal_wait prepare ret=0" "${log}"; then echo "ERROR: direct CCU signal/wait prepare did not return success in ${log}" >&2 exit 5 @@ -854,7 +883,7 @@ elif signal_wait_mode_enabled; then fi done else - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_direct_smoke prepare ret=0" "${log}"; then echo "ERROR: direct CCU prepare did not return success in ${log}" >&2 exit 5 @@ -874,13 +903,13 @@ rank_skipped_p2p_ccu_copy_submit() } if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "submitReady=1" "${log}"; then echo "ERROR: direct CCU submit requested but prepare did not reach submitReady=1 in ${log}" >&2 exit 6 fi done - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if alltoall_mode_enabled; then if ! grep -q "tilexr_ccu_alltoall submit ret=0" "${log}"; then echo "ERROR: direct CCU alltoall submit did not return success in ${log}" >&2 @@ -921,14 +950,16 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]; then fi if alltoall_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do - if ! grep -q "tilexr_ccu_alltoall result passed=1" "${log}"; then - echo "ERROR: direct CCU alltoall result did not pass in ${log}" >&2 - exit 9 - fi - done + loop_count="$(parse_int "${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}" 1)" + expected_results=$((rank_size * loop_count)) + actual_results="$(grep -h -c "tilexr_ccu_alltoall result passed=1" "${rank_logs[@]}" | awk '{ total += $1 } END { print total + 0 }')" + echo "tilexr_ccu_direct_smoke_runner alltoallCounts expectedResults=${expected_results} actualResults=${actual_results}" + if [ "${actual_results}" -ne "${expected_results}" ]; then + echo "ERROR: direct CCU alltoall result count mismatch expected=${expected_results} actual=${actual_results}" >&2 + exit 9 + fi elif signal_wait_mode_enabled; then - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_signal_wait result passed=1" "${log}"; then echo "ERROR: direct CCU signal/wait result did not pass in ${log}" >&2 exit 9 @@ -944,9 +975,9 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]; then delay_rank="${TILEXR_CCU_DIRECT_SMOKE_DELAY_RANK:-0}" min_sync_ms="${TILEXR_CCU_DIRECT_SMOKE_MIN_SYNC_MS:-100}" if [ "${delay_rank}" = "0" ]; then - wait_log="${rank1_log}" + wait_log="${rank_logs[1]}" else - wait_log="${rank0_log}" + wait_log="${rank_logs[0]}" fi wait_sync_ms="$( awk ' @@ -976,7 +1007,7 @@ if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_P2P_CCU_COPY:-0}" = "1" ]; then exit 11 fi p2p_passed_count=0 - for log in "${rank0_log}" "${rank1_log}"; do + for log in "${rank_logs[@]}"; do if ! grep -q "tilexr_ccu_direct_smoke p2pCcuCopy" "${log}"; then echo "ERROR: direct CCU P2P CCU-copy result missing in ${log}" >&2 exit 12 diff --git a/tests/ccu/test_tilexr_ccu_alltoall_program.py b/tests/ccu/test_tilexr_ccu_alltoall_program.py index 45a77265..7aeac4fe 100644 --- a/tests/ccu/test_tilexr_ccu_alltoall_program.py +++ b/tests/ccu/test_tilexr_ccu_alltoall_program.py @@ -120,6 +120,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): spec.preSyncMarkerArgIndex = 0; spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; + spec.preSyncMarkerChannelId = 0x14; spec.preSyncChannelId = 0x13; spec.preSyncTokenChannelId = 0x13; spec.copyCompletionCke = 0x301; @@ -178,7 +179,7 @@ def test_two_mb_program_has_presync_64_copy_blocks_postsync_and_finish(self): Slot(program[1], 0) != kSyncXnHeader || Slot(program[1], 1) != spec.preSyncRemoteMarkerXn || Slot(program[1], 2) != spec.preSyncLocalMarkerXn || - Slot(program[1], 4) != spec.preSyncChannelId || + Slot(program[1], 4) != spec.preSyncMarkerChannelId || Slot(program[1], 5) != spec.preSyncRemoteNotifyCke || Slot(program[1], 6) != markerMask || Slot(program[2], 0) != kLoadImdToXnHeader || @@ -310,6 +311,7 @@ def test_two_rank_program_uses_same_hccl_style_copy_region_for_both_ranks(self): spec.preSyncMarkerArgIndex = 0; spec.preSyncMarkerEnabled = true; spec.channelId = 0x12; + spec.preSyncMarkerChannelId = spec.channelId; spec.copyCompletionCke = 0x301; spec.preSyncRemoteAddrXn = 0x211; spec.preSyncRemoteTokenXn = 0x212; @@ -526,6 +528,221 @@ def test_local_rank_does_not_split_the_long_mission_into_copy_phases(self): self.assertIn("alltoallSpec.localRank = alltoall.localRank", orchestrator) self.assertIn("alltoall.localRank = static_cast(rank)", planner) + def test_four_rank_mesh_posts_all_peers_then_copies_remote_and_self_chunks(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_alltoall_program.h" + + #include + #include + #include + + using namespace TileXR; + + uint16_t Slot(const TileXRCcuInstr& instr, uint32_t slot) + { + return static_cast((instr.words[slot / 4U] >> ((slot % 4U) * 16U)) & 0xffffU); + } + + uint64_t Immediate(const TileXRCcuInstr& instr) + { + return (instr.words[0] >> 32U) | (instr.words[1] << 32U); + } + + TileXRCcuAllToAllMeshPeerSpec Peer(uint32_t localRank, uint32_t peerRank, uint16_t ordinal) + { + TileXRCcuAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + auto& route = peer.route; + route.localRank = localRank; + route.localSendAddr = 0x10000000ULL; + route.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + route.localRecvAddr = 0x20000000ULL; + route.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + route.remoteRecvAddr = 0x30000000ULL + static_cast(peerRank) * 0x1000000ULL; + route.remoteRecvToken = TileXRCcuPackMemoryToken(10 + peerRank, 20 + peerRank, true); + route.bytes = 2ULL * 1024ULL * 1024ULL; + route.localGsa = 0x100; + route.remoteGsa = 0x101; + route.localXn = 0x200; + route.remoteXn = 0x301; + route.lengthXn = 0x202; + route.preSyncLocalAddrXn = 0x200; + route.preSyncLocalTokenXn = 0x201; + route.preSyncRemoteAddrXn = 0x300; + route.preSyncRemoteTokenXn = 0x301; + route.preSyncMarkerEnabled = false; + route.preSyncChannelId = static_cast(0x10 + ordinal); + route.preSyncTokenChannelId = route.preSyncChannelId; + route.copyChannelId = route.preSyncChannelId; + route.postSyncChannelId = route.preSyncChannelId; + route.copyCompletionCke = 0x491; + route.preSyncLocalWaitCke = static_cast(0x401 + ordinal); + route.preSyncTokenLocalWaitCke = route.preSyncLocalWaitCke; + route.preSyncRemoteNotifyCke = static_cast(0x500 + ordinal); + route.preSyncRemoteTokenNotifyCke = route.preSyncRemoteNotifyCke; + route.postSyncLocalWaitCke = route.preSyncLocalWaitCke; + route.postSyncRemoteNotifyCke = route.preSyncRemoteNotifyCke; + route.sourceCke = 0x490; + route.ckeMask = 0x8; + return peer; + } + + int main() + { + TileXRCcuAllToAllMeshProgramSpec spec; + spec.rankSize = 4; + spec.localRank = 2; + spec.localSendAddr = 0x10000000ULL; + spec.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + spec.localRecvAddr = 0x20000000ULL; + spec.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + spec.chunkBytes = 2ULL * 1024ULL * 1024ULL; + spec.selfSourceGsa = 0x180; + spec.selfDestinationGsa = 0x181; + spec.selfSourceXn = 0x280; + spec.selfDestinationXn = 0x281; + spec.selfLengthXn = 0x282; + spec.selfChannelId = 0; + spec.selfCompletionCke = 0x480; + spec.remoteCompletionCke = 0x491; + spec.peers = {Peer(2, 3, 2), Peer(2, 0, 0), Peer(2, 1, 1)}; + + std::vector program; + TileXRCcuAllToAllProgramReport report; + const int ret = TileXRCcuBuildAllToAllMeshProgram(spec, &program, &report); + if (ret != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 1; + } + if (report.peerCount != 3 || report.syncResourceCount != 3 || + report.remoteBlockCount != 192 || report.selfBlockCount != 64 || + report.preSyncInstructionCount != 12 || report.copyInstructionCount != 1792 || + report.postSyncInstructionCount != 6 || report.finishInstructionCount != 1 || + report.totalInstructionCount != 1811 || program.size() != 1811) { + std::cerr << "unexpected mesh counts total=" << program.size() << "\n"; + return 2; + } + // Match HCCL: load both values, initialize source CKE, then post output/token per channel. + if (Slot(program[0], 1) != 0x200U || Slot(program[1], 1) != 0x201U || + Slot(program[2], 0) != 0x0802U || Slot(program[2], 2) != 0x490U || + Slot(program[2], 3) != 0xffffU) { + std::cerr << "unexpected HCCL-style presync prelude\n"; + return 3; + } + for (uint32_t ordinal = 0; ordinal < 3; ++ordinal) { + const uint32_t output = 3 + ordinal * 2; + const uint32_t token = output + 1; + if (Slot(program[output], 0) != 0x100dU || Slot(program[output], 6) != 0x2U || + Slot(program[token], 0) != 0x100dU || Slot(program[token], 6) != 0x4U || + Slot(program[output], 4) != Slot(program[token], 4)) { + std::cerr << "presync output/token are not paired by channel\n"; + return 4; + } + } + for (uint32_t i = 9; i < 12; ++i) { + if (Slot(program[i], 0) != 0x0802U || Slot(program[i], 5) != 0x6U) { + std::cerr << "missing presync wait mask\n"; + return 5; + } + } + // Sorted peer 0 copy: send[target=0] -> recv_peer0[source=2]. + if (Immediate(program[12]) != spec.localSendAddr || + Immediate(program[14]) != 0x30000000ULL + 2ULL * spec.chunkBytes || + Slot(program[17], 0) != 0x1009U) { + std::cerr << "unexpected first remote copy offsets\n"; + return 6; + } + const uint32_t selfStart = 30; + const uint64_t selfOffset = 2ULL * spec.chunkBytes; + if (Immediate(program[selfStart]) != spec.localSendAddr + selfOffset || + Immediate(program[selfStart + 2]) != spec.localRecvAddr + selfOffset || + Slot(program[selfStart + 5], 0) != 0x1000U || + Slot(program[selfStart + 5], 1) != 0U || + Slot(program[selfStart + 5], 5) != 0U || + Slot(program[selfStart + 7], 0) != 0x1002U || + Slot(program[selfStart + 7], 3) != 0U || + Slot(program[selfStart + 7], 5) != 0U) { + std::cerr << "unexpected CCU self copy offsets\n"; + return 7; + } + auto corrupted = program; + corrupted[3].words[1] ^= 1ULL; + if (TileXRCcuValidateAllToAllMeshProgramBindings(spec, corrupted, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("output SyncXn") == std::string::npos) { + std::cerr << "corrupted output channel accepted: " << report.message << "\n"; + return 8; + } + auto sharedRemoteIds = spec; + for (uint32_t ordinal = 1; ordinal < sharedRemoteIds.peers.size(); ++ordinal) { + sharedRemoteIds.peers[ordinal].route.remoteXn = sharedRemoteIds.peers[0].route.remoteXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteAddrXn = + sharedRemoteIds.peers[0].route.preSyncRemoteAddrXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteTokenXn = + sharedRemoteIds.peers[0].route.preSyncRemoteTokenXn; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteNotifyCke = + sharedRemoteIds.peers[0].route.preSyncRemoteNotifyCke; + sharedRemoteIds.peers[ordinal].route.preSyncRemoteTokenNotifyCke = + sharedRemoteIds.peers[0].route.preSyncRemoteTokenNotifyCke; + sharedRemoteIds.peers[ordinal].route.postSyncRemoteNotifyCke = + sharedRemoteIds.peers[0].route.postSyncRemoteNotifyCke; + } + if (TileXRCcuBuildAllToAllMeshProgram(sharedRemoteIds, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "per-peer remote resource IDs rejected: " << report.message << "\n"; + return 7; + } + auto overlappingCke = spec; + overlappingCke.remoteCompletionCke = overlappingCke.peers[0].route.sourceCke; + for (auto& peer : overlappingCke.peers) { + peer.route.copyCompletionCke = overlappingCke.remoteCompletionCke; + } + if (TileXRCcuBuildAllToAllMeshProgram(overlappingCke, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("overlaps source CKE") == std::string::npos) { + std::cerr << "overlapping source/completion CKE accepted: " << report.message << "\n"; + return 8; + } + auto duplicate = spec; + duplicate.peers[1].route.copyChannelId = duplicate.peers[0].route.copyChannelId; + if (TileXRCcuBuildAllToAllMeshProgram(duplicate, &program, &report) != + TILEXR_ERROR_PARA_CHECK_FAIL || + report.message.find("duplicate") == std::string::npos) { + std::cerr << "duplicate peer resource accepted: " << report.message << "\n"; + return 8; + } + for (uint32_t localRank = 0; localRank < 4; ++localRank) { + auto rankSpec = spec; + rankSpec.localRank = localRank; + rankSpec.peers.clear(); + uint16_t ordinal = 0; + for (uint32_t peerRank = 0; peerRank < 4; ++peerRank) { + if (peerRank != localRank) { + rankSpec.peers.push_back(Peer(localRank, peerRank, ordinal++)); + } + } + if (TileXRCcuBuildAllToAllMeshProgram(rankSpec, &program, &report) != TILEXR_SUCCESS) { + std::cerr << "rank " << localRank << " rejected: " << report.message << "\n"; + return 9; + } + const uint64_t rankOffset = static_cast(localRank) * rankSpec.chunkBytes; + if (program.size() != 1811 || + Immediate(program[selfStart]) != rankSpec.localSendAddr + rankOffset || + Immediate(program[selfStart + 2]) != rankSpec.localRecvAddr + rankOffset) { + std::cerr << "rank " << localRank << " self offset mismatch\n"; + return 9; + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_backend_boundary.py b/tests/ccu/test_tilexr_ccu_backend_boundary.py index 8118445d..6622c390 100644 --- a/tests/ccu/test_tilexr_ccu_backend_boundary.py +++ b/tests/ccu/test_tilexr_ccu_backend_boundary.py @@ -237,9 +237,47 @@ def test_alltoall_overrides_only_copy_route_memory_not_sync_routes(self): self.assertIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute(", prepare_alltoall) self.assertIn("0U", prepare_alltoall) self.assertIn("uint32_t routeIndex = 0", override_apply) - self.assertIn("routeIndex != directCcuRemoteRouteMemoryOverrideSyncRouteIndex_", override_apply) + self.assertIn("override.syncRouteIndex != routeIndex", override_apply) self.assertIn("++routeIndex", override_apply) - self.assertIn("directCcuRemoteRouteMemoryOverrideAllRoutes_", override_apply) + self.assertIn("override.allRoutes", override_apply) + + def test_four_rank_mesh_gathers_imports_and_maps_three_routes_per_peer(self): + header = PLANNER_HEADER.read_text(encoding="utf-8") + planner = PLANNER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", header) + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", planner) + mesh_body = planner[ + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuAllToAllMeshInstallAttempt"): + planner.index("int TileXRCcuCollectivePlanner::PrepareDirectCcuSyncXnPingInstallAttempt") + ] + self.assertIn("rankSize != 4", mesh_body) + self.assertEqual(1, mesh_body.count("session.AllGather(")) + self.assertIn("endpoint.rank != peerRank", mesh_body) + self.assertIn("session.ImportRemoteMemoryBuffer", mesh_body) + self.assertNotIn("routeWithinPeer", mesh_body) + self.assertNotIn("SetDirectCcuRemoteRouteMemoryOverrideForSyncRoute", mesh_body) + self.assertNotIn("peer.imported.targetSegVa", mesh_body) + self.assertIn("ClearDirectCcuRemoteRouteMemoryOverride", mesh_body) + self.assertIn("TileXRCcuRunDirectAllToAllMeshInstallAttempt", mesh_body) + + exchange = planner[ + planner.index("int TileXRCcuCollectivePlanner::ExchangeDirectCcuRemoteNotifyCke"): + planner.index("void TileXRCcuCollectivePlanner::SetDirectCcuRemoteRouteMemoryOverride") + ] + self.assertIn("routesPerPeer = syncRouteCount / routedPeerCount", exchange) + self.assertIn("peerBufferIndex = syncIndex / routesPerPeer", exchange) + self.assertIn("peerLocalResourceOffset =", exchange) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", exchange) + + self.assertIn("std::vector", header) + self.assertIn("directCcuRemoteRouteMemoryOverrides_", header) + override_apply = planner[ + planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride"): + planner.index("#endif", planner.index("void TileXRCcuCollectivePlanner::ApplyDirectCcuRemoteRouteMemoryOverride")) + ] + self.assertIn("for (const auto &override : directCcuRemoteRouteMemoryOverrides_)", override_apply) + self.assertIn("override.syncRouteIndex != routeIndex", override_apply) if __name__ == "__main__": diff --git a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py index 006553b3..f30e01bf 100644 --- a/tests/ccu/test_tilexr_ccu_direct_orchestrator.py +++ b/tests/ccu/test_tilexr_ccu_direct_orchestrator.py @@ -1741,13 +1741,19 @@ def test_direct_orchestrator_is_wired_and_has_no_private_runtime_surface(self): self.assertIn("decoded=LoadImdToGSA", source) self.assertIn("decoded=TransRmtMemToLocMem", source) self.assertIn("decoded=TransLocMemToRmtMem", source) + self.assertIn("decoded=TransLocMemToLocMem", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_RMT_MEM_TO_LOC_MEM_HEADER", source) self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_RMT_MEM_HEADER", source) + self.assertIn("TILEXR_CCU_TRACE_TRANS_LOC_MEM_TO_LOC_MEM_HEADER", source) def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") planner = PLANNER_SOURCE.read_text(encoding="utf-8") + two_rank_body = source[ + source.index("int BuildDirectAllToAll2RankLaunchPackage"): + source.index("int BuildDirectAllToAllMeshLaunchPackage") + ] self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_SYNC_RESOURCE_COUNT = 3U", source) self.assertIn("TILEXR_CCU_DIRECT_ALLTOALL_INSTRUCTION_COUNT =\n 7U + 64U * 7U", source) @@ -1777,6 +1783,7 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): self.assertIn("preSyncTokenChannelId = preResource.channelId", source) self.assertIn("preSyncLocalMarkerXn = copyResource.localXn", source) self.assertIn("preSyncRemoteMarkerXn = copyResource.remoteXn", source) + self.assertIn("preSyncMarkerChannelId = alltoallSpec.preSyncChannelId", source) self.assertIn("preSyncMarkerArgIndex = 0", source) self.assertIn("preSyncMarkerEnabled = true", source) self.assertNotIn("preSyncTokenChannelId = postResource.channelId", source) @@ -1813,9 +1820,9 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): ) self.assertIn("postSyncWait = false", source) self.assertIn("emitFinish = false", source) - self.assertNotIn("postSyncNotify = true", source) - self.assertNotIn("postSyncWait = true", source) - self.assertNotIn("emitFinish = true", source) + self.assertNotIn("postSyncNotify = true", two_rank_body) + self.assertNotIn("postSyncWait = true", two_rank_body) + self.assertNotIn("emitFinish = true", two_rank_body) self.assertIn("LocalToRemote", source) self.assertIn("uint32_t memSlicePerBlock", header) @@ -1824,6 +1831,98 @@ def test_direct_alltoall_uses_three_sync_resources_and_distinct_phases(self): with self.subTest(needle=needle): self.assertNotIn(needle, combined) + def test_direct_four_rank_mesh_builds_one_three_channel_launch_package(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_direct_orchestrator.h" + + #include + + using namespace TileXR; + + int main() + { + TileXRCcuBasicInfo basic; + basic.dieId = 1; + basic.msId = 0x45; + basic.msidToken.valid = true; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.tokenValue = 0x5678; + basic.missionKey = 0x059b0f03U; + basic.resourceAddr = 0x100000000ULL; + basic.caps.cap0 = (7U << 24) | (11U << 16) | 4095U; + basic.caps.cap1 = (63U << 16) | 31U; + basic.caps.cap2 = (63U << 16) | 63U; + basic.caps.cap3 = (127U << 16) | 31U; + basic.caps.cap4 = 15U; + + TileXRCcuDirectInstallOptions options; + options.basicInfo = &basic; + options.deviceId = 4; + options.rank = 2; + options.provider = "unit-test-direct-alltoall-mesh"; + options.missionStartId = 6; + options.instructionStartId = 475; + options.missionInstructionStartId = 489; + options.xnStartId = 1961; + options.gsaStartId = 510; + options.ckeStartId = 332; + options.channelStartId = 2; + options.offlineOnly = true; + + TileXRCcuDirectAllToAllMeshSpec mesh; + mesh.rankSize = 4; + mesh.localRank = 2; + mesh.localSendAddr = 0x10000000ULL; + mesh.localSendToken = TileXRCcuPackMemoryToken(1, 2, true); + mesh.localRecvAddr = 0x20000000ULL; + mesh.localRecvToken = TileXRCcuPackMemoryToken(2, 3, true); + mesh.chunkBytes = 2ULL * 1024ULL * 1024ULL; + for (uint32_t peerRank : {3U, 0U, 1U}) { + TileXRCcuDirectAllToAllMeshPeerSpec peer; + peer.peerRank = peerRank; + peer.remoteRecvAddr = 0x30000000ULL + peerRank * 0x1000000ULL; + peer.remoteRecvToken = TileXRCcuPackMemoryToken(10 + peerRank, 20 + peerRank, true); + mesh.peers.push_back(peer); + } + + TileXRCcuDirectInstallAttempt attempt; + TileXRCcuDirectInstallReport report; + const int ret = TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh, &attempt, &report); + (void)ret; + if (!report.pipelineBuilt || attempt.plan.syncResources.size() != 3 || + attempt.plan.taskWindows.size() != 1 || attempt.package.tasks.size() != 1 || + attempt.package.program.sync.size() != 1811 || + attempt.plan.taskWindows[0].instCnt != 1811 || + attempt.plan.kernelLocalGsa.num != 2 || attempt.allocation.sourceCke.num != 2 || + attempt.plan.barrierMode != TileXRCcuBarrierMode::SyncCke) { + std::cerr << "unexpected mesh package: " << report.message + << " resources=" << attempt.plan.syncResources.size() + << " instructions=" << attempt.package.program.sync.size() << "\n"; + return 1; + } + basic.caps.cap0 = (7U << 24) | (11U << 16) | 1599U; + TileXRCcuDirectInstallAttempt smallAttempt; + TileXRCcuDirectInstallReport smallReport; + if (TileXRCcuRunDirectAllToAllMeshInstallAttempt( + options, mesh, &smallAttempt, &smallReport) == TILEXR_SUCCESS || + smallReport.message.find("instruction") == std::string::npos || + smallReport.message.find("requested=") == std::string::npos || + smallReport.message.find("available=") == std::string::npos) { + std::cerr << "missing mesh capacity diagnostics: " << smallReport.message << "\n"; + return 2; + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_direct_install_options_default_to_lower_layer_first(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") @@ -1864,7 +1963,7 @@ def test_collective_planner_has_private_alltoall_prepare_path(self): self.assertIn("tilexr-comm-direct-ccu-alltoall", source) self.assertIn("TileXRCcuRunDirectAllToAll2RankInstallAttempt", source) - def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): + def test_direct_sync_xn_ping_uses_one_mission_route_and_full_4p_transport_resources(self): header = DIRECT_HEADER.read_text(encoding="utf-8") source = DIRECT_SOURCE.read_text(encoding="utf-8") planner_header = PLANNER_HEADER.read_text(encoding="utf-8") @@ -1877,13 +1976,23 @@ def test_direct_sync_xn_ping_uses_one_route_and_variable_bit_masks(self): self.assertIn("BuildDirectSyncXnPingLaunchPackage", source) self.assertIn("TileXRCcuEncodeSyncXn", source) self.assertIn("defaultRemoteNotifyMask = static_cast(1U << syncXnPing.localRank)", source) - self.assertIn("defaultLocalWaitMask = static_cast(1U << syncXnPing.peerRank)", source) - self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 5U", source) + self.assertIn("TILEXR_CCU_DIRECT_SYNC_XN_PING_INSTRUCTION_COUNT = 2U", source) + self.assertIn("SyncXnPingAllocationInstructionCount(options.syncResourceCount)", source) + self.assertIn("syncXnPing != nullptr ? TileXRCcuBarrierMode::SyncXn", source) + ping_body = source[ + source.index("int BuildDirectSyncXnPingLaunchPackage"): + source.index("void FillReportFromAttempt") + ] + self.assertNotIn("TileXRCcuEncodeSyncCke", ping_body) + self.assertNotIn("TileXRCcuEncodeSetCke", ping_body) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner_header) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", planner) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_NOTIFY_MASK", planner) self.assertIn("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_WAIT_MASK", planner) self.assertIn("RegisterCcuResourceRmaBuffer", planner) + self.assertIn("next.syncResourceCount = rankSize == 4 ? 3U : 1U", planner) + self.assertIn("attempt->plan.syncResources.empty()", source) + self.assertIn("syncXnPing != nullptr ? options.syncResourceCount", source) if __name__ == "__main__": diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py index c76a80a9..ae8ad68f 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_probe.py @@ -703,16 +703,73 @@ def test_alltoall_long_mission_reuses_prepare_with_loop_specific_state(self): self.assertIn("WaitForCollectiveSubmitDone(rank, rankSize, finalRet, loopIndex)", body) self.assertIn("adapter.ReadXnRange", source) self.assertIn("peerLoopMarker", source) + self.assertIn("attempt.plan.syncResources[0].remoteXn", body) + self.assertNotIn("attempt.plan.syncResources[0].localXn,", body) self.assertIn("loopIndex=", body) self.assertLess( body.index("PrepareDirectCcuAllToAll2RankInstallAttempt"), body.index("for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)"), ) + def test_four_rank_mesh_reuses_one_prepare_and_validates_full_matrix_each_loop(self): + source = PROBE_SOURCE.read_text(encoding="utf-8") + + self.assertIn( + 'kAllToAllMeshEnv = "TILEXR_CCU_DIRECT_SMOKE_ALLTOALL_MESH"', source + ) + self.assertIn("AllToAllMeshSmokeEnabled", source) + self.assertIn("InitAllToAllMeshState", source) + self.assertIn("ResetAllToAllMeshStateForLoop", source) + self.assertIn("BuildAllToAllMeshByte", source) + mesh_body = source[ + source.index("int RunAllToAllMeshLongMissionSmokeForRank"): + source.index("int RunAllToAllLongMissionSmokeForRank") + ] + self.assertNotIn("const uint32_t routeIndex = peerOrdinal * 3U", mesh_body) + self.assertNotIn("markerResource.remoteXn", mesh_body) + self.assertIn("RunAllToAllMeshLongMissionSmokeForRank", source) + mesh_dispatch = source[source.index("int RunAllToAllSmokeForRank"):] + self.assertLess( + mesh_dispatch.index("AllToAllMeshSmokeEnabled()"), + mesh_dispatch.index("rankSize != 2"), + ) + body = source[ + source.index("int RunAllToAllMeshLongMissionSmokeForRank"): + source.index("int RunAllToAllLongMissionSmokeForRank") + ] + loop = "for (int loopIndex = 0; loopIndex < loopCount; ++loopIndex)" + self.assertIn("rankSize != 4", body) + self.assertIn("PrepareDirectCcuAllToAllMeshInstallAttempt", body) + self.assertIn("aclrtCreateStream", body) + self.assertIn(loop, body) + self.assertLess(body.index("PrepareDirectCcuAllToAllMeshInstallAttempt"), body.index(loop)) + self.assertLess(body.index("aclrtCreateStream"), body.index(loop)) + self.assertNotIn("attempt.submitTasks.front().args[0] = localLoopMarker", body) + self.assertIn("WaitForCollectiveSubmitReadiness", body) + self.assertIn("WaitForCollectiveSubmitDone", body) + self.assertNotIn("peerOrdinal * 3U", body) + self.assertNotIn("ReadAndValidatePeerLoopMarker", body) + self.assertIn("CheckAllToAllState(&alltoall)", body) + self.assertIn("PrintCcuResourceState", body) + self.assertIn("resourceCount=3", body) + + pattern = source[ + source.index("uint8_t BuildAllToAllMeshByte"): + source.index("int InitAllToAllMeshState") + ] + for field in ["sourceRank", "targetRank", "loopIndex", "chunkOffset"]: + with self.subTest(field=field): + self.assertIn(field, pattern) + self.assertIn("static_cast(rankSize) * state->chunkBytes", source) + self.assertIn("sourceRank=", source) + self.assertIn("chunkOffset=", source) + self.assertIn("globalOffset=", source) + def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): source = PROBE_SOURCE.read_text(encoding="utf-8") self.assertIn('kSyncXnPingEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING"', source) + self.assertIn('kSyncXnPingPeerXorEnv = "TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR"', source) self.assertIn("SyncXnPingSmokeEnabled", source) self.assertIn("RunSyncXnPingSmokeForRank", source) self.assertIn("PrepareDirectCcuSyncXnPingInstallAttempt", source) @@ -722,10 +779,41 @@ def test_sync_xn_ping_smoke_mode_is_opt_in_and_uses_bounded_sync(self): ] self.assertIn("AllToAllState routeState", sync_ping_body) self.assertIn("InitAllToAllState(rank, peer, &routeState)", sync_ping_body) + self.assertIn("const int peer = rank ^ peerXor", sync_ping_body) + self.assertIn("peerXor < 1 || peerXor >= rankSize", sync_ping_body) + self.assertIn("options.syncInstructionCount = 2", sync_ping_body) + planner = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_collective_planner.cpp").read_text( + encoding="utf-8" + ) + self.assertIn("selectedDiagnosticPeer", planner) + self.assertIn("override.syncRouteIndex == 0U", planner) + self.assertIn("peerRanks.push_back(selectedDiagnosticPeer)", planner) + self.assertIn("selectedDiagnosticPeer >= 0 ?", planner) + runtime = (REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp").read_text( + encoding="utf-8" + ) + peer_route_body = runtime[ + runtime.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes("): + runtime.index("int TileXRCcuDirectRuntime::QueryTpHandleForPeer(", + runtime.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(")) + ] + self.assertIn("offer.qpKey = state.qpInfo.key", peer_route_body) + self.assertIn("importInfo.in.key = peerOffer.qpKey", peer_route_body) + self.assertNotIn("std::copy(offer.eid.begin(), offer.eid.end(), offer.qpKey.value)", runtime) + self.assertIn( + "offer.eid = state.resourceWindow.eid", + runtime, + ) + self.assertNotIn( + "std::copy(peerOffer.eid.begin(), peerOffer.eid.end(), importInfo.in.key.value)", + runtime, + ) self.assertIn("tilexr_ccu_sync_xn_ping prepare", source) self.assertIn("tilexr_ccu_sync_xn_ping submit", source) self.assertIn("tilexr_ccu_sync_xn_ping timing", source) self.assertIn("aclrtSynchronizeStreamWithTimeout", source) + self.assertIn('PrintCcuResourceState(\n context,', sync_ping_body) + self.assertIn('"tilexr_ccu_sync_xn_ping",\n 1U', sync_ping_body) def test_alltoall_timeout_prints_xn_and_cke_readback(self): source = PROBE_SOURCE.read_text(encoding="utf-8") diff --git a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py index cad3e2ba..99278d67 100644 --- a/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py +++ b/tests/ccu/test_tilexr_ccu_direct_smoke_runner.py @@ -17,10 +17,103 @@ class TileXRCcuDirectSmokeRunnerTest(unittest.TestCase): + def run_fake_mesh_runner(self, devices="4,5,6,7", rank_size="4", loop_count="10"): + temp_dir = tempfile.TemporaryDirectory() + temp_path = Path(temp_dir.name) + fake_bin = temp_path / "bin" + fake_bin.mkdir() + fake_cxx = fake_bin / "c++" + fake_cxx.write_text( + "#!/usr/bin/env bash\n" + "out=''\n" + "while [ $# -gt 0 ]; do\n" + " if [ \"$1\" = -o ]; then out=$2; shift 2; else shift; fi\n" + "done\n" + "cat > \"$out\" <<'PROBE'\n" + "#!/usr/bin/env bash\n" + "rank=${TILEXR_CCU_PROBE_RANK}\n" + "rank_size=${TILEXR_CCU_PROBE_RANK_SIZE}\n" + "loops=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}\n" + "echo \"tilexr_ccu_alltoall prepare ret=0 installSucceeded=1 submitReady=1\"\n" + "for ((loop=0; loop "${thread_log}" 2>&1', source) self.assertIn("ccu_thread.log", source) - self.assertIn("ccu_rank0.log", source) - self.assertIn("ccu_rank1.log", source) + self.assertIn('ccu_rank${rank}.log', source) self.assertIn("installSucceeded=1", source) self.assertIn("submitReady=1", source) self.assertLess(source.index("installSucceeded=1"), source.index("submitReady=1")) self.assertIn("${repo_root}/install/lib64/libtile-comm.so", source) - self.assertIn( - 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank0_env[@]}" TILEXR_CCU_PROBE_RANK=0', - source, - ) - self.assertIn( - 'timeout "${timeout_s}s" env "${common_env[@]}" "${rank1_env[@]}" TILEXR_CCU_PROBE_RANK=1', - source, - ) + self.assertIn('timeout "${timeout_s}s" env "${common_env[@]}" "${rank_env[@]}"', source) self.assertNotIn('bash -c "wait', source) self.assertIn("npu-smi rc=", source) self.assertIn("TILEXR_CCU_SMOKE_ALLOW_BUSY_NPU", source) @@ -104,10 +186,8 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): self.assertIn("TILEXR_CCU_SMOKE_REQUIRE_NPU_SMI", source) self.assertIn("ccu_npu_smi_busy_guard.py", source) self.assertIn("tilexr_ccu_direct_smoke_runner summary", source) - self.assertIn("rank0Status=", source) - self.assertIn("rank1Status=", source) - self.assertIn("rank0Log=", source) - self.assertIn("rank1Log=", source) + self.assertIn('rank${rank}Status=', source) + self.assertIn('rank${rank}Log=', source) self.assertIn("submitTiming", source) self.assertIn("syncMs=", source) self.assertIn("p2pCcuCopy", source) @@ -119,8 +199,7 @@ def test_runner_is_default_safe_and_documents_hardware_gate(self): "npu-smi info", '"${probe_bin}"', "TILEXR_CCU_DIRECT_SMOKE_ENABLE=1", - "TILEXR_CCU_PROBE_RANK=0", - "TILEXR_CCU_PROBE_RANK=1", + 'TILEXR_CCU_PROBE_RANK="${rank}"', ]: with self.subTest(needle=needle): self.assertLess(gate, source.index(needle)) @@ -177,19 +256,24 @@ def test_runner_sync_xn_ping_mode_applies_direct_ccu_resource_defaults(self): self.assertIn("sync_xn_ping_mode_enabled", source) self.assertIn("apply_sync_xn_ping_defaults", source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING:-0', source) + self.assertIn('common_env+=("TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR=${TILEXR_CCU_DIRECT_SMOKE_SYNC_XN_PING_PEER_XOR}")', source) self.assertIn('TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT="${TILEXR_CCU_DIRECT_SMOKE_DIRECT_CCU_ONLY_INIT:-1}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_BYTES="${TILEXR_CCU_ALLTOALL_BYTES:-2097152}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP="${TILEXR_CCU_ALLTOALL_MEM_SLICE_PER_LOOP:-8}"', source) self.assertIn('TILEXR_CCU_ALLTOALL_LOOP_COUNT="${TILEXR_CCU_ALLTOALL_LOOP_COUNT:-1}"', source) self.assertIn('common_env+=("TILEXR_CCU_ALLTOALL_LOOP_COUNT=${TILEXR_CCU_ALLTOALL_LOOP_COUNT}")', source) self.assertIn('TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT="${TILEXR_CCU_PROBE_SYNC_RESOURCE_COUNT:-1}"', source) - self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-3}"', source) + self.assertIn('TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT="${TILEXR_CCU_PROBE_SYNC_INSTRUCTION_COUNT:-2}"', source) + self.assertIn('TILEXR_CCU_PROBE_XN_START="${TILEXR_CCU_PROBE_XN_START:-1961}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_XN_START="${TILEXR_CCU_PROBE_REMOTE_XN_START:-2361}"', source) + self.assertIn('TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START="${TILEXR_CCU_PROBE_LOCAL_WAIT_CKE_START:-332}"', source) + self.assertIn('TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START="${TILEXR_CCU_PROBE_REMOTE_NOTIFY_CKE_START:-364}"', source) self.assertLess(source.index("apply_sync_xn_ping_defaults"), source.index("apply_alltoall_defaults")) def test_runner_allows_inactive_p2p_rank_to_skip_submit(self): source = RUNNER.read_text(encoding="utf-8") submit_check = source[ - source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]', source.index('if [ "${rank0_status}"')): + source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_SUBMIT:-0}" = "1" ]', source.index("any_rank_failed=0")): source.index('if [ "${TILEXR_CCU_DIRECT_SMOKE_EXPECT_BARRIER_WAIT:-0}" = "1" ]') ] @@ -287,10 +371,8 @@ def test_runner_passes_rank_specific_resource_window_eid_index(self): source = RUNNER.read_text(encoding="utf-8") self.assertIn("EID_INDEX", source[source.index("resource_window_token_fields=("):]) - self.assertIn('rank0_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK0"', source) - self.assertIn('rank1_token_var="TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}_RANK1"', source) - self.assertIn('echo "dryRun rank0 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank0_token_value}"', source) - self.assertIn('echo "dryRun rank1 TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank1_token_value}"', source) + self.assertIn('rank_var="${generic}_RANK${rank}"', source) + self.assertIn('echo "dryRun rank${rank} TILEXR_CCU_DIRECT_RESOURCE_WINDOW_${token_field}=${rank_token_value}"', source) def test_runner_dry_run_shows_repository_install_diagnostic_variants(self): with tempfile.TemporaryDirectory() as temp_dir: diff --git a/tests/ccu/test_tilexr_ccu_driver_adapter.py b/tests/ccu/test_tilexr_ccu_driver_adapter.py index 57066197..d402ef22 100644 --- a/tests/ccu/test_tilexr_ccu_driver_adapter.py +++ b/tests/ccu/test_tilexr_ccu_driver_adapter.py @@ -1031,7 +1031,9 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertIn("GetDieEnabled", header) self.assertIn("InstallInstructions", header) self.assertIn("InstallMsidToken", header) + self.assertIn("SetTaskKill", header) self.assertIn("CleanTaskKillState", header) + self.assertIn("TILEXR_CCU_U_OP_SET_TASKKILL, &out, report", source) self.assertIn("TILEXR_CCU_U_OP_CLEAN_TASKKILL_STATE, &out, report", source) self.assertIn("InstallPfeCtx", header) self.assertIn("InstallJettyCtx", header) @@ -1059,5 +1061,16 @@ def test_driver_adapter_is_wired_and_does_not_reference_hcomm_runtime_surface(se self.assertNotIn(needle, combined) + def test_jetty_install_batches_payloads_larger_than_custom_channel_array(self): + source = DRIVER_SOURCE.read_text(encoding="utf-8") + install = source[source.index("int TileXRCcuDriverAdapter::InstallJettyCtx"):] + install = install[:install.index("int TileXRCcuDriverAdapter::InstallChannelCtxV1")] + + self.assertIn("while (remaining > 0)", install) + self.assertIn("std::min(remaining, TILEXR_CCU_MAX_DATA_ARRAY_SIZE)", install) + self.assertIn("offset += batch", install) + self.assertIn("inputOffset += batch", install) + + if __name__ == "__main__": unittest.main() diff --git a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py index d9a8d372..6600960d 100644 --- a/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py +++ b/tests/ccu/test_tilexr_ccu_lower_layer_plan_builder.py @@ -4,6 +4,7 @@ # import shutil +import json import os import subprocess import tempfile @@ -17,6 +18,7 @@ BUILDER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_plan_builder.cpp" DIRECT_RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.h" DIRECT_RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp" +TOPOLOGY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_topology.cpp" PAYLOAD_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_lower_layer_payloads.cpp" SPECS_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_specs.cpp" ALLOCATOR_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_resource_allocator.cpp" @@ -57,6 +59,8 @@ def compile_and_run(self, code: str, env=None, extra_sources=None, extra_link_fl if compiler is None: self.skipTest("no local C++ compiler found") extra_sources = extra_sources or [] + if DIRECT_RUNTIME_SOURCE in extra_sources and TOPOLOGY_SOURCE not in extra_sources: + extra_sources = [*extra_sources, TOPOLOGY_SOURCE] extra_link_flags = extra_link_flags or [] with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) @@ -129,6 +133,75 @@ def compile_only(self, code: str): capture_output=True, ) + def test_topology_resolver_selects_peer_specific_hccs_eids(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_topology.h" + + #include + #include + #include + + using namespace TileXR; + + int main() + { + const char* rootPath = std::getenv("TILEXR_TEST_CCU_ROOT_INFO"); + std::vector routes; + std::string message; + const int ret = TileXRCcuResolvePeerEidRoutes( + rootPath == nullptr ? "" : rootPath, + 0, + {1, 2, 3}, + &routes, + &message); + if (ret != TILEXR_SUCCESS || routes.size() != 3) { + std::cerr << "resolve failed ret=" << ret << " message=" << message << "\n"; + return 1; + } + if (routes[0].localPort != "0/8" || routes[0].localEid[5] != 0x08 || + routes[1].localPort != "0/0" || routes[1].localEid[5] != 0x00 || + routes[2].localPort != "0/7" || routes[2].localEid[5] != 0x07) { + std::cerr << "peer-specific EID mapping mismatch\n"; + return 2; + } + return 0; + } + ''') + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + topo_path = temp_path / "topology.json" + root_path = temp_path / "rootinfo.json" + topo_path.write_text(json.dumps({ + "edge_list": [ + {"local_a": 0, "local_a_ports": ["0/8"], "local_b": 1, "local_b_ports": ["0/7"]}, + {"local_a": 0, "local_a_ports": ["0/0"], "local_b": 2, "local_b_ports": ["0/0"]}, + {"local_a": 0, "local_a_ports": ["0/7"], "local_b": 3, "local_b_ports": ["0/7"]}, + ] + }), encoding="utf-8") + root_path.write_text(json.dumps({ + "topo_file_path": str(topo_path), + "rank_list": [ + {"device_id": device, "local_id": device, "level_list": [{"rank_addr_list": addresses}]} + for device, addresses in [ + (0, [ + {"addr": "000000000000030000100000df160100", "ports": ["0/0"]}, + {"addr": "000000000008030000100000df160900", "ports": ["0/8"]}, + {"addr": "000000000007030000100000df160800", "ports": ["0/7"]}, + ]), + (1, []), + (2, []), + (3, []), + ] + ] + }), encoding="utf-8") + env = os.environ.copy() + env["TILEXR_TEST_CCU_ROOT_INFO"] = str(root_path) + result = self.compile_and_run(code, env=env, extra_sources=[TOPOLOGY_SOURCE]) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_builds_lower_layer_install_plan_from_tilexr_owned_specs(self): code = textwrap.dedent( r''' @@ -470,7 +543,7 @@ def test_transport_template_can_use_hcomm_ordered_pfe_partition_for_direct_ccu_e if (snapshot.pfeId != 3 || snapshot.startLocalJettyCtxId != 0 || snapshot.startJettyId != 1024 || - snapshot.pfeJettyCount != 23 || + snapshot.pfeJettyCount != 128 || snapshot.routes.size() != 2) { std::cerr << "hcomm ordered pfe partition not applied: pfeId=" << snapshot.pfeId << " startLocalJettyCtxId=" << snapshot.startLocalJettyCtxId @@ -553,6 +626,75 @@ def test_transport_template_can_use_hcomm_fe_id_pfe_partition_for_direct_ccu_exp self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_hcomm_pfe_window_keeps_base_and_maps_sparse_verified_jetty_context(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + + using namespace TileXR; + + int main() + { + setenv("TILEXR_CCU_DIRECT_LOWER_LAYER_PFE_PARTITION", "hcomm", 1); + TileXRCcuBasicInfo basic; + basic.dieId = 0; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {0, 2, 1}; + allocation.localXn = {0, 1961, 1}; + allocation.remoteXn = {0, 2361, 1}; + allocation.notifyCke = {0, 332, 1}; + + TileXRCcuRemoteCcuBufferInfo remote; + remote.remoteCcuVa = 0xf98000000000ULL; + remote.memoryTokenId = 0x1100; + remote.remoteEid[0] = 1; + remote.tpn = 0x51; + remote.doorbellVa = 0x3fffff85080ULL; + remote.doorbellTokenId = 0x1103; + remote.sqDepth = 8; + remote.startJettyId = 1026; + remote.endpointRouteVerified = true; + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, {remote}, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 1; + } + if (snapshot.startJettyId != 1024 || snapshot.pfeJettyCount != 128 || + snapshot.routes[0].wqeBasicBlockStartId != 64) { + std::cerr << "PFE window was narrowed\n"; + return 2; + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot( + snapshot, &plan, &report) != TILEXR_SUCCESS) { + std::cerr << report.message << "\n"; + return 3; + } + if (plan.pfes.size() != 1 || plan.jettys.size() != 1 || + plan.jettys[0].startJettyCtxId != 2 || plan.jettys[0].ctxs.size() != 1) { + std::cerr << "sparse jetty context mapping mismatch\n"; + return 4; + } + return 0; + } + ''') + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_transport_template_uses_local_pfe_id_from_ra_eid_func_id(self): code = textwrap.dedent( r''' @@ -1663,7 +1805,7 @@ def test_overlay_verified_endpoint_route_reuses_shared_jetty_for_multi_route_sna << " routeCount=" << snapshot.routes.size() << "\n"; return 3; } - if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 3 || plan.pfes.empty()) { + if (plan.jettys.empty() || plan.jettys[0].ctxs.size() != 1 || plan.pfes.empty()) { std::cerr << "install plan shape mismatch\n"; return 4; } @@ -1881,6 +2023,125 @@ def test_transport_template_carries_explicit_channel_owner_exchange_proof(self): self.assertEqual("", result.stderr) self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_shared_peer_jetty_routes_reuse_one_wqe_window_per_peer(self): + code = textwrap.dedent( + r''' + #include "ccu/tilexr_ccu_lower_layer_plan_builder.h" + + #include + #include + #include + + using namespace TileXR; + + uint16_t DecodeWqeStart(const TileXRCcuLocalJettyCtxData& ctx) + { + const uint16_t word22 = static_cast(ctx.raw[22]) | + static_cast(ctx.raw[23] << 8U); + const uint16_t word24 = static_cast(ctx.raw[24]) | + static_cast(ctx.raw[25] << 8U); + return static_cast(((word22 >> 12U) & 0xfU) | ((word24 & 0xffU) << 4U)); + } + + uint16_t DecodeChannelJetty(const TileXRCcuChannelCtxDataV1& ctx) + { + const uint16_t word18 = static_cast(ctx.raw[18]) | + static_cast(ctx.raw[19] << 8U); + const uint16_t word20 = static_cast(ctx.raw[20]) | + static_cast(ctx.raw[21] << 8U); + return static_cast(((word18 >> 12U) & 0xfU) | ((word20 & 0xfffU) << 4U)); + } + + int main() + { + setenv("TILEXR_CCU_DIRECT_LOWER_LAYER_WQE_MODE", "hcomm_cap", 1); + TileXRCcuBasicInfo basic; + basic.dieId = 0; + basic.msId = 0x45; + basic.msidToken.tokenId = 0x1234; + basic.msidToken.valid = true; + + TileXRCcuResourceAllocation allocation; + allocation.channels = {0, 2, 9}; + allocation.localXn = {0, 1961, 9}; + allocation.remoteXn = {0, 2361, 9}; + allocation.notifyCke = {0, 332, 9}; + allocation.localWaitCke = {0, 332, 9}; + allocation.remoteNotifyCke = {0, 364, 9}; + + std::vector buffers(9); + for (uint32_t route = 0; route < buffers.size(); ++route) { + const uint32_t peerOrdinal = route / 3U; + auto& buffer = buffers[route]; + buffer.remoteCcuVa = 0x90000000ULL + route * 0x1000ULL; + buffer.memoryTokenId = 0x2000U + route; + buffer.memoryTokenValue = 0x3000U + route; + buffer.remoteXnId = static_cast(2361U + route); + buffer.remoteNotifyCke = static_cast(364U + route); + buffer.peerRank = peerOrdinal + 1U; + for (uint32_t byte = 0; byte < buffer.remoteEid.size(); ++byte) { + buffer.remoteEid[byte] = static_cast(0x20U + peerOrdinal * 0x10U + byte); + } + buffer.tpn = 0x50U + peerOrdinal; + buffer.doorbellVa = 0x10000000ULL + peerOrdinal * 0x10000ULL; + buffer.doorbellTokenId = 0x4000U + peerOrdinal; + buffer.sqDepth = 8; + buffer.localDoorbellVa = 0x20000000ULL + peerOrdinal * 0x10000ULL; + buffer.localDoorbellTokenId = 0x5000U + peerOrdinal; + buffer.localSqDepth = 8; + buffer.startJettyId = static_cast(1024U + peerOrdinal); + buffer.endpointRouteVerified = true; + } + + TileXRCcuLowerLayerTransportSnapshot snapshot; + TileXRCcuLowerLayerPlanBuilderReport report; + if (TileXRCcuBuildLowerLayerTransportTemplate( + basic, allocation, buffers, &snapshot, &report) != TILEXR_SUCCESS) { + std::cerr << "template failed: " << report.message << "\n"; + return 1; + } + for (uint32_t route = 0; route < snapshot.routes.size(); ++route) { + const uint16_t expected = static_cast((route / 3U) * 32U); + if (snapshot.routes[route].wqeBasicBlockStartId != expected) { + std::cerr << "route WQE mismatch route=" << route << " observed=" + << snapshot.routes[route].wqeBasicBlockStartId << "\n"; + return 2; + } + } + + TileXRCcuLowerLayerInstallPlan plan; + if (TileXRCcuBuildLowerLayerInstallPlanFromTransportSnapshot(snapshot, &plan, &report) != + TILEXR_SUCCESS) { + std::cerr << "plan failed: " << report.message << "\n"; + return 3; + } + if (plan.jettys.size() != 1U || plan.jettys[0].ctxs.size() != 3U || + plan.channels.size() != 9U) { + std::cerr << "unexpected plan shape\n"; + return 4; + } + for (uint32_t peer = 0; peer < 3U; ++peer) { + if (DecodeWqeStart(plan.jettys[0].ctxs[peer]) != peer * 32U) { + std::cerr << "jetty context WQE mismatch peer=" << peer << "\n"; + return 5; + } + for (uint32_t route = 0; route < 3U; ++route) { + if (DecodeChannelJetty(plan.channels[peer * 3U + route].ctx) != 1024U + peer) { + std::cerr << "channel jetty mismatch peer=" << peer << " route=" << route << "\n"; + return 6; + } + } + } + return 0; + } + ''' + ) + + result = self.compile_and_run(code) + + self.assertEqual("", result.stderr) + self.assertEqual(0, result.returncode, result.stdout + result.stderr) + def test_plan_builder_rejects_incomplete_lower_layer_inputs(self): code = textwrap.dedent( r''' @@ -2256,16 +2517,13 @@ def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): "peerLocalXnId = static_cast(static_cast(peerResources.localXnStartId) + peerLocalXnOffset)", compact_body) self.assertIn("selectedRemoteXnOffset >= peerResources.remoteXnCount", compact_body) - self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", compact_body) self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertNotIn("SelectDirectCcuRemoteBindingOverride", compact_body) - self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn("(*remoteCcuBuffers)[routeIndex].remoteCcuVa +=", compact_body) self.assertNotIn("static_cast(peerLocalXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) self.assertNotIn("TILEXR_CCU_V1_XN_RESOURCE_OFFSET + static_cast(peerLocalXnId)", compact_body) - self.assertNotIn( - "uint16_t remoteXnId = static_cast(peerResources.localXnStartId + peerLocalIndex)", - compact_body) + self.assertIn("(*remoteCcuBuffers)[routeIndex].remoteXnId = channelBoundRemoteXnId", compact_body) self.assertNotIn( "channelBoundRemoteXnId = static_cast(allocation.remoteXn.startId + routeIndex)", compact_body) @@ -2273,6 +2531,14 @@ def test_remote_xn_exchange_uses_peer_channel_bound_remote_xn_operand(self): "static_cast((*remoteCcuBuffers)[routeIndex].remoteXnId) * TILEXR_CCU_XN_SLOT_BYTES", compact_body) + def test_lower_layer_clears_the_complete_allocated_remote_xn_range(self): + source = BUILDER_SOURCE.read_text(encoding="utf-8") + + self.assertIn("result.remoteXnStartId = allocation.remoteXn.startId", source) + self.assertIn("result.remoteXnCount = allocation.remoteXn.num", source) + self.assertIn("snapshot.remoteXnStartId", source) + self.assertIn("snapshot.remoteXnCount", source) + def test_remote_notify_cke_targets_peer_local_wait_cke(self): planner_source = CCU_PLANNER_SOURCE.read_text(encoding="utf-8") exchange_body = planner_source[ @@ -2305,14 +2571,19 @@ def test_peer_xn_exchange_expands_one_peer_window_to_multiple_sync_routes(self): self.assertIn("const size_t peerRouteCount = static_cast(rankSize - 1)", compact_body) self.assertIn("const size_t syncRouteCount = allocation.remoteXn.num", compact_body) - self.assertIn("allocation.remoteXn.num < static_cast(rankSize - 1)", compact_body) + self.assertIn("allocation.remoteXn.num < routedPeerCount", compact_body) self.assertNotIn("allocation.remoteXn.num != static_cast(rankSize - 1)", compact_body) - self.assertIn("std::vector peerCcuBuffers = *remoteCcuBuffers", compact_body) + self.assertIn("peerCcuBuffersByRank", compact_body) + self.assertIn("peerCcuBuffer.peerRank", compact_body) + self.assertIn("invalid direct CCU peer buffer rank mapping", compact_body) + self.assertIn("incomplete direct CCU peer buffer rank mapping", compact_body) self.assertIn("remoteCcuBuffers->assign(syncRouteCount, TileXRCcuRemoteCcuBufferInfo{})", compact_body) self.assertIn("for (uint32_t syncIndex = 0; syncIndex < allocation.remoteXn.num; ++syncIndex)", compact_body) - self.assertIn("const size_t peerBufferIndex = syncIndex % peerRouteCount", compact_body) - self.assertIn("(*remoteCcuBuffers)[routeIndex] = peerCcuBuffers[peerBufferIndex]", compact_body) - self.assertIn("SelectDirectCcuChannelBoundRemoteXnId(", compact_body) + self.assertIn("const size_t peerBufferIndex = syncIndex / routesPerPeer", compact_body) + self.assertIn( + "(*remoteCcuBuffers)[routeIndex] = *peerCcuBuffersByRank[static_cast(peer)]", + compact_body) + self.assertIn("peerLocalIndex * routesPerPeer + routeWithinPeer", compact_body) self.assertIn("peerResources.remoteXnStartId", compact_body) self.assertIn("DirectCcuRemoteXnProofSpan(allocation.remoteXn.num)", compact_body) @@ -2337,6 +2608,15 @@ def test_direct_ccu_runtime_imports_peer_endpoint_route_before_export(self): self.assertIn("remote.localDoorbellTokenId = localVerifiedEndpointRoute_.doorbellTokenId", compact_body) self.assertIn("remote.localDoorbellTokenValue = localVerifiedEndpointRoute_.doorbellTokenValue", compact_body) + def test_peer_endpoints_keep_per_peer_resource_and_jetty_tokens(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + + self.assertIn("offer.resourceTokenId = state.resourceWindow.tokenId", source) + self.assertIn("offer.resourceTokenValue = state.resourceWindow.tokenValue", source) + self.assertIn("offer.jettyTokenValue = state.resourceWindow.tokenValue", source) + self.assertIn("importInfo.in.ub.tokenValue = peerOffer.jettyTokenValue", source) + self.assertIn("state.route.memoryTokenValue = peerOffer.resourceTokenValue", source) + def test_direct_ccu_runtime_can_override_resource_window_token_from_rank_env(self): code = textwrap.dedent( r''' @@ -2601,10 +2881,17 @@ def test_direct_runtime_source_supports_selecting_ra_ctx_resource_window_eid(sel source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") self.assertIn("TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX", source) - self.assertIn("SelectRaCtxResourceWindowEidInfo", source) + self.assertIn("BuildRaCtxResourceWindowEidCandidates", source) self.assertIn("SelectRankedEnv(TILEXR_CCU_DIRECT_RESOURCE_WINDOW_EID_INDEX_ENV", source) + self.assertIn("SelectDirectCcuCleanupDieId()", source) + self.assertIn("it->dieId == dieId", source) + self.assertIn("TILEXR_CCU_UBOE_DEV_FLAG_RIGHT_SHIFT = 19U", source) + self.assertIn("QueryTpHandleForPeer(ctxHandle, candidateEid, candidateEid", source) + self.assertIn("loopEidCandidate", source) self.assertIn("TraceRaCtxEidInfos", source) - self.assertIn("ctxAttr.ub.eidIndex = selectedEid.eidIndex", source) + self.assertIn("ctxAttr.ub.eidIndex = candidate.eidIndex", source) + self.assertEqual(2, source.count( + "qpAttr.ub.errTimeout = TILEXR_CCU_DIRECT_ENDPOINT_ERR_TIMEOUT")) def test_direct_ccu_runtime_can_select_ra_ctx_resource_window_eid_by_env(self): code = textwrap.dedent( diff --git a/tests/ccu/test_tilexr_ccu_microcode.py b/tests/ccu/test_tilexr_ccu_microcode.py index bf06d5e5..b61e29be 100644 --- a/tests/ccu/test_tilexr_ccu_microcode.py +++ b/tests/ccu/test_tilexr_ccu_microcode.py @@ -465,6 +465,19 @@ def test_memory_transfer_microcode_encoders_match_hcomm_v1_layout(self): return 4; } + TileXRCcuInstr local; + if (TileXRCcuEncodeTransLocMemToLocMem(spec, &local) != TILEXR_SUCCESS) { + std::cerr << "trans loc->loc encode failed\n"; + return 5; + } + if (local.words[0] != 0x010102020201100aULL || + local.words[1] != 0x5a00001203010102ULL || + local.words[2] != 0x0007000000000000ULL || + local.words[3] != 0x0003040200020401ULL) { + std::cerr << "unexpected trans loc->loc words\n"; + return 6; + } + return 0; } ''' @@ -489,7 +502,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): TileXRCcuInstr instr; TileXRCcuMemTransferSpec empty; if (TileXRCcuEncodeTransRmtMemToLocMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || - TileXRCcuEncodeTransLocMemToRmtMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + TileXRCcuEncodeTransLocMemToRmtMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(empty, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "empty transfer accepted\n"; return 1; } @@ -505,7 +519,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): spec.setCkeMask = 8; if (TileXRCcuEncodeTransRmtMemToLocMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL || - TileXRCcuEncodeTransLocMemToRmtMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + TileXRCcuEncodeTransLocMemToRmtMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(spec, nullptr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "null output accepted\n"; return 2; } @@ -518,7 +533,8 @@ def test_memory_transfer_microcode_rejects_missing_required_fields(self): spec.reduceDataType = 0; spec.reduceOpCode = 0x10; - if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { + if (TileXRCcuEncodeTransLocMemToRmtMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL || + TileXRCcuEncodeTransLocMemToLocMem(spec, &instr) != TILEXR_ERROR_PARA_CHECK_FAIL) { std::cerr << "out-of-range reduce op code accepted\n"; return 4; } @@ -552,12 +568,14 @@ def test_microcode_builder_is_wired_and_has_no_private_hcomm_surface(self): self.assertIn("struct TileXRCcuMemTransferSpec", header) self.assertIn("TileXRCcuEncodeTransRmtMemToLocMem", header) self.assertIn("TileXRCcuEncodeTransLocMemToRmtMem", header) + self.assertIn("TileXRCcuEncodeTransLocMemToLocMem", header) self.assertIn("0x0001U", source) self.assertIn("0x0002U", source) self.assertIn("0x0003U", source) self.assertIn("0x0802U", source) self.assertIn("0x0804U", source) self.assertIn("0x1008U", source) + self.assertIn("0x100aU", source) self.assertIn("0x1009U", source) self.assertIn("0x100bU", source) self.assertIn("0x100dU", source) diff --git a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py index f03bfc52..56cc7fe2 100644 --- a/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py +++ b/tests/ccu/test_tilexr_ccu_ra_custom_channel_loader.py @@ -18,6 +18,7 @@ ABI_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_abi_constants.h" DIRECT_RUNTIME_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.h" DIRECT_RUNTIME_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_direct_runtime.cpp" +TOPOLOGY_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_topology.cpp" DRIVER_HEADER = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.h" DRIVER_SOURCE = REPO_ROOT / "src" / "comm" / "ccu" / "tilexr_ccu_driver_adapter.cpp" @@ -39,6 +40,8 @@ def compile_and_run(self, code: str, extra_sources=None, extra_link_flags=None, if compiler is None: self.skipTest("no local C++ compiler found") extra_sources = extra_sources or [] + if DIRECT_RUNTIME_SOURCE in extra_sources and TOPOLOGY_SOURCE not in extra_sources: + extra_sources = [*extra_sources, TOPOLOGY_SOURCE] extra_link_flags = extra_link_flags or [] with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) @@ -149,6 +152,8 @@ def test_ccu_hccp_loader_exposes_public_ra_ctx_endpoint_route_symbols(self): "TileXRCcuRaCtxQpBindFunc", "TileXRCcuRaCtxQpUnbindFunc", "TileXRCcuRaGetTpInfoListAsyncFunc", + "TileXRCcuRaGetTpAttrAsyncFunc", + "TileXRCcuRaSetTpAttrAsyncFunc", "TileXRCcuRaGetAsyncReqResultFunc", "TileXRCcuHccpQpCreateAttr", "TileXRCcuHccpQpImportInfo", @@ -167,6 +172,8 @@ def test_ccu_hccp_loader_exposes_public_ra_ctx_endpoint_route_symbols(self): "RaCtxQpBind", "RaCtxQpUnbind", "RaGetTpInfoListAsync", + "RaGetTpAttrAsync", + "RaSetTpAttrAsync", "RaGetAsyncReqResult", ]: with self.subTest(needle=needle): @@ -1270,8 +1277,50 @@ def test_direct_ccu_runtime_init_cleans_sticky_taskkill_state(self): source.index("void TileXRCcuDirectRuntime::Shutdown()") ] self.assertIn("adapter.CleanTaskKillState", init_body) - self.assertIn("TILEXR_CCU_DIRECT_DEFAULT_DIE_ID", init_body) - self.assertIn("TraceTaskKillCleanup", source) + self.assertIn("adapter.SetTaskKill", init_body) + self.assertIn("TILEXR_CCU_DIRECT_RECOVER_TASK_KILL_STATE", source) + self.assertLess( + init_body.index("adapter.SetTaskKill"), + init_body.index("adapter.CleanTaskKillState"), + ) + self.assertIn("cleanupRet = TILEXR_SUCCESS", init_body) + self.assertIn("RecoverTaskKillState() && cleanupRet != TILEXR_SUCCESS", init_body) + self.assertIn("SelectDirectCcuCleanupDieId", init_body) + self.assertIn("TraceTaskKillStep", source) + + def test_direct_runtime_selects_tp_sl_before_creating_peer_qp(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + create_body = source[ + source.index("int TileXRCcuDirectRuntime::CreatePeerEndpointState("): + source.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes(") + ] + + self.assertIn("TILEXR_CCU_TP_HANDLE_REQUEST_NUM = 8", source) + self.assertIn("RaGetTpAttrAsync", source) + self.assertIn("RaSetTpAttrAsync", source) + self.assertLess( + create_body.index("SelectTpRouteForPeer("), + create_body.index("RaCtxQpCreate("), + ) + self.assertIn("qpAttr.ub.priority = state->mappedJettyPriority", create_body) + self.assertNotIn("qpAttr.ub.priority = 2", create_body) + + def test_peer_endpoint_route_uses_the_driver_returned_jetty_id(self): + source = DIRECT_RUNTIME_SOURCE.read_text(encoding="utf-8") + prepare_body = source[ + source.index("int TileXRCcuDirectRuntime::PreparePeerEndpointRoutes("): + source.index("int TileXRCcuDirectRuntime::QueryTpHandleForPeer(") + ] + + self.assertIn( + "state.route.startJettyId = static_cast(state.qpInfo.ub.id)", + prepare_body, + ) + self.assertNotIn( + "state.route.startJettyId = static_cast(TILEXR_CCU_DIRECT_LOOP_JETTY_ID + ordinal)", + prepare_body, + ) + def test_direct_ccu_runtime_keeps_ra_custom_channel_provider_alive_for_created_adapters(self): header = DIRECT_RUNTIME_HEADER.read_text(encoding="utf-8") @@ -1484,7 +1533,7 @@ def test_direct_runtime_uses_optional_tilexr_endpoint_route_provider_before_env( return 2; } if (buffers.size() != 1 || !buffers[0].endpointRouteVerified || - buffers[0].remoteEid[0] != 0xaa || + buffers[0].remoteEid[15] != 0xaa || buffers[0].tpn != 0x10203 || buffers[0].doorbellVa != 0x1122334455667788ULL || buffers[0].doorbellTokenId != 0x3456 ||