diff --git a/src/main/scala/CMemoryLoader.scala b/src/main/scala/CMemoryLoader.scala index ef78bd8..e01ab50 100644 --- a/src/main/scala/CMemoryLoader.scala +++ b/src/main/scala/CMemoryLoader.scala @@ -139,6 +139,7 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ val IsLoadConherent = RegInit(true.B) val IsStoreConherent = RegInit(true.B) + val IsLoadTranspose = RegInit(false.B) val IsStoreTranspose = RegInit(false.B) val HasScarhpadRead = WireInit(false.B) @@ -168,6 +169,7 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ LoadTensorBlockBaseAddr := io.ConfigInfo.ApplicationTensor_C.BlockTensor_C_BaseVaddr ApplicationTensor_C_Stride_M := io.ConfigInfo.ApplicationTensor_C.ApplicationTensor_C_Stride_M IsLoadConherent := io.ConfigInfo.Conherent + IsLoadTranspose := io.ConfigInfo.Is_Transpose LoadMatrixRegTensor_M := io.ConfigInfo.MatrixRegTensor_M LoadMatrixRegTensor_N := io.ConfigInfo.MatrixRegTensor_N HasTail := io.ConfigInfo.ApplicationTensor_C.HasTail @@ -245,6 +247,7 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ //3.更新FIFO，更新Tail，更新Table val MReg_Fill_Table = RegInit((VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(0.U(outsideDataWidth.W))))) val MReg_Fill_Table_MReg_Addr = RegInit((VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(0.U(log2Ceil(CMatrixRegBankNEntries).W)))))//记录这个LLC回的数是在scp的哪个地址 + val MReg_Fill_Table_TransposeLane = RegInit(VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(0.U(log2Ceil(CMatrixRegNBanks).W)))) val MReg_Fill_Table_Time = RegInit((VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(0.U((log2Ceil(outsideDataWidthByte/CMatrixRegEntryByteSize)+1).W)))))//记录这个LLC回的数需要回填的次数，完成就可以将数据释放了 val MReg_Fill_Table_IsTail = RegInit(VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(false.B))) val MReg_Fill_Table_Free = MReg_Fill_Table_Time.map(_ === 0.U)//记录这个FIFO能否能填数据 @@ -292,6 +295,7 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ Bank_Fill_Search_FIFO_Tail := 0.U.asTypeOf(Bank_Fill_Search_FIFO_Tail) MReg_Fill_Table := 0.U.asTypeOf(MReg_Fill_Table) MReg_Fill_Table_MReg_Addr := 0.U.asTypeOf(MReg_Fill_Table_MReg_Addr) + MReg_Fill_Table_TransposeLane := 0.U.asTypeOf(MReg_Fill_Table_TransposeLane) MReg_Fill_Table_Time := 0.U.asTypeOf(MReg_Fill_Table_Time) MReg_Fill_Table_IsTail := VecInit(Seq.fill(CMemoryLoaderReadFromMemoryFIFODepth)(false.B)) Request_M_Iter_Time := 0.U @@ -350,6 +354,9 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ val RequestMatrixRegBankId = (CurrentLoaded_BlockTensor_M_Iter + Request_M_Iter_Time) % CMatrixRegNBanks.U //访存请求落在哪个MatrixRegBank上 val RequestMatrixRegAddr = ((CurrentLoaded_BlockTensor_M_Iter + Request_M_Iter_Time) / CMatrixRegNBanks.U) * (Tensor_MN.U / Matrix_MN.U) + (CurrentLoaded_BlockTensor_N_Iter << log2Ceil(MAX_Fill_Times)) //该访存请求的第零号数据，落在哪个MatrixRegBank的哪个地址上 + val RequestMatrixRegTransposeBaseAddr = + CurrentLoaded_BlockTensor_N_Iter * (MAX_Fill_Times * (Tensor_MN / Matrix_MN)).U + + (CurrentLoaded_BlockTensor_M_Iter + Request_M_Iter_Time) / CMatrixRegNBanks.U ReadRequest.bits.RequestAddr := LoadTensorBlockBaseAddr + (CurrentLoaded_BlockTensor_M_Iter + Request_M_Iter_Time) * ApplicationTensor_C_Stride_M + (CurrentLoaded_BlockTensor_N_Iter << log2Ceil(outsideDataWidthByte)) @@ -370,7 +377,7 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ when(ReadRequest.fire){ val TableItem = Wire(new CSourceIdSearch) TableItem.MatrixRegBankId := RequestMatrixRegBankId - TableItem.MatrixRegAddr := RequestMatrixRegAddr + TableItem.MatrixRegAddr := Mux(IsLoadTranspose, RequestMatrixRegTransposeBaseAddr, RequestMatrixRegAddr) TableItem.MatrixRegisTail := RequestBeatIsTail SoureceIdSearchTable(sourceId.bits) := TableItem.asUInt @@ -397,14 +404,14 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ } val current_fill_fifo_full = WireInit(false.B) - when(ReadResponse.valid) + when(ReadResponse.valid && !IsLoadTranspose) { val sourceId = ReadResponse.bits.ReseponseSourceID val MatrixRegBankId = SoureceIdSearchTable(sourceId).asTypeOf(new CSourceIdSearch).MatrixRegBankId current_fill_fifo_full := Bank_Fill_Search_FIFO_Full(MatrixRegBankId) } - ReadResponse.ready := MReg_Fill_Table_Not_Full && (current_fill_fifo_full === false.B) + ReadResponse.ready := MReg_Fill_Table_Not_Full && (IsLoadTranspose || !current_fill_fifo_full) //接受访存的返回值 //一个cam来存储访存请求的source_id对应的MatrixReg的地址和bank号 //根据response的sourceid，找到对应的MatrixReg的地址和bank号，回填数据 @@ -418,11 +425,14 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ MReg_Fill_Table(MReg_Fill_Table_Insert_Index) := ResponseData MReg_Fill_Table_MReg_Addr(MReg_Fill_Table_Insert_Index) := MatrixRegAddr + MReg_Fill_Table_TransposeLane(MReg_Fill_Table_Insert_Index) := MatrixRegBankId MReg_Fill_Table_Time(MReg_Fill_Table_Insert_Index) := MAX_Fill_Times.U MReg_Fill_Table_IsTail(MReg_Fill_Table_Insert_Index) := searchEntry.MatrixRegisTail - Bank_Fill_Search_FIFO(MatrixRegBankId)(FIFOIndex) := MReg_Fill_Table_Insert_Index - Bank_Fill_Search_FIFO_Head(MatrixRegBankId) := WrapInc(Bank_Fill_Search_FIFO_Head(MatrixRegBankId), CMemoryLoaderReadFromMemoryFIFODepth) + when(!IsLoadTranspose) { + Bank_Fill_Search_FIFO(MatrixRegBankId)(FIFOIndex) := MReg_Fill_Table_Insert_Index + Bank_Fill_Search_FIFO_Head(MatrixRegBankId) := WrapInc(Bank_Fill_Search_FIFO_Head(MatrixRegBankId), CMemoryLoaderReadFromMemoryFIFODepth) + } //输出回填的数据 if (YJPCMLDebugEnable) @@ -432,49 +442,99 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ } //检查每个bank是否有数据需要回填 - HasScarhpadWrite := Have_Bank_Fill val Current_Fill_MReg_Time = WireInit(VecInit(Seq.fill(CMatrixRegNBanks)(0.U(1.W)))) - for (i <- 0 until CMatrixRegNBanks){ - when(Bank_Fill_Search_FIFO_Empty(i) === false.B){ - val CurrentFIFOIndex = Bank_Fill_Search_FIFO(i)(Bank_Fill_Search_FIFO_Tail(i)) - when(io.ToMatrixRegIO.LoadReadWriteResponse(MatrixRegTaskType.WriteFromMemoryLoaderIndex) === true.B) - { - Current_Fill_MReg_Time(i) := 1.U - val MatrixRegWriteRequest = io.ToMatrixRegIO.WriteRequestToMatrixReg - val FIFOData = WireInit((VecInit(Seq.fill(MAX_Fill_Times)(0.U((8*CMatrixRegEntryByteSize).W))))) - val fillSlot = MAX_Fill_Times.U - MReg_Fill_Table_Time(CurrentFIFOIndex) - val currentIsTail = MReg_Fill_Table_IsTail(CurrentFIFOIndex) - val fullByteMask = Fill(CMatrixRegEntryByteSize, true.B) - val tailByteMaskVec = Wire(Vec(MAX_Fill_Times, UInt(CMatrixRegEntryByteSize.W))) - for (j <- 0 until MAX_Fill_Times) { - val high = (j + 1) * CMatrixRegEntryByteSize - 1 - val low = j * CMatrixRegEntryByteSize - tailByteMaskVec(j) := tailTaskMask(high, low) + when(IsLoadTranspose) { + HasScarhpadWrite := MReg_Fill_Table_Not_Empty + when(MReg_Fill_Table_Not_Empty && io.ToMatrixRegIO.LoadReadWriteResponse(MatrixRegTaskType.WriteFromMemoryLoaderIndex)) { + val CurrentFIFOIndex = MReg_Fill_Table_Valid_Index + val fillSlot = MAX_Fill_Times.U - MReg_Fill_Table_Time(CurrentFIFOIndex) + val currentIsTail = MReg_Fill_Table_IsTail(CurrentFIFOIndex) + val transposeLane = MReg_Fill_Table_TransposeLane(CurrentFIFOIndex) + val matrixRegAddr = MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex) + + fillSlot * (Tensor_MN / Matrix_MN).U + val responseElements = MReg_Fill_Table(CurrentFIFOIndex).asTypeOf( + Vec(outsideDataWidthByte / ResultWidthByte, UInt((ResultWidthByte * 8).W))) + val fillPhase = if (MAX_Fill_Times == 1) 0.U(1.W) + else fillSlot(log2Ceil(MAX_Fill_Times) - 1, 0) + val elementByteMask = VecInit((0 until CMatrixRegNBanks).map { lane => + (((BigInt(1) << ResultWidthByte) - 1) << (lane * ResultWidthByte)).U(CMatrixRegEntryByteSize.W) + })(transposeLane) + + Current_Fill_MReg_Time(0) := 1.U + for (i <- 0 until CMatrixRegNBanks) { + val sourceElement = if (MAX_Fill_Times == 1) responseElements(i) else { + VecInit((0 until MAX_Fill_Times).map { phase => + responseElements(phase * CMatrixRegNBanks + i) + })(fillPhase) } - FIFOData := MReg_Fill_Table(CurrentFIFOIndex).asTypeOf(FIFOData) - MatrixRegWriteRequest.BankAddr(i).bits := MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex) + fillSlot - MatrixRegWriteRequest.BankAddr(i).valid := true.B - MatrixRegWriteRequest.Data(i).bits := FIFOData(fillSlot) - MatrixRegWriteRequest.Data(i).valid := true.B - MatrixRegWriteRequest.ByteMask(i).bits := Mux(currentIsTail, tailByteMaskVec(fillSlot), fullByteMask) - MatrixRegWriteRequest.ByteMask(i).valid := true.B - - MReg_Fill_Table_Time(CurrentFIFOIndex) := MReg_Fill_Table_Time(CurrentFIFOIndex) - 1.U - when(MReg_Fill_Table_Time(CurrentFIFOIndex) === 1.U){ - Bank_Fill_Search_FIFO_Tail(i) := WrapInc(Bank_Fill_Search_FIFO_Tail(i), CMemoryLoaderReadFromMemoryFIFODepth) + val tailElementValid = if (MAX_Fill_Times == 1) { + tailTaskMask((i + 1) * ResultWidthByte - 1, i * ResultWidthByte).andR + } else { + VecInit((0 until MAX_Fill_Times).map { phase => + val element = phase * CMatrixRegNBanks + i + tailTaskMask((element + 1) * ResultWidthByte - 1, element * ResultWidthByte).andR + })(fillPhase) } + val sourceElementValid = !currentIsTail || tailElementValid + val shiftedData = VecInit((0 until CMatrixRegNBanks).map { lane => + (sourceElement.pad(CMatrixRegEntryBitSize) << (lane * ResultWidthByte * 8))(CMatrixRegEntryBitSize - 1, 0) + })(transposeLane) + val MatrixRegWriteRequest = io.ToMatrixRegIO.WriteRequestToMatrixReg + + MatrixRegWriteRequest.BankAddr(i).bits := matrixRegAddr + MatrixRegWriteRequest.BankAddr(i).valid := sourceElementValid + MatrixRegWriteRequest.Data(i).bits := shiftedData + MatrixRegWriteRequest.Data(i).valid := sourceElementValid + MatrixRegWriteRequest.ByteMask(i).bits := elementByteMask + MatrixRegWriteRequest.ByteMask(i).valid := sourceElementValid + } - if (YJPCMLDebugEnable) + MReg_Fill_Table_Time(CurrentFIFOIndex) := MReg_Fill_Table_Time(CurrentFIFOIndex) - 1.U + } + }.otherwise { + HasScarhpadWrite := Have_Bank_Fill + for (i <- 0 until CMatrixRegNBanks){ + when(Bank_Fill_Search_FIFO_Empty(i) === false.B){ + val CurrentFIFOIndex = Bank_Fill_Search_FIFO(i)(Bank_Fill_Search_FIFO_Tail(i)) + when(io.ToMatrixRegIO.LoadReadWriteResponse(MatrixRegTaskType.WriteFromMemoryLoaderIndex) === true.B) { - //输出fill_time 和 fifoindex - printf("[CMemoryLoader_Load<%d>]bankid: %d,CurrentFIFOIndex %d,ScartchPadAddr: %x, MReg_Fill_Table_Time(CurrentFIFOIndex): %d\n", io.DebugInfo.DebugTimeStampe,i.U, CurrentFIFOIndex, MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex), MReg_Fill_Table_Time(CurrentFIFOIndex)) - printf("[CMemoryLoader_Load<%d>]bankid: %d,ScartchPadAddr: %x, BankAddr: %x, Data: %x, ByteMask: %x\n", io.DebugInfo.DebugTimeStampe,i.U, MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex), MatrixRegWriteRequest.BankAddr(i).bits, MatrixRegWriteRequest.Data(i).bits, MatrixRegWriteRequest.ByteMask(i).bits) - } - }.otherwise - { - if (YJPCMLDebugEnable) + Current_Fill_MReg_Time(i) := 1.U + val MatrixRegWriteRequest = io.ToMatrixRegIO.WriteRequestToMatrixReg + val FIFOData = WireInit((VecInit(Seq.fill(MAX_Fill_Times)(0.U((8*CMatrixRegEntryByteSize).W))))) + val fillSlot = MAX_Fill_Times.U - MReg_Fill_Table_Time(CurrentFIFOIndex) + val currentIsTail = MReg_Fill_Table_IsTail(CurrentFIFOIndex) + val fullByteMask = Fill(CMatrixRegEntryByteSize, true.B) + val tailByteMaskVec = Wire(Vec(MAX_Fill_Times, UInt(CMatrixRegEntryByteSize.W))) + for (j <- 0 until MAX_Fill_Times) { + val high = (j + 1) * CMatrixRegEntryByteSize - 1 + val low = j * CMatrixRegEntryByteSize + tailByteMaskVec(j) := tailTaskMask(high, low) + } + FIFOData := MReg_Fill_Table(CurrentFIFOIndex).asTypeOf(FIFOData) + MatrixRegWriteRequest.BankAddr(i).bits := MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex) + fillSlot + MatrixRegWriteRequest.BankAddr(i).valid := true.B + MatrixRegWriteRequest.Data(i).bits := FIFOData(fillSlot) + MatrixRegWriteRequest.Data(i).valid := true.B + MatrixRegWriteRequest.ByteMask(i).bits := Mux(currentIsTail, tailByteMaskVec(fillSlot), fullByteMask) + MatrixRegWriteRequest.ByteMask(i).valid := true.B + + MReg_Fill_Table_Time(CurrentFIFOIndex) := MReg_Fill_Table_Time(CurrentFIFOIndex) - 1.U + when(MReg_Fill_Table_Time(CurrentFIFOIndex) === 1.U){ + Bank_Fill_Search_FIFO_Tail(i) := WrapInc(Bank_Fill_Search_FIFO_Tail(i), CMemoryLoaderReadFromMemoryFIFODepth) + } + + if (YJPCMLDebugEnable) + { + //输出fill_time 和 fifoindex + printf("[CMemoryLoader_Load<%d>]bankid: %d,CurrentFIFOIndex %d,ScartchPadAddr: %x, MReg_Fill_Table_Time(CurrentFIFOIndex): %d\n", io.DebugInfo.DebugTimeStampe,i.U, CurrentFIFOIndex, MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex), MReg_Fill_Table_Time(CurrentFIFOIndex)) + printf("[CMemoryLoader_Load<%d>]bankid: %d,ScartchPadAddr: %x, BankAddr: %x, Data: %x, ByteMask: %x\n", io.DebugInfo.DebugTimeStampe,i.U, MReg_Fill_Table_MReg_Addr(CurrentFIFOIndex), MatrixRegWriteRequest.BankAddr(i).bits, MatrixRegWriteRequest.Data(i).bits, MatrixRegWriteRequest.ByteMask(i).bits) + } + }.otherwise { - printf("[CMemoryLoader_Load<%d>]bankid: %d no authority\n", io.DebugInfo.DebugTimeStampe,i.U) + if (YJPCMLDebugEnable) + { + printf("[CMemoryLoader_Load<%d>]bankid: %d no authority\n", io.DebugInfo.DebugTimeStampe,i.U) + } } } } @@ -741,7 +801,8 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ val M_Get_IteratorMax = Mux(IsStoreTranspose, (StoreMatrixRegTensor_M / (Matrix_MN.U * 2.U) + (StoreMatrixRegTensor_M % (Matrix_MN.U * 2.U) =/= 0.U)) * 2.U, (StoreMatrixRegTensor_M / Matrix_MN.U) + ((StoreMatrixRegTensor_M % Matrix_MN.U) =/= 0.U)) val N_Get_IteratorMax = WireInit(0.U(log2Ceil(CMatrixRegBankNEntries).W)) N_Get_IteratorMax := (StoreMatrixRegTensor_N / Matrix_MN.U) - val transpose_scp_addr = WireInit(0.U(log2Ceil(CMatrixRegBankNEntries).W)) + val matrixRegScpAddr = WireInit(0.U(log2Ceil(CMatrixRegBankNEntries).W)) + matrixRegScpAddr := Current_Load_M_iter * (Tensor_MN / Matrix_MN).U + Current_Load_N_iter // val Max_Caculate_Iter = M_Get_IteratorMax * N_Get_IteratorMax @@ -813,23 +874,29 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ //如果MatrixReg的仲裁结果允许我们读取数据 HasScarhpadRead := !FromMatrixRegReadFIFO_ISSUE_Full && !FromMatrixRegReadFIFOFull && TotalStoreRequestSize < MaxIncStoreScpRequestSize when(HasScarhpadRead){ - transpose_scp_addr := Current_Load_N_iter + Current_Load_M_iter * N_Get_IteratorMax if(YJPCMLDebugEnable) { - printf("[CMemoryLoader_Store<%d>]N_Get_IteratorMax: %x, Current_Load_N_iter: %x, Current_Load_M_iter: %x, transpose_scp_addr: %x\n", io.DebugInfo.DebugTimeStampe, N_Get_IteratorMax, Current_Load_N_iter, Current_Load_M_iter, transpose_scp_addr) + printf("[CMemoryLoader_Store<%d>]N_Get_IteratorMax: %x, Current_Load_N_iter: %x, Current_Load_M_iter: %x, matrixRegScpAddr: %x\n", io.DebugInfo.DebugTimeStampe, N_Get_IteratorMax, Current_Load_N_iter, Current_Load_M_iter, matrixRegScpAddr) } //根据MatrixReg的仲裁结果，我们可以读取数据了 for (i <- 0 until CMatrixRegNBanks){ - io.ToMatrixRegIO.ReadRequestToMatrixReg.BankAddr(i).bits := Mux(IsStoreTranspose, transpose_scp_addr, Current_Load_Scp_addr) + io.ToMatrixRegIO.ReadRequestToMatrixReg.BankAddr(i).bits := matrixRegScpAddr io.ToMatrixRegIO.ReadRequestToMatrixReg.BankAddr(i).valid := true.B } when(io.ToMatrixRegIO.StoreReadWriteResponse(MatrixRegTaskType.ReadFromMemoryLoaderIndex)){ TotalStoreRequestSize := TotalStoreRequestSize + 1.U - // logic for transpose - Current_Load_M_iter := Current_Load_M_iter + 1.U - when(Current_Load_M_iter === (M_Get_IteratorMax - 1.U)){ - Current_Load_M_iter := 0.U + when(IsStoreTranspose) { + Current_Load_M_iter := Current_Load_M_iter + 1.U + when(Current_Load_M_iter === (M_Get_IteratorMax - 1.U)) { + Current_Load_M_iter := 0.U + Current_Load_N_iter := Current_Load_N_iter + 1.U + } + }.otherwise { Current_Load_N_iter := Current_Load_N_iter + 1.U + when(Current_Load_N_iter === (N_Get_IteratorMax - 1.U)) { + Current_Load_N_iter := 0.U + Current_Load_M_iter := Current_Load_M_iter + 1.U + } } Current_Load_Scp_addr := Current_Load_Scp_addr + 1.U @@ -856,10 +923,12 @@ class CMemoryLoader(implicit p: Parameters) extends CuteModule{ //只要fifo内的数据有效，就可以写入LLC WriteRequest.valid := false.B when(!FromMatrixRegReadFIFOEmpty && Reorder_ToLLC_Reg_Ready_Get){ - val Read_Data_list = WireInit(VecInit(Seq.fill(Matrix_MN)(0.U(Per_GetMatrix_NDim_Width.W)))) - Read_Data_list := FromMatrixRegReadFIFO(FromMatrixRegReadFIFOTail).asTypeOf(Read_Data_list) + val Read_Data_matrix = Wire(Vec(Matrix_MN, Vec(Matrix_MN, UInt(ResultWidth.W)))) + Read_Data_matrix := FromMatrixRegReadFIFO(FromMatrixRegReadFIFOTail).asTypeOf(Read_Data_matrix) for (i <- 0 until Matrix_MN){ - Reorder_ToLLC_Reg(Reorder_ToLLC_Reg_Get_Index)(i)(Fill_LLC_Iter) := Read_Data_list(i) + val Transposed_Block_Row = VecInit((0 until Matrix_MN).map(j => Read_Data_matrix(j)(i))).asUInt + Reorder_ToLLC_Reg(Reorder_ToLLC_Reg_Get_Index)(i)(Fill_LLC_Iter) := + Mux(IsStoreTranspose, Transposed_Block_Row, Read_Data_matrix(i).asUInt) } //更新相关迭代器 Fill_LLC_Iter := WrapInc(Fill_LLC_Iter, Fill_LLC_Max_Iter) diff --git a/src/test/scala/cute/CMemoryLoaderStoreAddressSpec.scala b/src/test/scala/cute/CMemoryLoaderStoreAddressSpec.scala new file mode 100644 index 0000000..24e6800 --- /dev/null +++ b/src/test/scala/cute/CMemoryLoaderStoreAddressSpec.scala @@ -0,0 +1,131 @@ +package cute + +import chisel3._ +import chisel3.util._ +import chiseltest._ +import org.chipsalliance.cde.config.{Config, Parameters} +import org.scalatest.flatspec.AnyFlatSpec + +object CMemoryLoaderStoreAddressTestConfig { + val params: Parameters = new Config((_, _, _) => { + case CuteParamsKey => CuteParams.CUTE_8Tops_512SCP + }) +} + +class CMemoryLoaderStoreAddressHarness(implicit p: Parameters) extends CuteModule { + val io = IO(new Bundle { + val start = Input(Bool()) + val transpose = Input(Bool()) + val taskReady = Output(Bool()) + val bankAddr = Output(Vec(CMatrixRegNBanks, Valid(UInt(log2Ceil(CMatrixRegBankNEntries).W)))) + }) + + val loader = Module(new CMemoryLoader) + loader.io.DebugInfo.DebugTimeStampe := 0.U + + loader.io.LoadLocalMMUIO.ConherentRequsetSourceID.valid := true.B + loader.io.LoadLocalMMUIO.ConherentRequsetSourceID.bits := 0.U + loader.io.LoadLocalMMUIO.nonConherentRequsetSourceID.valid := false.B + loader.io.LoadLocalMMUIO.nonConherentRequsetSourceID.bits := 0.U + loader.io.StoreLocalMMUIO.ConherentRequsetSourceID.valid := true.B + loader.io.StoreLocalMMUIO.ConherentRequsetSourceID.bits := 0.U + loader.io.StoreLocalMMUIO.nonConherentRequsetSourceID.valid := false.B + loader.io.StoreLocalMMUIO.nonConherentRequsetSourceID.bits := 0.U + for (channel <- 0 until ABMatrixRegNBanks) { + loader.io.LoadLocalMMUIO.Request(channel).ready := true.B + loader.io.LoadLocalMMUIO.Response(channel).valid := false.B + loader.io.LoadLocalMMUIO.Response(channel).bits := 0.U.asTypeOf(new MMUResponseIO) + loader.io.StoreLocalMMUIO.Request(channel).ready := true.B + loader.io.StoreLocalMMUIO.Response(channel).valid := false.B + loader.io.StoreLocalMMUIO.Response(channel).bits := 0.U.asTypeOf(new MMUResponseIO) + } + + loader.io.ToMatrixRegIO.LoadReadWriteResponse := loader.io.ToMatrixRegIO.LoadReadWriteRequest + loader.io.ToMatrixRegIO.StoreReadWriteResponse := loader.io.ToMatrixRegIO.StoreReadWriteRequest + for (bank <- 0 until CMatrixRegNBanks) { + loader.io.ToMatrixRegIO.ReadRequestToMatrixReg.ReadResponseData(bank).valid := false.B + loader.io.ToMatrixRegIO.ReadRequestToMatrixReg.ReadResponseData(bank).bits := 0.U + } + io.bankAddr := loader.io.ToMatrixRegIO.ReadRequestToMatrixReg.BankAddr + + val config = loader.io.ConfigInfo + config.ApplicationTensor_C.ApplicationTensor_C_BaseVaddr := 0.U + config.ApplicationTensor_C.BlockTensor_C_BaseVaddr := 0.U + config.ApplicationTensor_C.ApplicationTensor_C_Stride_M := 0.U + config.ApplicationTensor_C.dataType := ElementDataType.DataTypeWidth32 + config.ApplicationTensor_C.HasTail := false.B + config.ApplicationTensor_C.TailByteMask := 0.U + config.ApplicationTensor_C.N_Beat_Count := 0.U + config.ApplicationTensor_D.ApplicationTensor_D_BaseVaddr := 0x1000.U + config.ApplicationTensor_D.BlockTensor_D_BaseVaddr := 0x1000.U + config.ApplicationTensor_D.ApplicationTensor_D_Stride_M := 256.U + config.ApplicationTensor_D.dataType := ElementDataType.DataTypeWidth32 + config.LoadTaskInfo.Is_ZeroLoad := false.B + config.LoadTaskInfo.Is_RepeatRowLoad := false.B + config.LoadTaskInfo.Is_FullLoad := true.B + config.Conherent := true.B + config.Is_Transpose := io.transpose + config.MatrixRegTensor_M := 64.U + config.MatrixRegTensor_N := 64.U + config.MatrixRegId := 0.U + config.LoadMicroTaskValid := false.B + config.LoadMicroTaskEndReady := true.B + config.StoreMicroTaskValid := io.start + config.StoreMicroTaskEndReady := true.B + if (EnableDifftest) { + config.pc.get := 0.U + config.coreid.get := 0.U + } + + io.taskReady := config.StoreMicroTaskReady +} + +class CMemoryLoaderStoreAddressSpec extends AnyFlatSpec with ChiselScalatestTester { + behavior of "the legacy C memory loader store address generator" + + private def checkAddresses(transpose: Boolean, expected: Seq[Int]): Unit = { + test(new CMemoryLoaderStoreAddressHarness()(CMemoryLoaderStoreAddressTestConfig.params)) { dut => + dut.reset.poke(true.B) + dut.clock.step(2) + dut.reset.poke(false.B) + dut.io.start.poke(false.B) + dut.io.transpose.poke(transpose.B) + + assert(dut.io.taskReady.peek().litToBoolean, "CML store did not become ready") + dut.io.start.poke(true.B) + dut.clock.step() + dut.io.start.poke(false.B) + + val observed = collection.mutable.ArrayBuffer.empty[Int] + var cycles = 0 + while (observed.size < expected.size && cycles < 64) { + val valid = dut.io.bankAddr.head.valid.peek().litToBoolean + for (bank <- dut.io.bankAddr.indices) { + assert(dut.io.bankAddr(bank).valid.peek().litToBoolean == valid, + s"bank $bank read-valid diverged") + } + if (valid) { + val address = dut.io.bankAddr.head.bits.peek().litValue.toInt + for (bank <- dut.io.bankAddr.indices) { + assert(dut.io.bankAddr(bank).bits.peek().litValue.toInt == address, + s"bank $bank read address diverged") + } + observed += address + } + dut.clock.step() + cycles += 1 + } + + assert(observed.toSeq == expected, + s"store MatrixReg address sequence differed: expected=$expected observed=$observed") + } + } + + it should "skip unused physical columns between logical row groups" in { + checkAddresses(transpose = false, Seq(0, 1, 2, 3, 4, 5, 6, 7, 64)) + } + + it should "use the physical row-group stride while traversing transposed blocks" in { + checkAddresses(transpose = true, Seq(0, 64, 128, 192, 256, 320, 384, 448, 1)) + } +} diff --git a/src/test/scala/cute/CMemoryLoaderTransposeSpec.scala b/src/test/scala/cute/CMemoryLoaderTransposeSpec.scala new file mode 100644 index 0000000..17cbe21 --- /dev/null +++ b/src/test/scala/cute/CMemoryLoaderTransposeSpec.scala @@ -0,0 +1,251 @@ +package cute + +import chisel3._ +import chisel3.util._ +import chiseltest._ +import org.chipsalliance.cde.config.{Config, Parameters} +import org.scalatest.flatspec.AnyFlatSpec + +import scala.collection.mutable + +object CMemoryLoaderTransposeTestConfig { + val params: Parameters = new Config((_, _, _) => { + case CuteParamsKey => CuteParams.CUTE_1Tops_64SCP + }) +} + +class CMemoryLoaderTransposeHarness(implicit p: Parameters) extends CuteModule { + val io = IO(new Bundle { + val start = Input(Bool()) + val sourceRows = Input(UInt(MatrixRegMaxTensorDimBitSize.W)) + val sourceColumns = Input(UInt(MatrixRegMaxTensorDimBitSize.W)) + val beatsPerRow = Input(UInt(MatrixRegMaxTensorDimBitSize.W)) + val base = Input(UInt(MMUAddrWidth.W)) + val stride = Input(UInt(MMUAddrWidth.W)) + val sourceId = Input(UInt(LLCSourceMaxNumBitSize.W)) + + val taskReady = Output(Bool()) + val taskEnd = Output(Bool()) + val request = Decoupled(new MMURequestIO) + val response = Flipped(Decoupled(new MMUResponseIO)) + val bankAddr = Output(Vec(CMatrixRegNBanks, Valid(UInt(log2Ceil(CMatrixRegBankNEntries).W)))) + val data = Output(Vec(CMatrixRegNBanks, Valid(UInt(CMatrixRegEntryBitSize.W)))) + val byteMask = Output(Vec(CMatrixRegNBanks, Valid(UInt(CMatrixRegEntryByteSize.W)))) + }) + + val loader = Module(new CMemoryLoader) + loader.io.DebugInfo.DebugTimeStampe := 0.U + + loader.io.LoadLocalMMUIO.ConherentRequsetSourceID.valid := true.B + loader.io.LoadLocalMMUIO.ConherentRequsetSourceID.bits := io.sourceId + loader.io.LoadLocalMMUIO.nonConherentRequsetSourceID.valid := false.B + loader.io.LoadLocalMMUIO.nonConherentRequsetSourceID.bits := 0.U + loader.io.LoadLocalMMUIO.Request(0) <> io.request + loader.io.LoadLocalMMUIO.Response(0) <> io.response + for (channel <- 1 until ABMatrixRegNBanks) { + loader.io.LoadLocalMMUIO.Request(channel).ready := true.B + loader.io.LoadLocalMMUIO.Response(channel).valid := false.B + loader.io.LoadLocalMMUIO.Response(channel).bits := 0.U.asTypeOf(new MMUResponseIO) + } + + loader.io.StoreLocalMMUIO.ConherentRequsetSourceID.valid := false.B + loader.io.StoreLocalMMUIO.ConherentRequsetSourceID.bits := 0.U + loader.io.StoreLocalMMUIO.nonConherentRequsetSourceID.valid := false.B + loader.io.StoreLocalMMUIO.nonConherentRequsetSourceID.bits := 0.U + for (channel <- 0 until ABMatrixRegNBanks) { + loader.io.StoreLocalMMUIO.Request(channel).ready := true.B + loader.io.StoreLocalMMUIO.Response(channel).valid := false.B + loader.io.StoreLocalMMUIO.Response(channel).bits := 0.U.asTypeOf(new MMUResponseIO) + } + + loader.io.ToMatrixRegIO.LoadReadWriteResponse := loader.io.ToMatrixRegIO.LoadReadWriteRequest + loader.io.ToMatrixRegIO.StoreReadWriteResponse := loader.io.ToMatrixRegIO.StoreReadWriteRequest + for (bank <- 0 until CMatrixRegNBanks) { + loader.io.ToMatrixRegIO.ReadRequestToMatrixReg.ReadResponseData(bank).valid := false.B + loader.io.ToMatrixRegIO.ReadRequestToMatrixReg.ReadResponseData(bank).bits := 0.U + } + io.bankAddr := loader.io.ToMatrixRegIO.WriteRequestToMatrixReg.BankAddr + io.data := loader.io.ToMatrixRegIO.WriteRequestToMatrixReg.Data + io.byteMask := loader.io.ToMatrixRegIO.WriteRequestToMatrixReg.ByteMask + + val config = loader.io.ConfigInfo + config.ApplicationTensor_C.ApplicationTensor_C_BaseVaddr := io.base + config.ApplicationTensor_C.BlockTensor_C_BaseVaddr := io.base + config.ApplicationTensor_C.ApplicationTensor_C_Stride_M := io.stride + config.ApplicationTensor_C.dataType := ElementDataType.DataTypeWidth32 + config.ApplicationTensor_C.HasTail := false.B + config.ApplicationTensor_C.TailByteMask := 0.U + config.ApplicationTensor_C.N_Beat_Count := io.beatsPerRow + config.ApplicationTensor_D.ApplicationTensor_D_BaseVaddr := 0.U + config.ApplicationTensor_D.BlockTensor_D_BaseVaddr := 0.U + config.ApplicationTensor_D.ApplicationTensor_D_Stride_M := 0.U + config.ApplicationTensor_D.dataType := ElementDataType.DataTypeWidth32 + config.LoadTaskInfo.Is_ZeroLoad := false.B + config.LoadTaskInfo.Is_RepeatRowLoad := false.B + config.LoadTaskInfo.Is_FullLoad := true.B + config.Conherent := true.B + config.Is_Transpose := true.B + config.MatrixRegTensor_M := io.sourceRows + config.MatrixRegTensor_N := io.sourceColumns + config.MatrixRegId := 0.U + config.LoadMicroTaskValid := io.start + config.LoadMicroTaskEndReady := true.B + config.StoreMicroTaskValid := false.B + config.StoreMicroTaskEndReady := true.B + if (EnableDifftest) { + config.pc.get := 0.U + config.coreid.get := 0.U + } + + io.taskReady := config.LoadMicroTaskReady + io.taskEnd := config.LoadMicroTaskEndValid +} + +class CMemoryLoaderTransposeSpec extends AnyFlatSpec with ChiselScalatestTester { + behavior of "the legacy C memory loader transpose path" + + private val base = 0x1000 + private val responseBytes = 64 + private val resultBytes = 4 + private val bankCount = 4 + private val entryBytes = 16 + private val entriesPerRowGroup = 16 + private val sourceRows = 8 + private val sourceColumns = 32 + private val beatsPerRow = sourceColumns * resultBytes / responseBytes + + private case class RequestMeta(sourceId: Int, row: Int, beat: Int) + + private def elementValue(row: Int, column: Int): Int = + 0x01000000 | (row << 12) | column + + private def responseData(row: Int, beat: Int): BigInt = { + (0 until responseBytes / resultBytes).foldLeft(BigInt(0)) { case (packed, element) => + val column = beat * (responseBytes / resultBytes) + element + packed | (BigInt(elementValue(row, column)) << (element * 32)) + } + } + + private def expectedBytes: Map[(Int, Int, Int), Int] = { + (for { + row <- 0 until sourceRows + column <- 0 until sourceColumns + byte <- 0 until resultBytes + } yield { + val bank = column % bankCount + val entry = (column / bankCount) * entriesPerRowGroup + row / bankCount + val byteOffset = (row % bankCount) * resultBytes + byte + val value = (elementValue(row, column) >> (byte * 8)) & 0xff + (bank, entry, byteOffset) -> value + }).toMap + } + + private def pokeResponse(dut: CMemoryLoaderTransposeHarness, meta: Option[RequestMeta]): Unit = { + dut.io.response.valid.poke(meta.nonEmpty.B) + dut.io.response.bits.ReseponseConherent.poke(true.B) + dut.io.response.bits.ReseponseSourceID.poke(meta.map(_.sourceId).getOrElse(0).U) + dut.io.response.bits.ReseponseData.poke(meta.map(m => responseData(m.row, m.beat)).getOrElse(BigInt(0)).U) + } + + it should "scatter out-of-order e32 cacheline responses into transposed C-bank entries" in { + test(new CMemoryLoaderTransposeHarness()(CMemoryLoaderTransposeTestConfig.params)) + .withAnnotations(Seq(VerilatorBackendAnnotation)) { dut => + dut.reset.poke(true.B) + dut.clock.step(2) + dut.reset.poke(false.B) + + dut.io.start.poke(false.B) + dut.io.sourceRows.poke(sourceRows.U) + dut.io.sourceColumns.poke(sourceColumns.U) + dut.io.beatsPerRow.poke(beatsPerRow.U) + dut.io.base.poke(base.U) + dut.io.stride.poke((sourceColumns * resultBytes).U) + dut.io.sourceId.poke(0.U) + dut.io.request.ready.poke(true.B) + pokeResponse(dut, None) + + var readyWait = 0 + while (!dut.io.taskReady.peek().litToBoolean && readyWait < 32) { + dut.clock.step() + readyWait += 1 + } + assert(dut.io.taskReady.peek().litToBoolean, "CML did not become ready") + dut.io.start.poke(true.B) + dut.clock.step() + dut.io.start.poke(false.B) + + val requests = mutable.ArrayBuffer.empty[RequestMeta] + val actualBytes = mutable.Map.empty[(Int, Int, Int), Int] + var activeResponse = Option.empty[RequestMeta] + var nextSourceId = 0 + var responseCount = 0 + var endCycle = -1 + var cycle = 0 + val requestCount = sourceRows * beatsPerRow + + while (endCycle < 0 && cycle < 512) { + if (activeResponse.isEmpty && requests.size == requestCount && responseCount < requestCount) { + activeResponse = Some(requests(requestCount - 1 - responseCount)) + } + dut.io.sourceId.poke(nextSourceId.U) + pokeResponse(dut, activeResponse) + + for (bank <- 0 until bankCount) { + val addrValid = dut.io.bankAddr(bank).valid.peek().litToBoolean + val dataValid = dut.io.data(bank).valid.peek().litToBoolean + val maskValid = dut.io.byteMask(bank).valid.peek().litToBoolean + assert(addrValid == dataValid && dataValid == maskValid, + s"bank $bank emitted mismatched write-valid signals") + if (addrValid) { + val entry = dut.io.bankAddr(bank).bits.peek().litValue.toInt + val data = dut.io.data(bank).bits.peek().litValue + val mask = dut.io.byteMask(bank).bits.peek().litValue + assert(entry >= 0 && entry < 256, s"bank $bank wrote out-of-range entry $entry") + assert(mask.bitCount == resultBytes, + s"bank $bank entry $entry expected a $resultBytes-byte write, got mask 0x${mask.toString(16)}") + for (byteOffset <- 0 until entryBytes if ((mask >> byteOffset) & 1) != 0) { + val key = (bank, entry, byteOffset) + assert(!actualBytes.contains(key), s"duplicate C MatrixReg byte write at $key") + actualBytes(key) = ((data >> (byteOffset * 8)) & 0xff).toInt + } + } + } + + val requestFire = dut.io.request.valid.peek().litToBoolean && dut.io.request.ready.peek().litToBoolean + val responseFire = activeResponse.nonEmpty && dut.io.response.ready.peek().litToBoolean + if (requestFire) { + val requestAddr = dut.io.request.bits.RequestAddr.peek().litValue.toInt + val requestSourceId = dut.io.request.bits.RequestSourceID.peek().litValue.toInt + val stride = sourceColumns * resultBytes + val offset = requestAddr - base + val row = offset / stride + val beat = (offset % stride) / responseBytes + assert(requestAddr == base + row * stride + beat * responseBytes, + s"unexpected request address 0x${requestAddr.toHexString}") + assert(row >= 0 && row < sourceRows, s"request row $row outside source matrix") + assert(beat >= 0 && beat < beatsPerRow, s"request beat $beat outside source row") + assert(requestSourceId == nextSourceId, + s"request source ID $requestSourceId did not match allocator $nextSourceId") + requests += RequestMeta(requestSourceId, row, beat) + nextSourceId += 1 + } + if (responseFire) { + responseCount += 1 + activeResponse = None + } + if (dut.io.taskEnd.peek().litToBoolean) { + endCycle = cycle + } + + dut.clock.step() + cycle += 1 + } + + assert(endCycle >= 0, s"CML transpose task did not complete after $cycle cycles") + assert(requests.size == requestCount, s"expected $requestCount requests, observed ${requests.size}") + assert(responseCount == requestCount, s"expected $requestCount responses, observed $responseCount") + assert(actualBytes.toMap == expectedBytes, + s"transposed C writes differ: missing=${expectedBytes.keySet.diff(actualBytes.keySet).take(8)}, extra=${actualBytes.keySet.diff(expectedBytes.keySet).take(8)}") + } + } +}