From cae5bd6d249972160f9f94f78e085efc2ed4ec70 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 8 Jan 2021 21:19:26 -0800 Subject: [PATCH 001/123] Default to using single-ported scratchpad memories --- src/main/scala/gemmini/Configs.scala | 1 + src/main/scala/gemmini/DSEConfigs.scala | 1 + src/main/scala/gemmini/ExecuteController.scala | 6 ++++-- src/main/scala/gemmini/GemminiConfigs.scala | 1 + src/main/scala/gemmini/Scratchpad.scala | 14 ++++++++++---- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 5e6b0336f..bdf4b65df 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -50,6 +50,7 @@ object GemminiConfigs { ex_queue_length = 8, rob_entries = 16, sp_banks = 4, + sp_singleported = true, acc_banks = 2, sp_capacity = CapacityInKilobytes(256), shifter_banks = 1, // TODO add separate parameters for left and up shifter banks diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 51bdc1927..a0d0aa85b 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -24,6 +24,7 @@ object DSEBaseConfig { sp_banks = 4, // TODO support one-bank designs acc_banks = 1, sp_capacity = CapacityInKilobytes(64), + sp_singleported = false, shifter_banks = 1, // TODO add separate parameters for left and up shifter banks dataflow = Dataflow.OS, acc_capacity = CapacityInKilobytes(16), diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index c3601d50e..a3af93ffb 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -324,11 +324,13 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val a_fire = a_valid && a_ready - dontTouch(a_fire) + val a_blocked = a_valid && !a_ready val b_fire = b_valid && b_ready + val b_blocked = b_valid && !b_ready val d_fire = d_valid && d_ready + val d_blocked = d_valid && !d_ready - val firing = start_inputting_a || start_inputting_b || start_inputting_d + val firing = (start_inputting_a || start_inputting_b || start_inputting_d) && !a_blocked && !b_blocked && !d_blocked when (!firing) { a_fire_counter := 0.U diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index fb9026d49..9a40853e3 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -22,6 +22,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( ex_queue_length: Int, rob_entries: Int, sp_banks: Int, // TODO support one-bank designs + sp_singleported: Boolean, sp_capacity: GemminiMemCapacity, acc_banks: Int, acc_capacity: GemminiMemCapacity, diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 39596cc37..fbd3707ce 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -95,7 +95,7 @@ class ScratchpadWriteIO(val n: Int, val w: Int, val mask_len: Int) extends Bundl val data = Output(UInt(w.W)) } -class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int) extends Module { +class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, singlePorted: Boolean) extends Module { // This is essentially a pipelined SRAM with the ability to stall pipeline stages require(w % aligned_to == 0 || w < aligned_to) @@ -119,7 +119,12 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int) extends val raddr = io.read.req.bits.addr val ren = io.read.req.fire() - val rdata = mem.read(raddr, ren).asUInt() + val rdata = if (singlePorted) { + assert(!(ren && io.write.en)) + mem.read(raddr, ren && !io.write.en).asUInt() + } else { + mem.read(raddr, ren).asUInt() + } val fromDMA = io.read.req.bits.fromDMA // Make a queue which buffers the result of an SRAM read if it can't immediately be consumed @@ -297,7 +302,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, io.busy := writer.module.io.busy || reader.module.io.busy || write_issue_q.io.deq.valid { - val banks = Seq.fill(sp_banks) { Module(new ScratchpadBank(sp_bank_entries, spad_w, mem_pipeline, aligned_to)) } + val banks = Seq.fill(sp_banks) { Module(new ScratchpadBank(sp_bank_entries, spad_w, mem_pipeline, aligned_to, config.sp_singleported)) } val bank_ios = VecInit(banks.map(_.io)) // Getting the output of the bank that's about to be issued to the writer @@ -315,10 +320,11 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // TODO we tie the write dispatch queue's, and write issue queue's, ready and valid signals together here val dmawrite = write_dispatch_q.valid && write_issue_q.io.enq.ready && + !(bio.write.en && config.sp_singleported.B) && !write_dispatch_q.bits.laddr.is_acc_addr && write_dispatch_q.bits.laddr.sp_bank() === i.U bio.read.req.valid := exread || (dmawrite && !write_dispatch_q.bits.laddr.is_garbage()) - ex_read_req.ready := bio.read.req.ready + ex_read_req.ready := bio.read.req.ready && !(bio.write.en && config.sp_singleported.B) // The ExecuteController gets priority when reading from SRAMs when (exread) { From 48c2046399a4ed20520e87c6b71d2b2226a4d4e9 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 27 Jan 2021 16:58:29 +0400 Subject: [PATCH 002/123] Attempt to fix --- src/main/scala/gemmini/ExecuteController.scala | 5 +---- src/main/scala/gemmini/Scratchpad.scala | 10 ++++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index a3af93ffb..a8eb1a3e2 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -324,13 +324,10 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val a_fire = a_valid && a_ready - val a_blocked = a_valid && !a_ready val b_fire = b_valid && b_ready - val b_blocked = b_valid && !b_ready val d_fire = d_valid && d_ready - val d_blocked = d_valid && !d_ready - val firing = (start_inputting_a || start_inputting_b || start_inputting_d) && !a_blocked && !b_blocked && !d_blocked + val firing = start_inputting_a || start_inputting_b || start_inputting_d when (!firing) { a_fire_counter := 0.U diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index fbd3707ce..d67e31abe 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -95,7 +95,7 @@ class ScratchpadWriteIO(val n: Int, val w: Int, val mask_len: Int) extends Bundl val data = Output(UInt(w.W)) } -class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, singlePorted: Boolean) extends Module { +class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, single_ported: Boolean) extends Module { // This is essentially a pipelined SRAM with the ability to stall pipeline stages require(w % aligned_to == 0 || w < aligned_to) @@ -107,9 +107,11 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, singleP val write = Flipped(new ScratchpadWriteIO(n, w, mask_len)) }) - // val mem = SyncReadMem(n, UInt(w.W)) val mem = SyncReadMem(n, Vec(mask_len, mask_elem)) + // When the scratchpad is single-ported, the writes take precedence + val singleport_busy_with_write = single_ported.B && io.write.en + when (io.write.en) { if (aligned_to >= w) mem.write(io.write.addr, io.write.data.asTypeOf(Vec(mask_len, mask_elem))) @@ -119,7 +121,7 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, singleP val raddr = io.read.req.bits.addr val ren = io.read.req.fire() - val rdata = if (singlePorted) { + val rdata = if (single_ported) { assert(!(ren && io.write.en)) mem.read(raddr, ren && !io.write.en).asUInt() } else { @@ -134,7 +136,7 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, singleP q.io.enq.bits.fromDMA := RegNext(fromDMA) val q_will_be_empty = (q.io.count +& q.io.enq.fire()) - q.io.deq.fire() === 0.U - io.read.req.ready := q_will_be_empty + io.read.req.ready := q_will_be_empty && !singleport_busy_with_write // Build the rest of the resp pipeline val rdata_p = Pipeline(q.io.deq, mem_pipeline) From b9a22d5c96c8f8b79bdb3bfa22572fc6211cd47d Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 27 Jan 2021 21:13:15 +0400 Subject: [PATCH 003/123] Fix about_to_fire_all_rows --- src/main/scala/gemmini/ExecuteController.scala | 6 +++--- src/main/scala/gemmini/Scratchpad.scala | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index a8eb1a3e2..cc8d3b04f 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -370,9 +370,9 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In // The last line in this (long) Boolean is just to make sure that we don't think we're done as soon as we begin firing // TODO change when square requirement lifted - val about_to_fire_all_rows = ((a_fire_counter === (block_size-1).U && a_valid) || a_fire_counter === 0.U) && - ((b_fire_counter === (block_size-1).U && b_valid) || b_fire_counter === 0.U) && - ((d_fire_counter === (block_size-1).U && d_valid) || d_fire_counter === 0.U) && + val about_to_fire_all_rows = ((a_fire_counter === (block_size-1).U && a_fire) || a_fire_counter === 0.U) && + ((b_fire_counter === (block_size-1).U && b_fire) || b_fire_counter === 0.U) && + ((d_fire_counter === (block_size-1).U && d_fire) || d_fire_counter === 0.U) && (a_fire_counter =/= 0.U || b_fire_counter =/= 0.U || d_fire_counter =/= 0.U) && cntl_ready diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index d67e31abe..5a981b778 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -121,12 +121,7 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, single_ val raddr = io.read.req.bits.addr val ren = io.read.req.fire() - val rdata = if (single_ported) { - assert(!(ren && io.write.en)) - mem.read(raddr, ren && !io.write.en).asUInt() - } else { - mem.read(raddr, ren).asUInt() - } + val rdata = mem.read(raddr, ren).asUInt() val fromDMA = io.read.req.bits.fromDMA // Make a queue which buffers the result of an SRAM read if it can't immediately be consumed @@ -141,6 +136,8 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, single_ // Build the rest of the resp pipeline val rdata_p = Pipeline(q.io.deq, mem_pipeline) io.read.resp <> rdata_p + + assert(!(single_ported.B && ren && io.write.en)) } class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V]) From 3dfca8b9042e46eea04099d3bf25efdfe0d992ae Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 27 Jan 2021 21:53:08 +0400 Subject: [PATCH 004/123] Make SRAMs single-ported --- src/main/scala/gemmini/Scratchpad.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 5a981b778..65d92e6c0 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -121,7 +121,13 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, single_ val raddr = io.read.req.bits.addr val ren = io.read.req.fire() - val rdata = mem.read(raddr, ren).asUInt() + val rdata = if (single_ported) { + assert(!(ren && io.write.en)) + mem.read(raddr, ren && !io.write.en).asUInt() + } else { + mem.read(raddr, ren).asUInt() + } + val fromDMA = io.read.req.bits.fromDMA // Make a queue which buffers the result of an SRAM read if it can't immediately be consumed @@ -136,8 +142,6 @@ class ScratchpadBank(n: Int, w: Int, mem_pipeline: Int, aligned_to: Int, single_ // Build the rest of the resp pipeline val rdata_p = Pipeline(q.io.deq, mem_pipeline) io.read.resp <> rdata_p - - assert(!(single_ported.B && ren && io.write.en)) } class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V]) From 94a53f0cf88672712b089970ec1c118a2c6c4379 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 27 Jan 2021 22:09:09 +0400 Subject: [PATCH 005/123] Simplify the scratchpad --- src/main/scala/gemmini/Scratchpad.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 65d92e6c0..c26952076 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -327,7 +327,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, !write_dispatch_q.bits.laddr.is_acc_addr && write_dispatch_q.bits.laddr.sp_bank() === i.U bio.read.req.valid := exread || (dmawrite && !write_dispatch_q.bits.laddr.is_garbage()) - ex_read_req.ready := bio.read.req.ready && !(bio.write.en && config.sp_singleported.B) + ex_read_req.ready := bio.read.req.ready // The ExecuteController gets priority when reading from SRAMs when (exread) { From 61edc059cdbbad1d9911d1f225789f638c9038e9 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 4 Feb 2021 12:34:40 -0800 Subject: [PATCH 006/123] Fix max block len computation (#53) * Fix max_block_len computation * Reduce CI_MAKE_NPROC * Support large block mvins in loopmatmul for transposed matrices --- .circleci/build-toolchains.sh | 2 +- .circleci/defaults.sh | 2 +- src/main/scala/gemmini/LoopMatmul.scala | 31 +++++++++++++------------ 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/.circleci/build-toolchains.sh b/.circleci/build-toolchains.sh index 39caa7720..1a4e4b131 100755 --- a/.circleci/build-toolchains.sh +++ b/.circleci/build-toolchains.sh @@ -28,5 +28,5 @@ if [ ! -d "$HOME/$1-install" ]; then cd $HOME # init all submodules including the tools (doesn't use CI_MAKE_PROC due to mem. constraints) - CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=$CI_MAKE_PROC $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh esp-tools + CHIPYARD_DIR="$LOCAL_CHIPYARD_DIR" NPROC=$CI_MAKE_NPROC $LOCAL_CHIPYARD_DIR/scripts/build-toolchains.sh esp-tools fi diff --git a/.circleci/defaults.sh b/.circleci/defaults.sh index 6100774a7..2d200104f 100755 --- a/.circleci/defaults.sh +++ b/.circleci/defaults.sh @@ -14,7 +14,7 @@ ############# # make parallelism -CI_MAKE_NPROC=8 +CI_MAKE_NPROC=4 LOCAL_MAKE_NPROC=$CI_MAKE_NPROC # verilator version diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 181202b3f..16285d482 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -57,6 +57,7 @@ class LoopMatmulLdA(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val max_col_dim = Mux(req.transpose, req.max_i, req.max_k) val max_blocks = Mux(max_col_dim <= max_block_len.U, max_col_dim, max_block_len.U) + // TODO: why do we have to use 1 when transpose val sp_addr_start = req.addr_start @@ -84,13 +85,13 @@ class LoopMatmulLdA(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (io.cmd.fire()) { // The order here is k, j, i - val next_i = floorAdd(i, 1.U, req.max_i) - val next_k = floorAdd(k, max_blocks, req.max_k, next_i === 0.U) + val next_row_iterator = floorAdd(row_iterator, 1.U, max_row_iterator) + val next_col_iterator = floorAdd(col_iterator, max_blocks, max_col_iterator, next_row_iterator === 0.U) - i := next_i - k := next_k + i := Mux(req.transpose, next_col_iterator, next_row_iterator) + k := Mux(req.transpose, next_row_iterator, next_col_iterator) - when (next_i === 0.U && next_k === 0.U) { + when (next_col_iterator === 0.U && next_row_iterator === 0.U) { state := idle } } @@ -182,13 +183,13 @@ class LoopMatmulLdB(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (io.cmd.fire()) { // The order here is k, j, i - val next_j = floorAdd(j, max_blocks, req.max_j) - val next_k = floorAdd(k, 1.U, req.max_k, next_j === 0.U) + val next_col_iterator = floorAdd(col_iterator, max_blocks, max_col_iterator) + val next_row_iterator = floorAdd(row_iterator, 1.U, max_row_iterator, next_col_iterator === 0.U) - k := next_k - j := next_j + k := Mux(req.transpose, next_col_iterator, next_row_iterator) + j := Mux(req.transpose, next_row_iterator, next_col_iterator) - when (next_j === 0.U && next_k === 0.U) { + when (next_row_iterator === 0.U && next_col_iterator === 0.U) { state := idle } } @@ -269,9 +270,9 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (req.dram_addr === 0.U) { state := idle }.elsewhen (io.cmd.fire()) { - // The order here is k, j, i - val next_i = floorAdd(i, max_blocks, req.max_i) - val next_j = floorAdd(j, 1.U, req.max_j, next_i === 0.U) + // The order here is j, i + val next_i = floorAdd(i, 1.U, req.max_i) + val next_j = floorAdd(j, max_blocks, req.max_j, next_i === 0.U) i := next_i j := next_j @@ -595,8 +596,8 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: max_addr: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, dma_max_bytes: Int) (implicit p: Parameters) extends Module { val iterator_bitwidth = 16 - val max_block_len = (dma_max_bytes / (block_size * input_w * 8)) max 1 - val max_block_len_acc = (dma_max_bytes / (block_size * acc_w * 8)) max 1 + val max_block_len = (dma_max_bytes / (block_size * input_w / 8)) max 1 + val max_block_len_acc = (dma_max_bytes / (block_size * acc_w / 8)) max 1 val io = IO(new Bundle { val in = Flipped(Decoupled(new RoCCCommand)) From 9dc6ec85919c92c587ef8cd64e22e8e50b5c365c Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Sat, 6 Feb 2021 18:52:09 -0800 Subject: [PATCH 007/123] Stopgap measure to fix convs (#54) --- SPIKE.hash | 2 +- software/gemmini-rocc-tests | 2 +- src/main/scala/gemmini/BeatMerger.scala | 3 +- src/main/scala/gemmini/Controller.scala | 15 +- src/main/scala/gemmini/DMA.scala | 11 +- .../scala/gemmini/DMAReadCommandTracker.scala | 16 - .../scala/gemmini/ExecuteController.scala | 2 +- src/main/scala/gemmini/GemminiConfigs.scala | 13 +- src/main/scala/gemmini/GemminiISA.scala | 9 + src/main/scala/gemmini/LoadController.scala | 27 +- src/main/scala/gemmini/LoopConv.scala | 1026 +++++++++++++++++ src/main/scala/gemmini/LoopMatmul.scala | 36 +- src/main/scala/gemmini/ROB.scala | 183 ++- src/main/scala/gemmini/Scratchpad.scala | 139 ++- src/main/scala/gemmini/StoreController.scala | 2 +- src/main/scala/gemmini/Util.scala | 9 + src/main/scala/gemmini/XactTracker.scala | 3 +- src/main/scala/gemmini/ZeroWriter.scala | 70 ++ 18 files changed, 1421 insertions(+), 147 deletions(-) create mode 100644 src/main/scala/gemmini/LoopConv.scala create mode 100644 src/main/scala/gemmini/ZeroWriter.scala diff --git a/SPIKE.hash b/SPIKE.hash index 1d05e3ee6..3ab5b634f 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -8626fb144e019895767830d850deca7711773e5c +a4ed25a96fdb47642b39d893b7e1ca36d07700aa diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 2802ca406..dab14715e 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 2802ca406323ef511e1c2939387144a47c250638 +Subproject commit dab14715ea5a8ba9049feb65d98022119b7a72f2 diff --git a/src/main/scala/gemmini/BeatMerger.scala b/src/main/scala/gemmini/BeatMerger.scala index cac08aacb..c3922f157 100644 --- a/src/main/scala/gemmini/BeatMerger.scala +++ b/src/main/scala/gemmini/BeatMerger.scala @@ -31,7 +31,6 @@ class BeatMerger[U <: Data](beatBits: Int, maxShift: Int, spadWidth: Int, accWid val io = IO(new Bundle { val req = Flipped(Decoupled(new XactTrackerEntry(maxShift, spadWidth, accWidth, spadRows, accRows, maxReqBytes, mvin_scale_t_bits, nCmds))) val in = Flipped(Decoupled(UInt(beatBits.W))) - // val in = Flipped(Decoupled(new BeatPackerIn(beatBits))) val out = Decoupled(new BeatMergerOut(spadWidth, accWidth, spadRows, accRows, alignedTo)) }) @@ -72,7 +71,7 @@ class BeatMerger[U <: Data](beatBits: Int, maxShift: Int, spadWidth: Int, accWid i.U >= spad_row_offset && i.U < spad_row_offset +& (req.bits.bytes_to_read - bytesSent) }) - io.out.bits.addr := req.bits.addr + meshRows.U * { + io.out.bits.addr := req.bits.addr + req.bits.block_stride * { val total_bytes_sent = req.bits.spad_row_offset + bytesSent Mux(req.bits.has_acc_bitwidth, // We only add "if" statements here to satisfy the Verilator linter. The code would be cleaner without the diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index a3d117898..599e134df 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -179,13 +179,22 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] */ // Incoming commands and ROB - val rob = Module(new ROB(new RoCCCommand, rob_entries, local_addr_t, meshRows*tileRows, meshColumns*tileColumns)) + val rob = Module(new ROB(outer.config, new RoCCCommand)) val raw_cmd = Queue(io.cmd) + // TODO replace 4,12,2 with parameters based on ROB size + val loop_conv_unroller_busy = false.B + /*val (unrolled_cmd_after_conv, loop_conv_unroller_busy) = LoopConv(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, + inputType.getWidth, accType.getWidth, dma_maxbytes) + unrolled_cmd_after_conv.ready := false.B*/ + // val (compressed_cmd, compressor_busy) = InstCompressor(unrolled_cmd) // compressed_cmd.ready := false.B - val (unrolled_cmd, loop_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + + // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) unrolled_cmd.ready := false.B @@ -361,7 +370,7 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] rob_completed_arb.io.out.ready := true.B // Wire up global RoCC signals - io.busy := raw_cmd.valid || loop_unroller_busy || rob.io.busy || spad.module.io.busy + io.busy := raw_cmd.valid || loop_conv_unroller_busy || loop_matmul_unroller_busy || rob.io.busy || spad.module.io.busy io.interrupt := tlb.io.exp.interrupt rob.io.solitary_preload := ex_controller.io.solitary_preload diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index d79295b38..af23998f5 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -24,6 +24,7 @@ class StreamReadRequest[U <: Data](spad_rows: Int, acc_rows: Int, mvin_scale_t_b val status = new MStatus val len = UInt(16.W) // TODO magic number val repeats = UInt(16.W) // TODO magic number + val block_stride = UInt(16.W) // TODO magic number val cmd_id = UInt(8.W) // TODO magic number override def cloneType: StreamReadRequest.this.type = new StreamReadRequest(spad_rows, acc_rows, mvin_scale_t_bits).asInstanceOf[this.type] @@ -38,7 +39,7 @@ class StreamReadResponse[U <: Data](spadWidth: Int, accWidth: Int, spad_rows: In val accumulate = Bool() val has_acc_bitwidth = Bool() val scale = UInt(mvin_scale_t_bits.W) - val rows = UInt(16.W) // TODO magic number + val repeats = UInt(16.W) // TODO magic number val last = Bool() val bytes_read = UInt(8.W) // TODO magic number val cmd_id = UInt(8.W) // TODO magic number @@ -93,7 +94,7 @@ class StreamReader[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T io.resp.bits.accumulate := beatPacker.io.out.bits.accumulate io.resp.bits.has_acc_bitwidth := beatPacker.io.out.bits.has_acc_bitwidth io.resp.bits.scale := RegEnable(xactTracker.io.peek.entry.scale, beatPacker.io.req.fire()) - io.resp.bits.rows := RegEnable(xactTracker.io.peek.entry.rows, beatPacker.io.req.fire()) + io.resp.bits.repeats := RegEnable(xactTracker.io.peek.entry.repeats, beatPacker.io.req.fire()) io.resp.bits.cmd_id := RegEnable(xactTracker.io.peek.entry.cmd_id, beatPacker.io.req.fire()) io.resp.bits.bytes_read := RegEnable(xactTracker.io.peek.entry.bytes_to_read, beatPacker.io.req.fire()) io.resp.bits.last := beatPacker.io.out.bits.last @@ -240,18 +241,20 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf io.reserve.entry.accumulate := req.accumulate io.reserve.entry.has_acc_bitwidth := req.has_acc_bitwidth io.reserve.entry.scale := req.scale - io.reserve.entry.rows := req.repeats + io.reserve.entry.repeats := req.repeats + io.reserve.entry.block_stride := req.block_stride io.reserve.entry.lg_len_req := DontCare // TODO just remove this from the IO completely io.reserve.entry.bytes_to_read := read_bytes_read io.reserve.entry.cmd_id := req.cmd_id - io.reserve.entry.addr := req.spaddr + meshRows.U * + io.reserve.entry.addr := req.spaddr + req.block_stride * Mux(req.has_acc_bitwidth, // We only add "if" statements here to satisfy the Verilator linter. The code would be cleaner without the // "if" condition and the "else" clause if (bytesRequested.getWidth >= log2Up(accWidthBytes+1)) bytesRequested / accWidthBytes.U else 0.U, if (bytesRequested.getWidth >= log2Up(spadWidthBytes+1)) bytesRequested / spadWidthBytes.U else 0.U) io.reserve.entry.spad_row_offset := Mux(req.has_acc_bitwidth, bytesRequested % accWidthBytes.U, bytesRequested % spadWidthBytes.U) + when (untranslated_a.fire()) { val next_vaddr = req.vaddr + read_bytes_read // send_size val new_page = next_vaddr(pgIdxBits-1, 0) === 0.U diff --git a/src/main/scala/gemmini/DMAReadCommandTracker.scala b/src/main/scala/gemmini/DMAReadCommandTracker.scala index 386bf52e2..a7eecfda6 100644 --- a/src/main/scala/gemmini/DMAReadCommandTracker.scala +++ b/src/main/scala/gemmini/DMAReadCommandTracker.scala @@ -24,12 +24,6 @@ class DMAReadCommandTracker[T <: Data](val nCmds: Int, val maxBytes: Int, tag_t: override def cloneType: this.type = new BitsT(tag_t.cloneType, cmd_id_t.cloneType).asInstanceOf[this.type] } - /*val bits = new Bundle { - val tag = Input(tag_t) - val bytes_to_read = Input(UInt(log2Up(maxBytes+1).W)) - val cmd_id = Output(cmd_id_t) - }*/ - val bits = new BitsT(tag_t.cloneType, cmd_id_t.cloneType) def fire(dummy: Int = 0) = valid && ready @@ -43,11 +37,6 @@ class DMAReadCommandTracker[T <: Data](val nCmds: Int, val maxBytes: Int, tag_t: override def cloneType: this.type = new RequestReturnedT(cmd_id_t.cloneType).asInstanceOf[this.type] } - /*val request_returned = Flipped(Valid(new Bundle { - val bytes_read = UInt(log2Up(maxBytes+1).W) - val cmd_id = cmd_id_t - }))*/ - val request_returned = Flipped(Valid(new RequestReturnedT(cmd_id_t.cloneType))) class CmdCompletedT(cmd_id_t: UInt, tag_t: T) extends Bundle { @@ -57,11 +46,6 @@ class DMAReadCommandTracker[T <: Data](val nCmds: Int, val maxBytes: Int, tag_t: override def cloneType: this.type = new CmdCompletedT(cmd_id_t.cloneType, tag_t.cloneType).asInstanceOf[this.type] } - /*val cmd_completed = Decoupled(new Bundle { - val cmd_id = cmd_id_t - val tag = tag_t - })*/ - val cmd_completed = Decoupled(new CmdCompletedT(cmd_id_t.cloneType, tag_t.cloneType)) val busy = Output(Bool()) diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index cc8d3b04f..4f7994ddc 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -513,7 +513,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In in_shift := rs2s(0)(31, 0) // TODO magic number acc_scale := rs1s(0)(xLen - 1, 32).asTypeOf(acc_scale_args.multiplicand_t) // TODO magic number relu6_shift := rs2s(0)(xLen - 1, 32) // TODO magic number - a_addr_stride := rs1s(0)(31, 16) // TODO magic number + a_addr_stride := rs1s(0)(31, 16) // TODO magic number // TODO this needs to be kept in sync with ROB.scala a_transpose := rs1s(0)(8) bd_transpose := rs1s(0)(9) diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 9a40853e3..889928257 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -82,13 +82,14 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( val acc_scale_t_bits = acc_scale_t.getWidth - // val max_in_flight_reqs = 16 // TODO calculate this somehow - - val mvin_len_bits = log2Up(((dma_maxbytes / (inputType.getWidth / 8)) max (meshColumns * tileColumns)) + 1) - val mvin_rows_bits = 16 // log2Up(meshRows * tileRows + 1) - val mvout_len_bits = log2Up(meshColumns * tileColumns + 1) + val mvin_cols_bits = log2Up(((dma_maxbytes / (inputType.getWidth / 8)) max (meshColumns * tileColumns)) + 1) + val mvin_rows_bits = log2Up(meshRows * tileRows + 1) + val mvout_cols_bits = log2Up(meshColumns * tileColumns + 1) val mvout_rows_bits = log2Up(meshRows * tileRows + 1) + val load_states = 3 + val block_stride_bits = 16 + //========================================================================== // sanity check mesh size //========================================================================== @@ -365,7 +366,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( header ++= s"#define ACC_READ_FULL_WIDTH\n" header ++= s"\n" - header ++= s"#endif // $guard" + header ++= s"#endif // $guard\n" header.toString() } diff --git a/src/main/scala/gemmini/GemminiISA.scala b/src/main/scala/gemmini/GemminiISA.scala index 91b30a219..b49087c7f 100644 --- a/src/main/scala/gemmini/GemminiISA.scala +++ b/src/main/scala/gemmini/GemminiISA.scala @@ -22,6 +22,15 @@ object GemminiISA { val LOAD3_CMD = 14.U + // TODO add orows and ocols to this as well + val LOOP_CONV_WS = 15.U // no_bias, no_pool + val LOOP_CONV_WS_CONFIG_1 = 16.U // batch_size, in_dim, in_channels, out_channels | out_dim, pool_out_dim, stride, padding + val LOOP_CONV_WS_CONFIG_2 = 17.U // kernel_dim, pool_size, pool_stride, pool_padding | batches, porows, pocols, pochs + val LOOP_CONV_WS_CONFIG_3 = 18.U // krows, kcols, kchs, lpad | rpad, upad, dpad, plpad + val LOOP_CONV_WS_CONFIG_4 = 19.U // prad, pupad, pdpad, orows | ocols + val LOOP_CONV_WS_CONFIG_5 = 20.U // *weights | *output + val LOOP_CONV_WS_CONFIG_6 = 21.U // *bias, *input + // rs1[2:0] values val CONFIG_EX = 0.U val CONFIG_LOAD = 1.U diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index cf5f0c577..65146b2be 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -6,6 +6,7 @@ import GemminiISA._ import Util._ import freechips.rocketchip.config.Parameters +// TODO we need to check for WAW errors here // TODO deal with errors when reading scratchpad responses class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V], coreMaxAddrBits: Int, local_addr_t: LocalAddr) (implicit p: Parameters) extends Module { @@ -24,9 +25,10 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val waiting_for_command :: waiting_for_dma_req_ready :: sending_rows :: Nil = Enum(3) val control_state = RegInit(waiting_for_command) - val strides = Reg(Vec(3, UInt(coreMaxAddrBits.W))) - val scales = Reg(Vec(3, UInt(mvin_scale_t_bits.W))) - val shrinks = Reg(Vec(3, Bool())) // Shrink inputs to accumulator + val strides = Reg(Vec(load_states, UInt(coreMaxAddrBits.W))) + val scales = Reg(Vec(load_states, UInt(mvin_scale_t_bits.W))) + val shrinks = Reg(Vec(load_states, Bool())) // Shrink inputs to accumulator + val block_strides = Reg(Vec(load_states, UInt(block_stride_bits.W))) // Spad stride during block move-ins val block_rows = meshRows * tileRows val block_cols = meshColumns * tileColumns val row_counter = RegInit(0.U(log2Ceil(block_rows).W)) @@ -34,22 +36,26 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val cmd = Queue(io.cmd, ld_queue_length) val vaddr = cmd.bits.cmd.rs1 val localaddr = cmd.bits.cmd.rs2.asTypeOf(local_addr_t) - val cols = cmd.bits.cmd.rs2(32 + mvin_len_bits - 1, 32) // TODO magic numbers + val cols = cmd.bits.cmd.rs2(32 + mvin_cols_bits - 1, 32) // TODO magic numbers val rows = cmd.bits.cmd.rs2(48 + mvin_rows_bits - 1, 48) // TODO magic numbers val config_stride = cmd.bits.cmd.rs2 val config_scale = cmd.bits.cmd.rs1(32 + mvin_scale_t_bits - 1, 32) // TODO magic numbers - val config_shrink = cmd.bits.cmd.rs1(2) + val config_shrink = cmd.bits.cmd.rs1(2) // TODO magic numbers + val config_block_stride = cmd.bits.cmd.rs1(31, 16) // TODO magic numbers val mstatus = cmd.bits.cmd.status val load_state_id = MuxCase(0.U, Seq((cmd.bits.cmd.inst.funct === LOAD2_CMD) -> 1.U, (cmd.bits.cmd.inst.funct === LOAD3_CMD) -> 2.U)) - val config_state_id = cmd.bits.cmd.rs1(4,3) + val config_state_id = cmd.bits.cmd.rs1(4,3) // TODO magic numbers val state_id = Mux(cmd.bits.cmd.inst.funct === CONFIG_CMD, config_state_id, load_state_id) val stride = strides(state_id) val scale = scales(state_id) val shrink = shrinks(state_id) + val block_stride = block_strides(state_id) + + val all_zeros = vaddr === 0.U val localaddr_plus_row_counter = localaddr + row_counter @@ -81,10 +87,12 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig (control_state === sending_rows && row_counter =/= 0.U) io.dma.req.bits.vaddr := vaddr + row_counter * stride io.dma.req.bits.laddr := localaddr_plus_row_counter - io.dma.req.bits.len := cols - io.dma.req.bits.repeats := Mux(stride === 0.U, rows - 1.U, 0.U) + io.dma.req.bits.cols := cols + io.dma.req.bits.repeats := Mux(stride === 0.U && !all_zeros, rows - 1.U, 0.U) + io.dma.req.bits.block_stride := block_stride io.dma.req.bits.scale := scale io.dma.req.bits.has_acc_bitwidth := localaddr_plus_row_counter.is_acc_addr && !shrink + io.dma.req.bits.all_zeros := all_zeros io.dma.req.bits.status := mstatus // Command tracker IO @@ -109,6 +117,8 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig // Row counter when (io.dma.req.fire()) { row_counter := wrappingAdd(row_counter, 1.U, actual_rows_read) + + assert(block_stride >= rows) } // Control logic @@ -120,6 +130,7 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig stride := config_stride scale := config_scale shrink := config_shrink + block_stride := config_block_stride cmd.ready := true.B } diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala new file mode 100644 index 000000000..0ec656886 --- /dev/null +++ b/src/main/scala/gemmini/LoopConv.scala @@ -0,0 +1,1026 @@ +package gemmini + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import freechips.rocketchip.tile.RoCCCommand +import freechips.rocketchip.config.Parameters +import GemminiISA._ +import Util._ + +class LoopConvOuterBounds(val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int) extends Bundle { + val batch_size = UInt(large_iterator_bitwidth.W) + val in_dim = UInt(small_iterator_bitwidth.W) + val in_channels = UInt(large_iterator_bitwidth.W) + val out_channels = UInt(large_iterator_bitwidth.W) + val out_dim = UInt(small_iterator_bitwidth.W) + val pool_out_dim = UInt(small_iterator_bitwidth.W) + val stride = UInt(tiny_iterator_bitwidth.W) + val padding = UInt(tiny_iterator_bitwidth.W) + val kernel_dim = UInt(tiny_iterator_bitwidth.W) + val pool_size = UInt(tiny_iterator_bitwidth.W) + val pool_stride = UInt(tiny_iterator_bitwidth.W) + val pool_padding = UInt(tiny_iterator_bitwidth.W) +} + +class LoopConvInnerBounds(val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int) extends Bundle { + val batches = UInt(large_iterator_bitwidth.W) + val porows = UInt(small_iterator_bitwidth.W) + val pocols = UInt(small_iterator_bitwidth.W) + val pochs = UInt(large_iterator_bitwidth.W) + val krows = UInt(tiny_iterator_bitwidth.W) + val kcols = UInt(tiny_iterator_bitwidth.W) + val kchs = UInt(large_iterator_bitwidth.W) + val lpad = UInt(tiny_iterator_bitwidth.W) + val rpad = UInt(tiny_iterator_bitwidth.W) + val upad = UInt(tiny_iterator_bitwidth.W) + val dpad = UInt(tiny_iterator_bitwidth.W) + val plpad = UInt(tiny_iterator_bitwidth.W) + val prad = UInt(tiny_iterator_bitwidth.W) + val pupad = UInt(tiny_iterator_bitwidth.W) + val pdpad = UInt(tiny_iterator_bitwidth.W) + val orows = UInt(small_iterator_bitwidth.W) + val ocols = UInt(small_iterator_bitwidth.W) +} + +class LoopConvDerivedParams(val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int) extends Bundle { + val ochs = UInt(large_iterator_bitwidth.W) + + val irows = UInt(small_iterator_bitwidth.W) + val icols = UInt(small_iterator_bitwidth.W) + val irows_unpadded = UInt(small_iterator_bitwidth.W) + val icols_unpadded = UInt(small_iterator_bitwidth.W) + val ichs = UInt(large_iterator_bitwidth.W) + + val out_channels_per_bank = UInt(small_iterator_bitwidth.W) // TODO this won't work for systolic arrays above 256 in size + + val bias_spad_stride = UInt(large_iterator_bitwidth.W) + val input_spad_stride = UInt(large_iterator_bitwidth.W) + val weight_spad_stride = UInt(large_iterator_bitwidth.W) + + val ex_overwrite = Bool() +} + +class LoopConvLdBiasReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val max_acc_addr: Int, val concurrent_loops: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val addr_start = UInt(log2Up(max_acc_addr).W) + val dram_addr = UInt(coreMaxAddrBits.W) + val no_bias = Bool() + val loop_id = UInt(log2Up(concurrent_loops).W) +} + +class LoopConvLdBias(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, acc_w: Int, + max_block_len_acc: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { + val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new LoopConvLdBiasReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth: Int, max_acc_addr, concurrent_loops))) + val cmd = Decoupled(Output(new RoCCCommand)) + + val idle = Output(Bool()) + val rob_overloaded = Input(Bool()) + val wait_for_prev_loop = Input(Bool()) + + val loop_id = Output(UInt(log2Up(concurrent_loops).W)) + }) + + object State extends ChiselEnum { + val idle, config, ld = Value + } + import State._ + val state = RegInit(idle) + + val req = Reg(new LoopConvLdBiasReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth: Int, max_acc_addr, concurrent_loops)) + import req.inner_bounds._ + import req.derived_params._ + + val acc_addr_start = (BigInt(1) << 31).U | req.addr_start + + // Derived parameters + val max_ochs_per_mvin = Mux(ochs < (max_block_len_acc * block_size).U, ochs, (max_block_len_acc * block_size).U) + + val skip = req.no_bias || (req.dram_addr === 0.U) + + // Iterators + val b = Reg(UInt(large_iterator_bitwidth.W)) + val orow = Reg(UInt(small_iterator_bitwidth.W)) + val ocol = Reg(UInt(small_iterator_bitwidth.W)) + val och = Reg(UInt(large_iterator_bitwidth.W)) + + // Addresses + val dram_addr = req.dram_addr +& och * (acc_w/8).U + val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol + + // Sizes + val I = Mux(ocols - ocol > block_size.U, block_size.U, ocols - ocol) + val J = Mux(ochs - och > max_ochs_per_mvin, max_ochs_per_mvin, ochs - och) + + // Commands + val config_cmd = Wire(new RoCCCommand) + config_cmd := DontCare + config_cmd.inst.funct := CONFIG_CMD + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.bias_spad_stride << 16.U) | (2.U << 3) | 1.U + config_cmd.rs2 := 0.U + + val mvin_cmd = Wire(new RoCCCommand) + mvin_cmd := DontCare + mvin_cmd.inst.funct := LOAD3_CMD + mvin_cmd.rs1 := dram_addr + mvin_cmd.rs2 := (I << 48.U) | (J << 32.U) | spad_addr + + // Inputs and outputs + io.req.ready := state === idle + io.idle := state === idle + io.loop_id := req.loop_id + + io.cmd.valid := state =/= idle && !io.rob_overloaded && !io.wait_for_prev_loop && !skip + io.cmd.bits := Mux(state === config, config_cmd, mvin_cmd) + + // Sending outputs + when (skip) { + state := idle + }.elsewhen(io.cmd.fire()) { + when (state === config) { + state := ld + }.otherwise { + val next_och = floorAdd(och, max_ochs_per_mvin, ochs) + val next_ocol = floorAdd(ocol, block_size.U, ocols, next_och === 0.U) + val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U && next_och === 0.U) + val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && next_ocol === 0.U && next_och === 0.U) + + och := next_och + ocol := next_ocol + orow := next_orow + b := next_b + + state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U, + idle, ld) + } + } + + // Accepting requests + when (io.req.fire()) { + req := io.req.bits + state := config + b := 0.U + orow := 0.U + ocol := 0.U + och := 0.U + } +} + +class LoopConvLdInputReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val max_acc_addr: Int, val concurrent_loops: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val addr_start = UInt(log2Up(max_acc_addr).W) + val dram_addr = UInt(coreMaxAddrBits.W) + val loop_id = UInt(log2Up(concurrent_loops).W) +} + +class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, + max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { + val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new LoopConvLdInputReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, concurrent_loops))) + val cmd = Decoupled(Output(new RoCCCommand)) + + val idle = Output(Bool()) + val rob_overloaded = Input(Bool()) + val wait_for_prev_loop = Input(Bool()) + + val loop_id = Output(UInt(log2Up(concurrent_loops).W)) + }) + + object State extends ChiselEnum { + val idle, config, ld = Value + } + import State._ + val state = RegInit(idle) + + val req = Reg(new LoopConvLdInputReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, concurrent_loops)) + import req.outer_bounds._ + import req.inner_bounds._ + import req.derived_params._ + + // Derived parameters + val max_ichs_per_mvin = Mux(ichs < (max_block_len * block_size).U, ichs, (max_block_len * block_size).U).zext() + + // Iterators + val b = Reg(SInt(large_iterator_bitwidth.W)) + val irow = Reg(SInt(small_iterator_bitwidth.W)) + val icol = Reg(SInt(small_iterator_bitwidth.W)) + val ich = Reg(SInt(large_iterator_bitwidth.W)) + + // Calculated params + val irow_padded = irow +& upad.zext() + val icol_padded = icol +& lpad.zext() + val is_zeros = irow < 0.S || irow >= irows_unpadded.zext() || icol < 0.S || icol >= icols_unpadded.zext() + + // Addresses + val dram_addr = Mux(is_zeros, 0.U, + req.dram_addr +& (((b * in_dim * in_dim +& irow*in_dim +& icol) * in_channels +& ich) * (input_w/8).U).asUInt()) + val spad_addr = req.addr_start.zext() +& (ich / block_size.S) * batches * irows * icols +& b * irows * icols +& irow_padded * icols +& icol_padded + + // Sizes + val I = MuxCase( + Mux(icols_unpadded.zext() -& icol > block_size.S, block_size.S, icols_unpadded.zext() -& icol), + Seq( + (icol < 0.S) -> Mux((0.S-&icol) > block_size.S, block_size.S, 0.S-&icol), + (icol >= icols_unpadded.zext()) -> Mux(icols_unpadded.zext() +& rpad.zext() -& icol > block_size.S, block_size.S, icols_unpadded.zext() +& rpad.zext() -& icol) + ) + ) + val K = Mux(ochs.zext() -& ich > max_ichs_per_mvin, max_ichs_per_mvin, ochs.zext() -& ich) + + // Commands + val config_cmd = Wire(new RoCCCommand) + config_cmd := DontCare + config_cmd.inst.funct := CONFIG_CMD + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.input_spad_stride << 16.U) | (0.U << 3) | 1.U + config_cmd.rs2 := in_channels * (input_w/8).U + + val mvin_cmd = Wire(new RoCCCommand) + mvin_cmd := DontCare + mvin_cmd.inst.funct := LOAD_CMD + mvin_cmd.rs1 := dram_addr + mvin_cmd.rs2 := (I << 48.U).asUInt() | (K << 32.U).asUInt() | spad_addr.asUInt() + + // Inputs and outputs + io.req.ready := state === idle + io.idle := state === idle + io.loop_id := req.loop_id + + io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded + io.cmd.bits := Mux(state === config, config_cmd, mvin_cmd) + + // Sending outputs + when(io.cmd.fire()) { + when (state === config) { + state := ld + }.otherwise { + val next_ich = sFloorAdd(ich, max_ichs_per_mvin.asUInt(), ichs.zext(), 0.S) + val next_icol = sFloorAdd(icol, I.asUInt(), (icols_unpadded +& rpad).zext(), 0.S-&lpad.zext(), + next_ich === 0.S) + val next_irow = sFloorAdd(irow, 1.U, (irows_unpadded +& dpad).zext(), 0.S-&upad.zext(), + next_icol === 0.S-&lpad.zext() && next_ich === 0.S) + val next_b = sFloorAdd(b, 1.U, batches.zext(), 0.S, + next_irow === 0.S-&upad.zext() && next_icol === 0.S-&lpad.zext() && next_ich === 0.S) + + ich := next_ich + icol := next_icol + irow := next_irow + b := next_b + + state := Mux(next_b === 0.S && next_irow === 0.S-&upad.zext() && next_icol === 0.S-&lpad.zext() && next_ich === 0.S, + idle, ld) + } + } + + // Accepting requests + when (io.req.fire()) { + req := io.req.bits + state := config + b := 0.S + irow := 0.S -& io.req.bits.inner_bounds.upad.zext() + icol := 0.S -& io.req.bits.inner_bounds.lpad.zext() + ich := 0.S + } +} + +class LoopConvLdWeightReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val max_addr: Int, val concurrent_loops: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val addr_end = UInt(log2Up(max_addr).W) + val dram_addr = UInt(coreMaxAddrBits.W) + val loop_id = UInt(log2Up(concurrent_loops).W) +} + +class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, + max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { + val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new LoopConvLdWeightReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, concurrent_loops))) + val cmd = Decoupled(Output(new RoCCCommand)) + + val idle = Output(Bool()) + val rob_overloaded = Input(Bool()) + val wait_for_prev_loop = Input(Bool()) + + val loop_id = Output(UInt(log2Up(concurrent_loops).W)) + }) + + object State extends ChiselEnum { + val idle, config, ld = Value + } + import State._ + val state = RegInit(idle) + + val req = Reg(new LoopConvLdWeightReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, concurrent_loops)) + import req.outer_bounds._ + import req.inner_bounds._ + import req.derived_params._ + + // Derived parameters + val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) + val B_rows = out_channels_per_bank * kcols * krows * kchs + val addr_start = req.addr_end - B_rows + + // Iterators + val och = Reg(UInt(large_iterator_bitwidth.W)) + val krow = Reg(UInt(tiny_iterator_bitwidth.W)) + val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) + val kch = Reg(UInt(large_iterator_bitwidth.W)) + + // Addresses + val dram_addr = req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channels +& och) * (input_w/8).U + val spad_addr = addr_start + (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch + + // Sizes + val J = Mux(ochs - och > max_ochs_per_mvin, max_ochs_per_mvin, ochs - och) + val K = Mux(kchs - kch > block_size.U, block_size.U, kchs - kch) + + // Commands + val config_cmd = Wire(new RoCCCommand) + config_cmd := DontCare + config_cmd.inst.funct := CONFIG_CMD + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.weight_spad_stride << 16.U) | (1.U << 3) | 1.U + config_cmd.rs2 := out_channels * (input_w/8).U + + val mvin_cmd = Wire(new RoCCCommand) + mvin_cmd := DontCare + mvin_cmd.inst.funct := LOAD2_CMD + mvin_cmd.rs1 := dram_addr + mvin_cmd.rs2 := (K << 48.U) | (J << 32.U) | spad_addr + + // Inputs and outputs + io.req.ready := state === idle + io.idle := state === idle + io.loop_id := req.loop_id + + io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded + io.cmd.bits := Mux(state === config, config_cmd, mvin_cmd) + + // Sending outputs + when(io.cmd.fire()) { + when (state === config) { + state := ld + }.otherwise { + val next_kch = floorAdd(kch, block_size.U, kchs) + val next_kcol = floorAdd(kcol, 1.U, kcols, next_kch === 0.U) + val next_krow = floorAdd(krow, 1.U, krows, next_kcol === 0.U && next_kch === 0.U) + val next_och = floorAdd(och, max_ochs_per_mvin, ochs, next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + + kch := next_kch + kcol := next_kcol + krow := next_krow + och := next_och + + state := Mux(next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U, + idle, ld) + } + } + + // Accepting requests + when (io.req.fire()) { + req := io.req.bits + state := config + kch := 0.U + kcol := 0.U + krow := 0.U + och := 0.U + } +} + +class LoopConvExecuteReq(val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val max_addr: Int, val max_acc_addr: Int, val concurrent_loops: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val a_addr_start = UInt(log2Up(max_addr).W) + val b_addr_end = UInt(log2Up(max_addr).W) + val c_addr_start = UInt(log2Up(max_acc_addr).W) + val loop_id = UInt(log2Up(concurrent_loops).W) +} + +class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, + max_acc_addr: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { + val GARBAGE_ADDR = (~0.U(32.W)).asUInt() + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new LoopConvExecuteReq(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, max_acc_addr, concurrent_loops))) + val cmd = Decoupled(Output(new RoCCCommand)) + + val lda_completed = Input(Bool()) + val ldb_completed = Input(Bool()) + val ldd_completed = Input(Bool()) + + val idle = Output(Bool()) + val rob_overloaded = Input(Bool()) + + val loop_id = Output(UInt(log2Up(concurrent_loops).W)) + }) + + object State extends ChiselEnum { + val idle, pre, comp = Value + } + import State._ + val state = RegInit(idle) + + val req = Reg(new LoopConvExecuteReq(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, + max_addr, max_acc_addr, concurrent_loops)) + import req.outer_bounds._ + import req.inner_bounds._ + import req.derived_params._ + + // Derived parameters + val B_rows = out_channels_per_bank * kcols * krows * kchs + + val a_addr_start = req.a_addr_start + val b_addr_start = req.b_addr_end - B_rows + val d_addr_start = (BigInt(1) << 31).U | req.c_addr_start + val c_addr_start = (BigInt(3) << 30).U | req.c_addr_start + + // Iterators + val b = Reg(UInt(large_iterator_bitwidth.W)) + val orow = Reg(UInt(small_iterator_bitwidth.W)) + val ocol = Reg(UInt(small_iterator_bitwidth.W)) + val och = Reg(UInt(large_iterator_bitwidth.W)) + val krow = Reg(UInt(tiny_iterator_bitwidth.W)) + val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) + val kch = Reg(UInt(large_iterator_bitwidth.W)) + + val irow = orow * stride +& krow + val icol = ocol * stride +& kcol + + val I = Mux(ocols - ocol > block_size.U, block_size.U, ocols - ocol) + val J = Mux(ochs - och > block_size.U, block_size.U, ochs - och) + val K = Mux(kchs - kch > block_size.U, block_size.U, kchs - kch) + + // Addresses + val a_addr = a_addr_start +& (kch / block_size.U) * batches * irows * icols +& b * irows * icols +& irow * icols +& icol + val b_addr = b_addr_start +& (och / block_size.U) * krows * kcols * kchs +& krow * kcols * kchs +& kcol * kchs +& kch + val c_addr = Mux(ex_overwrite && krow === 0.U && kcol === 0.U && kch === 0.U, d_addr_start, c_addr_start) +& + (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol + + // Commands + val pre_cmd = Wire(new RoCCCommand) + pre_cmd := DontCare + pre_cmd.inst.funct := PRELOAD_CMD + pre_cmd.rs1 := (K << 48) | (J << 32) | b_addr + pre_cmd.rs2 := (I << 48) | (J << 32) | c_addr + + val comp_cmd = Wire(new RoCCCommand()) + comp_cmd := DontCare + comp_cmd.inst.funct := COMPUTE_AND_FLIP_CMD + comp_cmd.rs1 := (I << 48) | (K << 32) | a_addr + comp_cmd.rs2 := (I << 48) | (J << 32) | GARBAGE_ADDR + + // Inputs and outputs + io.req.ready := state === idle + io.idle := state === idle + + val ld_ahead = io.lda_completed && io.ldb_completed && io.ldd_completed + + io.cmd.valid := state =/= idle && !io.rob_overloaded && ld_ahead + io.cmd.bits := Mux(state === pre, pre_cmd, comp_cmd) + + io.loop_id := req.loop_id + + // Sending outputs + when (io.cmd.fire()) { + when (state === pre) { + state := comp + }.otherwise { + val next_kch = floorAdd(kch, block_size.U, kchs) + val next_kcol = floorAdd(kcol, 1.U, kcols, next_kch === 0.U) + val next_krow = floorAdd(krow, 1.U, krows, next_kcol === 0.U && next_kch === 0.U) + val next_och = floorAdd(och, block_size.U, ochs, + next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + val next_ocol = floorAdd(ocol, block_size.U, ocols, + next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + val next_orow = floorAdd(orow, 1.U, orows, + next_ocol === 0.U && next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && + next_ocol === 0.U && next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + + kch := next_kch + kcol := next_kcol + krow := next_krow + och := next_och + ocol := next_ocol + orow := next_orow + b := next_b + + state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && + next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U, + idle, pre) + } + } + + // Accepting requests + when (io.req.fire()) { + req := io.req.bits + state := pre + + b := 0.U + orow := 0.U + ocol := 0.U + och := 0.U + krow := 0.U + kcol := 0.U + kch := 0.U + } +} + +class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val max_acc_addr: Int, val concurrent_loops: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val addr_start = UInt(log2Up(max_acc_addr).W) + val dram_addr = UInt(coreMaxAddrBits.W) + val no_pool = Bool() + val loop_id = UInt(log2Up(concurrent_loops).W) +} + +class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { + val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new LoopConvStReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth: Int, max_acc_addr, concurrent_loops))) + val cmd = Decoupled(Output(new RoCCCommand)) + + val ex_completed = Input(Bool()) + + val idle = Output(Bool()) + val rob_overloaded = Input(Bool()) + + val loop_id = Output(UInt(log2Up(concurrent_loops).W)) + }) + + object State extends ChiselEnum { + val idle, st = Value + } + import State._ + val state = RegInit(idle) + + val req = Reg(new LoopConvStReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth: Int, max_acc_addr, concurrent_loops)) + import req.outer_bounds._ + import req.inner_bounds._ + import req.derived_params._ + + val acc_addr_start = (BigInt(1) << 31).U | req.addr_start + + // Derived parameters + val skip = !(req.no_pool && (req.dram_addr =/= 0.U)) + + // Iterators + val b = Reg(UInt(large_iterator_bitwidth.W)) + val orow = Reg(UInt(small_iterator_bitwidth.W)) + val ocol = Reg(UInt(small_iterator_bitwidth.W)) + val och = Reg(UInt(large_iterator_bitwidth.W)) + + // Addresses + val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * out_channels + och) * (input_w/8).U + val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol + + // Sizes + val I = Mux(ocols - ocol > block_size.U, block_size.U, ocols - ocol) + val J = Mux(ochs - och > block_size.U, block_size.U, ochs - och) + + // Commands + val mvout_cmd = Wire(new RoCCCommand) + mvout_cmd := DontCare + mvout_cmd.inst.funct := STORE_CMD + mvout_cmd.rs1 := dram_addr + mvout_cmd.rs2 := (I << 48.U) | (J << 32.U) | spad_addr + + // Inputs and outputs + io.req.ready := state === idle + io.idle := state === idle + io.loop_id := req.loop_id + + io.cmd.valid := state =/= idle && !io.rob_overloaded && !skip && io.ex_completed + io.cmd.bits := mvout_cmd + + // Sending outputs + when (skip) { + state := idle + }.elsewhen(io.cmd.fire()) { + val next_och = floorAdd(och, block_size.U, ochs) + val next_ocol = floorAdd(ocol, block_size.U, ocols, next_och === 0.U) + val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U && next_och === 0.U) + val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && next_ocol === 0.U && next_och === 0.U) + + och := next_och + ocol := next_ocol + orow := next_orow + b := next_b + + state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U, + idle, st) + } + + // Accepting requests + when (io.req.fire()) { + req := io.req.bits + state := st + + b := 0.U + orow := 0.U + ocol := 0.U + och := 0.U + } +} + +class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val small_iterator_bitwidth: Int, val tiny_iterator_bitwidth: Int, val coreMaxAddrBits: Int, val max_addr: Int, val max_acc_addr: Int) extends Bundle { + val outer_bounds = new LoopConvOuterBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + val inner_bounds = new LoopConvInnerBounds(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) + + val bias_dram_addr = UInt(coreMaxAddrBits.W) + val weights_dram_addr = UInt(coreMaxAddrBits.W) + val input_dram_addr = UInt(coreMaxAddrBits.W) + val output_dram_addr = UInt(coreMaxAddrBits.W) + + val no_bias = Bool() + val no_pool = Bool() + + val configured = Bool() + + val running = Bool() + + val ld_bias_started = Bool() + val ld_input_started = Bool() + val ld_weights_started = Bool() + val ex_started = Bool() + val st_started = Bool() + + val ld_bias_completed = Bool() + val ld_input_completed = Bool() + val ld_weights_completed = Bool() + val ex_completed = Bool() + val st_completed = Bool() + + def all_completed(dummy: Int=0): Bool = ld_bias_completed && ld_input_completed && ld_weights_completed && ex_completed && st_completed + + val a_addr_start = UInt(log2Up(max_addr).W) + val b_addr_end = UInt(log2Up(max_addr).W) + + def derived_params(dummy: Int=0): LoopConvDerivedParams = { + import outer_bounds.stride + import inner_bounds.{batches, pochs, orows, ocols, krows, kcols, upad, dpad, lpad, rpad, kchs} + + val result = Wire(new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth)) + + result.ochs := pochs + + result.irows := orows * stride +& krows - 1.U + result.icols := ocols * stride +& kcols - 1.U + result.irows_unpadded := result.irows - upad - dpad + result.icols_unpadded := result.icols - lpad - rpad + result.ichs := kchs + + result.out_channels_per_bank := result.ochs / block_size.U +& (result.ochs % block_size.U =/= 0.U) + + result.bias_spad_stride := batches * orows * ocols + result.input_spad_stride := batches * result.irows * result.icols + result.weight_spad_stride := krows * kcols * kchs + + result.ex_overwrite := bias_dram_addr =/= 0.U && no_bias + + result + } + + def reset(): Unit = { + configured := false.B + + running := false.B + + ld_bias_started := false.B + ld_input_started := false.B + ld_weights_started := false.B + ex_started := false.B + st_started := false.B + + ld_bias_completed := false.B + ld_input_completed := false.B + ld_weights_completed := false.B + ex_completed := false.B + st_completed := false.B + } +} + +class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: Int, max_exs: Int, max_sts: Int, + max_addr: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, dma_max_bytes: Int) + (implicit p: Parameters) extends Module { + val large_iterator_bitwidth = 16 + val small_iterator_bitwidth = 8 + val tiny_iterator_bitwidth = 4 + + val max_block_len = (dma_max_bytes / (block_size * (input_w / 8))) max 1 + val max_block_len_acc = (dma_max_bytes / (block_size * (acc_w / 8))) max 1 + + val io = IO(new Bundle { + val in = Flipped(Decoupled(new RoCCCommand)) + val out = Decoupled(new RoCCCommand) + val ld_utilization = Input(UInt(log2Up(rob_size).W)) + val st_utilization = Input(UInt(log2Up(rob_size).W)) + val ex_utilization = Input(UInt(log2Up(rob_size).W)) + val busy = Output(Bool()) + }) + + // Create states + val concurrent_loops = 2 + val loops = Reg(Vec(concurrent_loops, new LoopConvState(block_size, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, coreMaxAddrBits, max_addr, max_acc_addr))) + // val head_loop_id = Reg(UInt(log2Up(concurrent_loops).W)) + val head_loop_id = RegInit(0.U(log2Up(concurrent_loops).W)) + val tail_loop_id = (~head_loop_id).asUInt() // This is the loop that we always try to configure if available + val head_loop = loops(head_loop_id) + val tail_loop = loops(tail_loop_id) + + val loop_configured = loops.map(_.configured).reduce(_ || _) + + val loop_being_configured_id = Mux(head_loop.configured, tail_loop_id, head_loop_id) + val loop_being_configured = loops(loop_being_configured_id) + + // Create inner modules + val ld_bias = Module(new LoopConvLdBias(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, acc_w, max_block_len_acc, concurrent_loops)) + val ld_input = Module(new LoopConvLdInput(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) + val ld_weights = Module(new LoopConvLdWeight(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) + val ex = Module(new LoopConvExecute(block_size, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, max_acc_addr, concurrent_loops)) + val st = Module(new LoopConvSt(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, input_w, concurrent_loops)) + + // Create command queue + val cmd = Queue(io.in) + + io.busy := cmd.valid || loop_configured + + // Create arbiter + val arb = Module(new Arbiter(new RoCCCommand, 5)) + arb.io.in(0) <> st.io.cmd + arb.io.in(1) <> ex.io.cmd + arb.io.in(2) <> ld_bias.io.cmd + arb.io.in(3) <> ld_weights.io.cmd + arb.io.in(4) <> ld_input.io.cmd + val unrolled_cmd = arb.io.out + + // Wire up unrolled command output + val is_loop_run_cmd = cmd.bits.inst.funct === LOOP_CONV_WS + val is_loop_config_cmd = cmd.bits.inst.funct >= LOOP_CONV_WS_CONFIG_1 && cmd.bits.inst.funct <= LOOP_CONV_WS_CONFIG_6 + val is_loop_cmd = is_loop_run_cmd || is_loop_config_cmd + + io.out.bits := Mux(loop_configured, unrolled_cmd.bits, cmd.bits) + io.out.bits.status := cmd.bits.status // TODO This is not guaranteed to be the correct fix! We must fix this + io.out.valid := Mux(loop_configured, unrolled_cmd.valid, cmd.valid && !is_loop_config_cmd && !is_loop_run_cmd) + + cmd.ready := Mux(is_loop_cmd, !loop_being_configured.configured, !loop_configured && io.out.ready) + arb.io.out.ready := io.out.ready + + // Wire up waiting-for-loads signals + val ex_is_waiting_for_loads = loops(ex.io.loop_id).ex_started && !loops(ex.io.loop_id).ex_completed && + !(loops(ex.io.loop_id).ld_input_completed && loops(ex.io.loop_id).ld_weights_completed && + loops(ex.io.loop_id).ld_bias_completed) + + ld_bias.io.wait_for_prev_loop := ex_is_waiting_for_loads && ld_bias.io.loop_id =/= ex.io.loop_id + ld_weights.io.wait_for_prev_loop := ex_is_waiting_for_loads && ld_weights.io.loop_id =/= ex.io.loop_id + ld_input.io.wait_for_prev_loop := ex_is_waiting_for_loads && ld_input.io.loop_id =/= ex.io.loop_id + + // Wire up overloaded signals + ld_bias.io.rob_overloaded := io.ld_utilization >= max_lds.U + ld_input.io.rob_overloaded := io.ld_utilization >= max_lds.U + ld_weights.io.rob_overloaded := io.ld_utilization >= max_lds.U + ex.io.rob_overloaded := io.ex_utilization >= max_exs.U + st.io.rob_overloaded := io.st_utilization >= max_sts.U + + // Wire up iterator inputs + ex.io.lda_completed := (ld_input.io.loop_id =/= ex.io.loop_id) || ld_input.io.idle + ex.io.ldb_completed := (ld_weights.io.loop_id =/= ex.io.loop_id) || ld_weights.io.idle + ex.io.ldd_completed := (ld_bias.io.loop_id =/= ex.io.loop_id) || ld_bias.io.idle + st.io.ex_completed := (ex.io.loop_id =/= st.io.loop_id) || ex.io.idle + + // Create config registers + when(cmd.valid && is_loop_cmd && !loop_being_configured.configured) { + + switch (cmd.bits.inst.funct) { + is (LOOP_CONV_WS_CONFIG_1) { + loop_being_configured.outer_bounds.out_channels := cmd.bits.rs1(63, 48) + loop_being_configured.outer_bounds.in_channels := cmd.bits.rs1(47, 32) + loop_being_configured.outer_bounds.in_dim := cmd.bits.rs1(31, 16) + loop_being_configured.outer_bounds.batch_size := cmd.bits.rs1(15, 0) + + loop_being_configured.outer_bounds.padding := cmd.bits.rs2(63, 48) + loop_being_configured.outer_bounds.stride := cmd.bits.rs2(47, 32) + loop_being_configured.outer_bounds.pool_out_dim := cmd.bits.rs2(31, 16) + loop_being_configured.outer_bounds.out_dim := cmd.bits.rs2(15, 0) + } + + is (LOOP_CONV_WS_CONFIG_2) { + loop_being_configured.outer_bounds.kernel_dim := cmd.bits.rs1(63, 48) + loop_being_configured.outer_bounds.pool_size := cmd.bits.rs1(47, 32) + loop_being_configured.outer_bounds.pool_stride := cmd.bits.rs1(31, 16) + loop_being_configured.outer_bounds.pool_padding := cmd.bits.rs1(15, 0) + + loop_being_configured.inner_bounds.batches := cmd.bits.rs2(63, 48) + loop_being_configured.inner_bounds.porows := cmd.bits.rs2(47, 32) + loop_being_configured.inner_bounds.pocols := cmd.bits.rs2(31, 16) + loop_being_configured.inner_bounds.pochs := cmd.bits.rs2(15, 0) + } + + is (LOOP_CONV_WS_CONFIG_3) { + loop_being_configured.inner_bounds.krows := cmd.bits.rs1(63, 48) + loop_being_configured.inner_bounds.kcols := cmd.bits.rs1(47, 32) + loop_being_configured.inner_bounds.kchs := cmd.bits.rs1(31, 16) + loop_being_configured.inner_bounds.lpad := cmd.bits.rs1(15, 0) + + loop_being_configured.inner_bounds.rpad := cmd.bits.rs2(63, 48) + loop_being_configured.inner_bounds.upad := cmd.bits.rs2(47, 32) + loop_being_configured.inner_bounds.dpad := cmd.bits.rs2(31, 16) + loop_being_configured.inner_bounds.plpad := cmd.bits.rs2(15, 0) + } + + is (LOOP_CONV_WS_CONFIG_4) { + loop_being_configured.inner_bounds.orows := cmd.bits.rs1(63, 48) + loop_being_configured.inner_bounds.prad := cmd.bits.rs1(47, 32) + loop_being_configured.inner_bounds.pupad := cmd.bits.rs1(31, 16) + loop_being_configured.inner_bounds.pdpad := cmd.bits.rs1(15, 0) + + loop_being_configured.inner_bounds.ocols := cmd.bits.rs2(15, 0) + } + + is (LOOP_CONV_WS_CONFIG_5) { + loop_being_configured.weights_dram_addr := cmd.bits.rs1 + + loop_being_configured.output_dram_addr := cmd.bits.rs2 + } + + is (LOOP_CONV_WS_CONFIG_6) { + loop_being_configured.bias_dram_addr := cmd.bits.rs1 + + loop_being_configured.input_dram_addr := cmd.bits.rs2 + } + + is (LOOP_CONV_WS) { + loop_being_configured.no_bias := cmd.bits.rs1(0) + + loop_being_configured.no_pool := cmd.bits.rs2(0) + + loop_being_configured.configured := true.B + } + } + } + + // Wire up request signals + val ld_bias_addr_start = RegInit(0.U(log2Up(max_acc_addr).W)) + val ex_c_addr_start = RegInit(0.U(log2Up(max_acc_addr).W)) + val st_addr_start = RegInit(0.U(log2Up(max_acc_addr).W)) + + val loop_requesting_ld_bias_id = Mux(head_loop.ld_bias_started, tail_loop_id, head_loop_id) + val loop_requesting_ld_bias = loops(loop_requesting_ld_bias_id) + ld_bias.io.req.bits.outer_bounds := loop_requesting_ld_bias.outer_bounds + ld_bias.io.req.bits.inner_bounds := loop_requesting_ld_bias.inner_bounds + ld_bias.io.req.bits.derived_params := loop_requesting_ld_bias.derived_params() + ld_bias.io.req.bits.addr_start := ld_bias_addr_start + ld_bias.io.req.bits.dram_addr := loop_requesting_ld_bias.bias_dram_addr + ld_bias.io.req.bits.no_bias := loop_requesting_ld_bias.no_bias + ld_bias.io.req.bits.loop_id := loop_requesting_ld_bias_id + + ld_bias.io.req.valid := !loop_requesting_ld_bias.ld_bias_started && loop_requesting_ld_bias.configured + + when (ld_bias.io.req.fire()) { + loop_requesting_ld_bias.running := true.B + loop_requesting_ld_bias.ld_bias_started := true.B + + // when (loop_requesting_ld_bias.bias_dram_addr =/= 0.U) { + when (loop_requesting_ld_bias.output_dram_addr =/= 0.U) { + ld_bias_addr_start := floorAdd(ld_bias_addr_start, (max_acc_addr / concurrent_loops).U, max_acc_addr.U) + } + } + + val loop_requesting_ld_input_id = Mux(head_loop.ld_input_started, tail_loop_id, head_loop_id) + val loop_requesting_ld_input = loops(loop_requesting_ld_input_id) + ld_input.io.req.bits.outer_bounds := loop_requesting_ld_input.outer_bounds + ld_input.io.req.bits.inner_bounds := loop_requesting_ld_input.inner_bounds + ld_input.io.req.bits.derived_params := loop_requesting_ld_input.derived_params() + ld_input.io.req.bits.addr_start := loop_requesting_ld_input.a_addr_start + ld_input.io.req.bits.dram_addr := loop_requesting_ld_input.input_dram_addr + ld_input.io.req.bits.loop_id := loop_requesting_ld_input_id + + ld_input.io.req.valid := !loop_requesting_ld_input.ld_input_started && loop_requesting_ld_input.configured + + when (ld_input.io.req.fire()) { + loop_requesting_ld_input.running := true.B + loop_requesting_ld_input.ld_input_started := true.B + } + + val loop_requesting_ld_weights_id = Mux(head_loop.ld_weights_started, tail_loop_id, head_loop_id) + val loop_requesting_ld_weights = loops(loop_requesting_ld_weights_id) + ld_weights.io.req.bits.outer_bounds := loop_requesting_ld_weights.outer_bounds + ld_weights.io.req.bits.inner_bounds := loop_requesting_ld_weights.inner_bounds + ld_weights.io.req.bits.derived_params := loop_requesting_ld_weights.derived_params() + ld_weights.io.req.bits.addr_end := loop_requesting_ld_weights.b_addr_end + ld_weights.io.req.bits.dram_addr := loop_requesting_ld_weights.weights_dram_addr + ld_weights.io.req.bits.loop_id := loop_requesting_ld_weights_id + + ld_weights.io.req.valid := !loop_requesting_ld_weights.ld_weights_started && loop_requesting_ld_weights.configured + + when (ld_weights.io.req.fire()) { + loop_requesting_ld_weights.running := true.B + loop_requesting_ld_weights.ld_weights_started := true.B + } + + val loop_requesting_ex_id = Mux(head_loop.ex_started, tail_loop_id, head_loop_id) + val loop_requesting_ex = loops(loop_requesting_ex_id) + ex.io.req.bits.outer_bounds := loop_requesting_ex.outer_bounds + ex.io.req.bits.inner_bounds := loop_requesting_ex.inner_bounds + ex.io.req.bits.derived_params := loop_requesting_ex.derived_params() + ex.io.req.bits.a_addr_start := loop_requesting_ex.a_addr_start + ex.io.req.bits.b_addr_end := loop_requesting_ex.b_addr_end + ex.io.req.bits.c_addr_start := ex_c_addr_start + ex.io.req.bits.loop_id := loop_requesting_ex_id + + ex.io.req.valid := !loop_requesting_ex.ex_started && loop_requesting_ex.ld_bias_started && + loop_requesting_ex.ld_input_started && loop_requesting_ex.ld_weights_started && loop_requesting_ex.configured + + when (ex.io.req.fire()) { + loop_requesting_ex.running := true.B + loop_requesting_ex.ex_started := true.B + + when (loop_requesting_ex.output_dram_addr =/= 0.U) { + ex_c_addr_start := floorAdd(ex_c_addr_start, (max_acc_addr / concurrent_loops).U, max_acc_addr.U) + } + } + + val loop_requesting_st_id = Mux(head_loop.st_started, tail_loop_id, head_loop_id) + val loop_requesting_st = loops(loop_requesting_st_id) + st.io.req.bits.outer_bounds := loop_requesting_st.outer_bounds + st.io.req.bits.inner_bounds := loop_requesting_st.inner_bounds + st.io.req.bits.derived_params := loop_requesting_st.derived_params() + st.io.req.bits.addr_start := st_addr_start + st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr + st.io.req.bits.no_pool := loop_requesting_st.no_pool + st.io.req.bits.loop_id := loop_requesting_st_id + + st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured + + when (st.io.req.fire()) { + loop_requesting_st.running := true.B + loop_requesting_st.st_started := true.B + + when (loop_requesting_st.output_dram_addr =/= 0.U) { + st_addr_start := floorAdd(st_addr_start, (max_acc_addr / concurrent_loops).U, max_acc_addr.U) + } + } + + // Handle completed signals + when (ld_bias.io.idle && loops(ld_bias.io.loop_id).running && loops(ld_bias.io.loop_id).ld_bias_started) { + loops(ld_bias.io.loop_id).ld_bias_completed := true.B + } + + when (ld_input.io.idle && loops(ld_input.io.loop_id).running && loops(ld_input.io.loop_id).ld_input_started) { + loops(ld_input.io.loop_id).ld_input_completed := true.B + } + + when (ld_weights.io.idle && loops(ld_weights.io.loop_id).running && loops(ld_weights.io.loop_id).ld_weights_started) { + loops(ld_weights.io.loop_id).ld_weights_completed := true.B + } + + when (ex.io.idle && loops(ex.io.loop_id).running && loops(ex.io.loop_id).ex_started) { + loops(ex.io.loop_id).ex_completed := true.B + } + + when (st.io.idle && loops(st.io.loop_id).running && loops(st.io.loop_id).st_started) { + loops(st.io.loop_id).st_completed := true.B + } + + when (head_loop.running && head_loop.all_completed()) { + head_loop.reset() + head_loop_id := ~head_loop_id + } + + // Resets + when (reset.toBool()) { + loops.zipWithIndex.foreach { case (l, i) => + l.reset() + l.a_addr_start := (i * (max_addr / concurrent_loops)).U + l.b_addr_end := ((i+1) * (max_addr / concurrent_loops) - block_size).U + } + } +} + +object LoopConv { + def apply(in: DecoupledIO[RoCCCommand], ld_utilization: UInt, st_utilization: UInt, ex_utilization: UInt, + block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: Int, max_exs: Int, max_sts: Int, + max_addr: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, dma_max_bytes: Int) + (implicit p: Parameters): Tuple2[DecoupledIO[RoCCCommand], Bool] = { + val mod = Module(new LoopConv(block_size, coreMaxAddrBits, rob_size, max_lds, max_exs, max_sts, + max_addr, max_acc_addr, input_w, acc_w, dma_max_bytes)) + mod.io.in <> in + mod.io.ld_utilization := ld_utilization + mod.io.st_utilization := st_utilization + mod.io.ex_utilization := ex_utilization + (mod.io.out, mod.io.busy) + } +} diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 16285d482..9b549c348 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -57,7 +57,6 @@ class LoopMatmulLdA(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val max_col_dim = Mux(req.transpose, req.max_i, req.max_k) val max_blocks = Mux(max_col_dim <= max_block_len.U, max_col_dim, max_block_len.U) - // TODO: why do we have to use 1 when transpose val sp_addr_start = req.addr_start @@ -85,13 +84,16 @@ class LoopMatmulLdA(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (io.cmd.fire()) { // The order here is k, j, i - val next_row_iterator = floorAdd(row_iterator, 1.U, max_row_iterator) - val next_col_iterator = floorAdd(col_iterator, max_blocks, max_col_iterator, next_row_iterator === 0.U) + val i_blocks = Mux(req.transpose, max_blocks, 1.U) + val k_blocks = Mux(req.transpose, 1.U, max_blocks) - i := Mux(req.transpose, next_col_iterator, next_row_iterator) - k := Mux(req.transpose, next_row_iterator, next_col_iterator) + val next_i = floorAdd(i, i_blocks, req.max_i) + val next_k = floorAdd(k, k_blocks, req.max_k, next_i === 0.U) - when (next_col_iterator === 0.U && next_row_iterator === 0.U) { + i := next_i + k := next_k + + when (next_i === 0.U && next_k === 0.U) { state := idle } } @@ -183,13 +185,16 @@ class LoopMatmulLdB(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (io.cmd.fire()) { // The order here is k, j, i - val next_col_iterator = floorAdd(col_iterator, max_blocks, max_col_iterator) - val next_row_iterator = floorAdd(row_iterator, 1.U, max_row_iterator, next_col_iterator === 0.U) + val j_blocks = Mux(req.transpose, 1.U, max_blocks) + val k_blocks = Mux(req.transpose, max_blocks, 1.U) - k := Mux(req.transpose, next_col_iterator, next_row_iterator) - j := Mux(req.transpose, next_row_iterator, next_col_iterator) + val next_j = floorAdd(j, j_blocks, req.max_j) + val next_k = floorAdd(k, k_blocks, req.max_k, next_j === 0.U) - when (next_row_iterator === 0.U && next_col_iterator === 0.U) { + j := next_j + k := next_k + + when (next_j === 0.U && next_k === 0.U) { state := idle } } @@ -230,7 +235,7 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In }) object State extends ChiselEnum { - val idle, st = Value + val idle, ld = Value } import State._ val state = RegInit(idle) @@ -270,7 +275,7 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (req.dram_addr === 0.U) { state := idle }.elsewhen (io.cmd.fire()) { - // The order here is j, i + // The order here is k, j, i val next_i = floorAdd(i, 1.U, req.max_i) val next_j = floorAdd(j, max_blocks, req.max_j, next_i === 0.U) @@ -284,7 +289,7 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In when (io.req.fire()) { req := io.req.bits - state := st + state := ld j := 0.U i := 0.U } @@ -309,7 +314,6 @@ class LoopMatmulExecuteReq(val block_size: Int, val coreMaxAddrBits: Int, val it class LoopMatmulExecute(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: Int, max_addr: Int, max_acc_addr: Int, concurrent_loops: Int) (implicit p: Parameters) extends Module { - val MAX_BLOCK_LEN = 4 // TODO get this from configs val GARBAGE_ADDR = (~0.U(32.W)).asUInt() val io = IO(new Bundle { @@ -655,7 +659,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: val is_loop_cmd = is_loop_run_cmd || is_loop_config_cmd io.out.bits := Mux(loop_configured, unrolled_cmd.bits, cmd.bits) - io.out.bits.status := cmd.bits.status + io.out.bits.status := cmd.bits.status // TODO This is not guaranteed to be the correct fix! We must fix this io.out.valid := Mux(loop_configured, unrolled_cmd.valid, cmd.valid && !is_loop_config_cmd && !is_loop_run_cmd) cmd.ready := Mux(is_loop_cmd, !loop_being_configured.configured, !loop_configured && io.out.ready) diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index ccc6dbd21..2c026dc07 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -11,58 +11,67 @@ import Util._ //import midas.targetutils.FpgaDebug // TODO unify this class with GemminiCmdWithDeps -class ROBIssue[T <: Data](cmd_t: T, nEntries: Int) extends Bundle { +class ROBIssue[T <: Data](cmd_t: T, rob_entries: Int) extends Bundle { val valid = Output(Bool()) val ready = Input(Bool()) val cmd = Output(cmd_t.cloneType) - val rob_id = Output(UInt(log2Up(nEntries).W)) + val rob_id = Output(UInt(log2Up(rob_entries).W)) def fire(dummy: Int=0) = valid && ready - override def cloneType: this.type = new ROBIssue(cmd_t, nEntries).asInstanceOf[this.type] + override def cloneType: this.type = new ROBIssue(cmd_t, rob_entries).asInstanceOf[this.type] } // TODO we don't need to store the full command in here. We should be able to release the command directly into the relevant controller and only store the associated metadata in the ROB. This would reduce the size considerably -class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows: Int, block_cols: Int) extends Module { +class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V], cmd_t: RoCCCommand) extends Module { + import config._ + + val block_rows = tileRows * meshRows + val block_cols = tileColumns * meshColumns + val io = IO(new Bundle { val alloc = Flipped(Decoupled(cmd_t.cloneType)) - val completed = Flipped(Valid(UInt(log2Up(nEntries).W))) + val completed = Flipped(Valid(UInt(log2Up(rob_entries).W))) val issue = new Bundle { - val ld = new ROBIssue(cmd_t, nEntries) - val st = new ROBIssue(cmd_t, nEntries) - val ex = new ROBIssue(cmd_t, nEntries) + val ld = new ROBIssue(cmd_t, rob_entries) + val st = new ROBIssue(cmd_t, rob_entries) + val ex = new ROBIssue(cmd_t, rob_entries) } - val ld_utilization = Output(UInt(log2Up(nEntries).W)) - val st_utilization = Output(UInt(log2Up(nEntries).W)) - val ex_utilization = Output(UInt(log2Up(nEntries).W)) + val ld_utilization = Output(UInt(log2Up(rob_entries).W)) + val st_utilization = Output(UInt(log2Up(rob_entries).W)) + val ex_utilization = Output(UInt(log2Up(rob_entries).W)) val busy = Output(Bool()) val solitary_preload = Input(Bool()) // TODO very hacky. from ExecuteController, to prevent infinite fence stalls. remove later }) + // TODO make this a ChiselEnum val ldq :: stq :: exq :: Nil = Enum(3) val q_t = ldq.cloneType + class OpT extends Bundle { + val start = local_addr_t.cloneType + val end = local_addr_t.cloneType + val wraps_around = Bool() + + def overlaps(other: OpT): Bool = { + (other.start <= start && (start <= other.end || other.wraps_around)) || + (start <= other.start && (other.start <= end || wraps_around)) + } + } + class Entry extends Bundle { val q = q_t.cloneType val is_config = Bool() - val op1 = UDValid(local_addr_t.cloneType) - val op2 = UDValid(local_addr_t.cloneType) - // val op3 = UDValid(local_addr_t.cloneType) - - val dst = UDValid(new Bundle { - val start = local_addr_t.cloneType - val len = UInt(8.W) // TODO magic number - - def end(dummy: Int = 0): LocalAddr = start + len * block_rows.U - def wraps_around(dummy: Int = 0): Bool = start.add_with_overflow(len * block_rows.U)._2 - }) + val op1 = UDValid(new OpT) + val op2 = UDValid(new OpT) + val dst = UDValid(new OpT) val issued = Bool() @@ -70,16 +79,17 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows val cmd = cmd_t.cloneType - val deps = Vec(nEntries, Bool()) + val deps = Vec(rob_entries, Bool()) def ready(dummy: Int = 0): Bool = !deps.reduce(_ || _) } - val entries = Reg(Vec(nEntries, UDValid(new Entry))) + val entries = Reg(Vec(rob_entries, UDValid(new Entry))) val empty = !entries.map(_.valid).reduce(_ || _) val full = entries.map(_.valid).reduce(_ && _) - // io.busy := !empty + // TODO we could also check for a solitary preload by recording the last instruction that was allocated, rather than + // reading all entries to check for preloads, which is an O(n) operation in terms of area cost val utilization = PopCount(entries.map(_.valid)) val solitary_preload = utilization === 1.U && entries.map(e => e.valid && e.bits.cmd.inst.funct === PRELOAD_CMD).reduce(_ || _) io.busy := !empty && !(solitary_preload && io.solitary_preload) @@ -87,13 +97,27 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows // Read in commands to the buffer io.alloc.ready := !full - val last_allocated = Reg(UInt(log2Up(nEntries).W)) + val last_allocated = Reg(UInt(log2Up(rob_entries).W)) + val a_stride = Reg(UInt(16.W)) // TODO magic numbers // TODO we also need to check the transpose to see how many rows we're reading + val block_strides = Reg(Vec(load_states, UInt(block_stride_bits.W))) val new_entry = Wire(new Entry) new_entry := DontCare - val new_entry_id = MuxCase((nEntries-1).U, entries.zipWithIndex.map { case (e, i) => !e.valid -> i.U }) + val new_entry_id = MuxCase((rob_entries-1).U, entries.zipWithIndex.map { case (e, i) => !e.valid -> i.U }) val alloc_fire = io.alloc.fire() + val raws_probe = WireInit(0.U(rob_entries.W)) + val waws_probe = WireInit(0.U(rob_entries.W)) + val wars_probe = WireInit(0.U(rob_entries.W)) + val older_in_same_q_probe = WireInit(0.U(rob_entries.W)) + val is_st_and_must_wait_for_prior_ex_config_probe = WireInit(0.U(rob_entries.W)) + val is_ex_config_and_must_wait_for_prior_st_probe = WireInit(0.U(rob_entries.W)) + + val wars_op1_probe = WireInit(0.U(rob_entries.W)) + val wars_op2_probe = WireInit(0.U(rob_entries.W)) + + dontTouch(new_entry) + when (io.alloc.fire()) { val spAddrBits = 32 val cmd = io.alloc.bits @@ -107,18 +131,47 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows new_entry.is_config := funct === CONFIG_CMD new_entry.op1.valid := funct === PRELOAD_CMD || funct_is_compute - new_entry.op1.bits := cmd.rs1.asTypeOf(local_addr_t) + new_entry.op1.bits.start := cmd.rs1.asTypeOf(local_addr_t) + when (funct === PRELOAD_CMD) { + val preload_rows = cmd.rs1(48 + log2Up(block_rows + 1) - 1, 48) + new_entry.op1.bits.end := new_entry.op1.bits.start + preload_rows + new_entry.op1.bits.wraps_around := new_entry.op1.bits.start.add_with_overflow(preload_rows)._2 + }.otherwise { + val compute_rows = cmd.rs1(48 + log2Up(block_rows + 1) - 1, 48) * a_stride + new_entry.op1.bits.end := new_entry.op1.bits.start + compute_rows + new_entry.op1.bits.wraps_around := new_entry.op1.bits.start.add_with_overflow(compute_rows)._2 + } new_entry.op2.valid := funct_is_compute || funct === STORE_CMD - new_entry.op2.bits := cmd.rs2.asTypeOf(local_addr_t) - - // new_entry.op3.valid := funct_is_compute - // new_entry.op3.bits := cmd.rs1(63, 32).asTypeOf(local_addr_t) + new_entry.op2.bits.start := cmd.rs2.asTypeOf(local_addr_t) + when (funct_is_compute) { + val compute_rows = cmd.rs2(48 + log2Up(block_rows + 1) - 1, 48) + new_entry.op2.bits.end := new_entry.op2.bits.start + compute_rows + new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(compute_rows)._2 + }.otherwise { + val mvout_rows = cmd.rs2(48 + mvout_rows_bits - 1, 48) + new_entry.op2.bits.end := new_entry.op2.bits.start + mvout_rows + new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(mvout_rows)._2 + } - val mvin_mvout_len = cmd.rs2(48, spAddrBits) new_entry.dst.valid := funct === PRELOAD_CMD || funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD new_entry.dst.bits.start := cmd.rs2(31, 0).asTypeOf(local_addr_t) - new_entry.dst.bits.len := Mux(funct === PRELOAD_CMD, 1.U, mvin_mvout_len / block_cols.U + (mvin_mvout_len % block_cols.U =/= 0.U)) + when (funct === PRELOAD_CMD) { + val preload_rows = cmd.rs2(48 + log2Up(block_rows + 1) - 1, 48) + new_entry.dst.bits.end := new_entry.dst.bits.start + preload_rows + new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(preload_rows)._2 + }.otherwise { + val id = MuxCase(0.U, Seq((new_entry.cmd.inst.funct === LOAD2_CMD) -> 1.U, + (new_entry.cmd.inst.funct === LOAD3_CMD) -> 2.U)) + val block_stride = block_strides(id) + + val mvin_cols = cmd.rs2(spAddrBits + mvin_cols_bits - 1, spAddrBits) + val mvin_mats = mvin_cols / block_cols.U + (mvin_cols % block_cols.U =/= 0.U) + val mvin_rows = mvin_mats * block_stride + + new_entry.dst.bits.end := new_entry.dst.bits.start + mvin_rows + new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(mvin_rows)._2 + } val is_load = funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD || (funct === CONFIG_CMD && config_cmd_type === CONFIG_LOAD) val is_store = funct === STORE_CMD || (funct === CONFIG_CMD && config_cmd_type === CONFIG_STORE) @@ -130,30 +183,41 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows is_ex -> exq )) + assert(is_load || is_store || is_ex) + + // TODO we should checck whether op1 and op2 are valid here val raws = entries.map { e => // We search for all entries which write to an address which we read from e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( - (new_entry.op1.valid && e.bits.dst.bits.start <= new_entry.op1.bits && (e.bits.dst.bits.end() > new_entry.op1.bits || e.bits.dst.bits.wraps_around())) || - (new_entry.op2.valid && e.bits.dst.bits.start <= new_entry.op2.bits && (e.bits.dst.bits.end() > new_entry.op2.bits || e.bits.dst.bits.wraps_around()))) /* || - (new_entry.op3.valid && e.bits.dst.bits.start <= new_entry.op3.bits && e.bits.dst.bits.end() > new_entry.op3.bits)) */ + (new_entry.op1.valid && new_entry.op1.bits.overlaps(e.bits.dst.bits)) || + (new_entry.op2.valid && new_entry.op2.bits.overlaps(e.bits.dst.bits))) } + // TODO we should checck whether op1 and op2 are valid here val wars = entries.map { e => // We search for all entries which read from an address that we write to e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( - (e.bits.op1.valid && new_entry.dst.bits.start <= e.bits.op1.bits && (new_entry.dst.bits.end() > e.bits.op1.bits || new_entry.dst.bits.wraps_around())) || - (e.bits.op2.valid && new_entry.dst.bits.start <= e.bits.op2.bits && (new_entry.dst.bits.end() > e.bits.op2.bits || new_entry.dst.bits.wraps_around()))) /* || - (e.bits.op3.valid && new_entry.dst.bits.start <= e.bits.op3.bits && new_entry.dst.bits.end() > e.bits.op3.bits)) */ + (e.bits.op1.valid && e.bits.op1.bits.overlaps(new_entry.dst.bits)) || + (e.bits.op2.valid && e.bits.op2.bits.overlaps(new_entry.dst.bits))) } - val waws = entries.map { e => - def is_accumulative(laddr: LocalAddr): Bool = laddr.is_acc_addr && laddr.accumulate + val wars_op1 = entries.map { e => + // We search for all entries which read from an address that we write to + e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( + e.bits.op1.bits.overlaps(new_entry.dst.bits)) + } + val wars_op2 = entries.map { e => + // We search for all entries which read from an address that we write to + e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( + e.bits.op2.bits.overlaps(new_entry.dst.bits)) + } + + // TODO we should checck whether op1 and op2 are valid here + val waws = entries.map { e => // We search for all entries which write to an address that we write to e.valid && new_entry.dst.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && - !(is_accumulative(new_entry.dst.bits.start) && is_accumulative(e.bits.dst.bits.start)) && - ((new_entry.dst.bits.start <= e.bits.dst.bits.start && (new_entry.dst.bits.end() > e.bits.dst.bits.start || new_entry.dst.bits.wraps_around())) || - (e.bits.dst.bits.start <= new_entry.dst.bits.start && (e.bits.dst.bits.end() > new_entry.dst.bits.start || e.bits.dst.bits.wraps_around()))) + (new_entry.dst.bits.overlaps(e.bits.dst.bits) || e.bits.dst.bits.overlaps(new_entry.dst.bits)) } val older_in_same_q = entries.map { e => @@ -171,17 +235,43 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows new_entry.deps := (Cat(raws) | Cat(wars) | Cat(waws) | Cat(older_in_same_q) | Cat(is_st_and_must_wait_for_prior_ex_config) | Cat(is_ex_config_and_must_wait_for_prior_st)).asBools().reverse + raws_probe := Cat(raws) + waws_probe := Cat(waws) + wars_probe := Cat(wars) + wars_op1_probe := Cat(wars_op1) + wars_op2_probe := Cat(wars_op2) + older_in_same_q_probe := Cat(older_in_same_q) + is_st_and_must_wait_for_prior_ex_config_probe := Cat(is_st_and_must_wait_for_prior_ex_config) + is_ex_config_and_must_wait_for_prior_st_probe := Cat(is_ex_config_and_must_wait_for_prior_st) + + dontTouch(raws_probe) + dontTouch(waws_probe) + dontTouch(wars_probe) + dontTouch(wars_op1_probe) + dontTouch(wars_op2_probe) + dontTouch(older_in_same_q_probe) + dontTouch(is_st_and_must_wait_for_prior_ex_config_probe) + dontTouch(is_ex_config_and_must_wait_for_prior_st_probe) + new_entry.complete_on_issue := new_entry.is_config && new_entry.q =/= exq entries(new_entry_id).valid := true.B entries(new_entry_id).bits := new_entry last_allocated := new_entry_id + + when (new_entry.is_config && new_entry.q === exq) { + a_stride := new_entry.cmd.rs1(31, 16) // TODO magic numbers // TODO this needs to be kept in sync with ExecuteController.scala + }.elsewhen(new_entry.is_config && new_entry.q === ldq) { + val id = new_entry.cmd.rs1(4,3) // TODO magic numbers + val block_stride = new_entry.cmd.rs1(31, 16) // TODO magic numbers + block_strides(id) := block_stride + } } // Issue commands which are ready to be issued Seq((ldq, io.issue.ld), (stq, io.issue.st), (exq, io.issue.ex)).foreach { case (q, io) => - val issue_id = MuxCase((nEntries-1).U, entries.zipWithIndex.map { case (e, i) => + val issue_id = MuxCase((rob_entries-1).U, entries.zipWithIndex.map { case (e, i) => (e.valid && e.bits.ready() && !e.bits.issued && e.bits.q === q) -> i.U }) @@ -245,7 +335,6 @@ class ROB(cmd_t: RoCCCommand, nEntries: Int, local_addr_t: LocalAddr, block_rows } assert(cycles_since_issue < 10000.U, "pipeline stall") - val cntr = Counter(10000000) when (cntr.inc()) { printf(p"Utilization: $utilization\n") diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index c26952076..5f11cb0b9 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -15,15 +15,13 @@ class ScratchpadMemReadRequest[U <: Data](local_addr_t: LocalAddr, scale_t_bits: val vaddr = UInt(coreMaxAddrBits.W) val laddr = local_addr_t.cloneType - val len = UInt(16.W) // TODO don't use a magic number for the width here + val cols = UInt(16.W) // TODO don't use a magic number for the width here val repeats = UInt(16.W) // TODO don't use a magic number for the width here - val scale = UInt(scale_t_bits.W) - val has_acc_bitwidth = Bool() - + val all_zeros = Bool() + val block_stride = UInt(16.W) // TODO magic numbers val cmd_id = UInt(8.W) // TODO don't use a magic number here - val status = new MStatus override def cloneType: this.type = new ScratchpadMemReadRequest(local_addr_t, scale_t_bits).asInstanceOf[this.type] @@ -33,11 +31,8 @@ class ScratchpadMemWriteRequest(local_addr_t: LocalAddr) (implicit p: Parameters) extends CoreBundle { val vaddr = UInt(coreMaxAddrBits.W) val laddr = local_addr_t.cloneType - val len = UInt(16.W) // TODO don't use a magic number for the width here - val cmd_id = UInt(8.W) // TODO don't use a magic number here - val status = new MStatus // Pooling variables @@ -192,7 +187,6 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // Accumulator ports val acc = new Bundle { val read = Flipped(Vec(acc_banks, new AccumulatorReadIO(acc_bank_entries, log2Up(accType.getWidth), Vec(meshColumns, Vec(tileColumns, inputType)), Vec(meshColumns, Vec(tileColumns, accType)), acc_scale_args.multiplicand_t))) - // val write = Flipped(Vec(acc_banks, new AccumulatorWriteReq(acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType))))) val write = Flipped(Vec(acc_banks, Decoupled(new AccumulatorWriteReq(acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType)))))) } @@ -237,26 +231,38 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, writer.module.io.req.bits.pool_en := write_issue_q.io.deq.bits.pool_en writer.module.io.req.bits.store_en := write_issue_q.io.deq.bits.store_en - // FpgaDebug(write_issue_q.io.deq.bits.laddr.data) - // FpgaDebug(write_issue_q.io.deq.bits.laddr.accumulate) - // FpgaDebug(write_issue_q.io.deq.bits.laddr.is_acc_addr) - io.dma.write.resp.valid := false.B io.dma.write.resp.bits.cmd_id := write_dispatch_q.bits.cmd_id read_issue_q.io.enq <> io.dma.read.req + val zero_writer = Module(new ZeroWriter(config, new ScratchpadMemReadRequest(local_addr_t, mvin_scale_t_bits))) + + when (io.dma.read.req.bits.all_zeros) { + read_issue_q.io.enq.valid := false.B + io.dma.read.req.ready := zero_writer.io.req.ready + } + + zero_writer.io.req.valid := io.dma.read.req.valid && io.dma.read.req.bits.all_zeros + zero_writer.io.req.bits.laddr := io.dma.read.req.bits.laddr + zero_writer.io.req.bits.cols := io.dma.read.req.bits.cols + zero_writer.io.req.bits.block_stride := io.dma.read.req.bits.block_stride + zero_writer.io.req.bits.tag := io.dma.read.req.bits + + zero_writer.io.resp.ready := false.B + reader.module.io.req.valid := read_issue_q.io.deq.valid read_issue_q.io.deq.ready := reader.module.io.req.ready reader.module.io.req.bits.vaddr := read_issue_q.io.deq.bits.vaddr reader.module.io.req.bits.spaddr := Mux(read_issue_q.io.deq.bits.laddr.is_acc_addr, read_issue_q.io.deq.bits.laddr.full_acc_addr(), read_issue_q.io.deq.bits.laddr.full_sp_addr()) - reader.module.io.req.bits.len := read_issue_q.io.deq.bits.len + reader.module.io.req.bits.len := read_issue_q.io.deq.bits.cols reader.module.io.req.bits.repeats := read_issue_q.io.deq.bits.repeats reader.module.io.req.bits.scale := read_issue_q.io.deq.bits.scale reader.module.io.req.bits.is_acc := read_issue_q.io.deq.bits.laddr.is_acc_addr reader.module.io.req.bits.accumulate := read_issue_q.io.deq.bits.laddr.accumulate reader.module.io.req.bits.has_acc_bitwidth := read_issue_q.io.deq.bits.has_acc_bitwidth + reader.module.io.req.bits.block_stride := read_issue_q.io.deq.bits.block_stride reader.module.io.req.bits.status := read_issue_q.io.deq.bits.status reader.module.io.req.bits.cmd_id := read_issue_q.io.deq.bits.cmd_id @@ -269,7 +275,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, mvin_scale_in.bits.in := reader.module.io.resp.bits.data.asTypeOf(chiselTypeOf(mvin_scale_in.bits.in)) mvin_scale_in.bits.scale := reader.module.io.resp.bits.scale.asTypeOf(mvin_scale_t) - mvin_scale_in.bits.repeats := reader.module.io.resp.bits.rows + mvin_scale_in.bits.repeats := reader.module.io.resp.bits.repeats mvin_scale_in.bits.last := reader.module.io.resp.bits.last mvin_scale_in.bits.tag := reader.module.io.resp.bits @@ -280,7 +286,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, (reader.module.io.resp.bits.is_acc && reader.module.io.resp.bits.has_acc_bitwidth) mvin_scale_acc_in.bits.in := reader.module.io.resp.bits.data.asTypeOf(chiselTypeOf(mvin_scale_acc_in.bits.in)) mvin_scale_acc_in.bits.scale := reader.module.io.resp.bits.scale.asTypeOf(mvin_scale_acc_t) - mvin_scale_acc_in.bits.repeats := reader.module.io.resp.bits.rows + mvin_scale_acc_in.bits.repeats := reader.module.io.resp.bits.repeats mvin_scale_acc_in.bits.last := reader.module.io.resp.bits.last mvin_scale_acc_in.bits.tag := reader.module.io.resp.bits @@ -292,9 +298,22 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val mvin_scale_finished = mvin_scale_out.fire() && mvin_scale_out.bits.last val mvin_scale_acc_finished = mvin_scale_acc_out.fire() && mvin_scale_acc_out.bits.last - io.dma.read.resp.valid := mvin_scale_finished || mvin_scale_acc_finished - io.dma.read.resp.bits.cmd_id := Mux(mvin_scale_finished, mvin_scale_out.bits.tag.cmd_id, mvin_scale_acc_out.bits.tag.cmd_id) - io.dma.read.resp.bits.bytesRead := Mux(mvin_scale_finished, mvin_scale_out.bits.tag.bytes_read, mvin_scale_acc_out.bits.tag.bytes_read) + val zero_writer_finished = zero_writer.io.resp.fire() && zero_writer.io.resp.bits.last + + val zero_writer_bytes_read = Mux(zero_writer.io.resp.bits.laddr.is_acc_addr, + zero_writer.io.resp.bits.tag.cols * (accType.getWidth / 8).U, + zero_writer.io.resp.bits.tag.cols * (inputType.getWidth / 8).U) + + // For DMA read responses, mvin_scale gets first priority, then mvin_scale_acc, and then zero_writer + io.dma.read.resp.valid := mvin_scale_finished || mvin_scale_acc_finished || zero_writer_finished + + io.dma.read.resp.bits.cmd_id := MuxCase(zero_writer.io.resp.bits.tag.cmd_id, Seq( + mvin_scale_finished -> mvin_scale_out.bits.tag.cmd_id, + mvin_scale_acc_finished -> mvin_scale_acc_out.bits.tag.cmd_id)) + + io.dma.read.resp.bits.bytesRead := MuxCase(zero_writer_bytes_read, Seq( + mvin_scale_finished -> mvin_scale_out.bits.tag.bytes_read, + mvin_scale_acc_finished -> mvin_scale_acc_out.bits.tag.bytes_read)) io.tlb(0) <> writer.module.io.tlb io.tlb(1) <> reader.module.io.tlb @@ -366,7 +385,13 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val dmaread = mvin_scale_out.valid && !mvin_scale_out.bits.tag.is_acc && laddr.sp_bank() === i.U - bio.write.en := exwrite || dmaread + // We need to make sure that we don't try to return a dma read resp from both zero_writer and either mvin_scale + // or mvin_acc_scale at the same time. The scalers always get priority in those cases + val zerowrite = zero_writer.io.resp.valid && !zero_writer.io.resp.bits.laddr.is_acc_addr && + zero_writer.io.resp.bits.laddr.sp_bank() === i.U && + !((mvin_scale_out.valid && mvin_scale_out.bits.last) || (mvin_scale_acc_out.valid && mvin_scale_acc_out.bits.last)) + + bio.write.en := exwrite || dmaread || zerowrite when (exwrite) { bio.write.addr := io.srams.write(i).addr @@ -378,6 +403,12 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, bio.write.mask := mvin_scale_out.bits.tag.mask take ((spad_w / (aligned_to * 8)) max 1) mvin_scale_out.ready := true.B // TODO we combinationally couple valid and ready signals + }.elsewhen (zerowrite) { + bio.write.addr := zero_writer.io.resp.bits.laddr.sp_row() + bio.write.data := 0.U + bio.write.mask := zero_writer.io.resp.bits.mask + + zero_writer.io.resp.ready := true.B // TODO we combinationally couple valid and ready signals }.otherwise { bio.write.addr := DontCare bio.write.data := DontCare @@ -449,6 +480,9 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // Writing to the accumulator banks bank_ios.zipWithIndex.foreach { case (bio, i) => + // Order of precedence during writes is ExecuteController, and then mvin_scale, and then mvin_scale_acc, and + // then zero_writer + val exwrite = io.acc.write(i).valid io.acc.write(i).ready := true.B assert(!(exwrite && !bio.write.ready), "Execute controller write to AccumulatorMem was skipped") @@ -456,48 +490,73 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val from_mvin_scale = mvin_scale_out.valid && mvin_scale_out.bits.tag.is_acc val from_mvin_scale_acc = mvin_scale_acc_out.valid && mvin_scale_acc_out.bits.tag.is_acc - val mvin_scale_acc_laddr = mvin_scale_acc_out.bits.tag.addr.asTypeOf(local_addr_t) + mvin_scale_acc_out.bits.row val mvin_scale_laddr = mvin_scale_out.bits.tag.addr.asTypeOf(local_addr_t) + mvin_scale_out.bits.row + val mvin_scale_acc_laddr = mvin_scale_acc_out.bits.tag.addr.asTypeOf(local_addr_t) + mvin_scale_acc_out.bits.row - val dmaread_bank = Mux(from_mvin_scale_acc, mvin_scale_acc_laddr.acc_bank(), - mvin_scale_laddr.acc_bank()) - val dmaread_row = Mux(from_mvin_scale_acc, mvin_scale_acc_laddr.acc_row(), mvin_scale_laddr.acc_row()) + val dmaread_bank = Mux(from_mvin_scale, mvin_scale_laddr.acc_bank(), + mvin_scale_acc_laddr.acc_bank()) + val dmaread_row = Mux(from_mvin_scale, mvin_scale_laddr.acc_row(), mvin_scale_acc_laddr.acc_row()) // We need to make sure that we don't try to return a dma read resp from both mvin_scale and mvin_scale_acc // at the same time. mvin_scale always gets priority in this cases - val mvin_scale_out_last = mvin_scale_out.valid && mvin_scale_out.bits.last + val spad_dmaread_last = mvin_scale_out.valid && mvin_scale_out.bits.last && !mvin_scale_out.bits.tag.is_acc + val spad_zerowrite_last = zero_writer.io.resp.valid && zero_writer.io.resp.bits.last && + !zero_writer.io.resp.bits.laddr.is_acc_addr + val spad_last = spad_dmaread_last || spad_zerowrite_last val dmaread = (from_mvin_scale || from_mvin_scale_acc) && - dmaread_bank === i.U && - (mvin_scale_same.B || from_mvin_scale || !mvin_scale_out_last) + dmaread_bank === i.U /* && + (mvin_scale_same.B || from_mvin_scale || !spad_dmaread_last) */ + + // We need to make sure that we don't try to return a dma read resp from both zero_writer and either mvin_scale + // or mvin_acc_scale at the same time. The scalers always get priority in those cases + val zerowrite = zero_writer.io.resp.valid && zero_writer.io.resp.bits.laddr.is_acc_addr && + zero_writer.io.resp.bits.laddr.acc_bank() === i.U && + !((mvin_scale_out.valid && mvin_scale_out.bits.last) || (mvin_scale_acc_out.valid && mvin_scale_acc_out.bits.last)) - bio.write.valid := exwrite || dmaread - bio.write.bits.acc := Mux(exwrite, io.acc.write(i).bits.acc, - Mux(from_mvin_scale_acc, mvin_scale_acc_out.bits.tag.accumulate, mvin_scale_out.bits.tag.accumulate)) - bio.write.bits.addr := Mux(exwrite, io.acc.write(i).bits.addr, dmaread_row) + bio.write.valid := exwrite || ((dmaread || zerowrite) && !spad_last) + + bio.write.bits.acc := MuxCase(zero_writer.io.resp.bits.laddr.accumulate, + Seq(exwrite -> io.acc.write(i).bits.acc, + from_mvin_scale -> mvin_scale_out.bits.tag.accumulate, + from_mvin_scale_acc -> mvin_scale_acc_out.bits.tag.accumulate)) + + bio.write.bits.addr := MuxCase(zero_writer.io.resp.bits.laddr.acc_row(), + Seq(exwrite -> io.acc.write(i).bits.addr, + (from_mvin_scale || from_mvin_scale_acc) -> dmaread_row)) when (exwrite) { bio.write.bits.data := io.acc.write(i).bits.data bio.write.bits.mask := io.acc.write(i).bits.mask }.elsewhen (dmaread && bio.write.fire()) { - bio.write.bits.data := Mux(from_mvin_scale_acc, - mvin_scale_acc_out.bits.out.asTypeOf(acc_row_t), - VecInit(mvin_scale_out.bits.out.map(e => e.withWidthOf(accType))).asTypeOf(acc_row_t)) + bio.write.bits.data := Mux(from_mvin_scale, + VecInit(mvin_scale_out.bits.out.map(e => e.withWidthOf(accType))).asTypeOf(acc_row_t), + mvin_scale_acc_out.bits.out.asTypeOf(acc_row_t)) bio.write.bits.mask := - Mux(from_mvin_scale_acc, - mvin_scale_acc_out.bits.tag.mask, + Mux(from_mvin_scale, { val n = accType.getWidth / inputType.getWidth val mask = mvin_scale_out.bits.tag.mask take ((spad_w / (aligned_to * 8)) max 1) val expanded = VecInit(mask.flatMap(e => Seq.fill(n)(e))) expanded - }) + }, + mvin_scale_acc_out.bits.tag.mask) - when (from_mvin_scale_acc) { - mvin_scale_acc_out.ready := true.B - }.otherwise { + when(from_mvin_scale) { mvin_scale_out.ready := true.B + }.otherwise { + mvin_scale_acc_out.ready := true.B } + }.elsewhen (zerowrite && bio.write.fire()) { + bio.write.bits.data := 0.U.asTypeOf(acc_row_t) + bio.write.bits.mask := { + val n = accType.getWidth / inputType.getWidth + val mask = zero_writer.io.resp.bits.mask + val expanded = VecInit(mask.flatMap(e => Seq.fill(n)(e))) + expanded + } + + zero_writer.io.resp.ready := true.B }.otherwise { bio.write.bits.data := DontCare bio.write.bits.mask := DontCare diff --git a/src/main/scala/gemmini/StoreController.scala b/src/main/scala/gemmini/StoreController.scala index 07399a168..83609ffb7 100644 --- a/src/main/scala/gemmini/StoreController.scala +++ b/src/main/scala/gemmini/StoreController.scala @@ -69,7 +69,7 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm val cmd = Queue(io.cmd, st_queue_length) val vaddr = cmd.bits.cmd.rs1 val localaddr = cmd.bits.cmd.rs2.asTypeOf(local_addr_t) - val cols = cmd.bits.cmd.rs2(32 + mvout_len_bits - 1, 32) // TODO magic numbers + val cols = cmd.bits.cmd.rs2(32 + mvout_cols_bits - 1, 32) // TODO magic numbers val rows = cmd.bits.cmd.rs2(48 + mvout_rows_bits - 1, 48) // TODO magic numbers val config_stride = cmd.bits.cmd.rs2 val config_pool_stride = cmd.bits.cmd.rs1(5, 4) // TODO magic numbers diff --git a/src/main/scala/gemmini/Util.scala b/src/main/scala/gemmini/Util.scala index 593d20707..dd837c7d6 100644 --- a/src/main/scala/gemmini/Util.scala +++ b/src/main/scala/gemmini/Util.scala @@ -44,6 +44,15 @@ object Util { )) } + def sFloorAdd(s: SInt, n: UInt, max_plus_one: SInt, min: SInt, en: Bool = true.B): SInt = { + val max = max_plus_one - 1.S + + MuxCase(s + n.zext(), Seq( + (!en) -> s, + ((s +& n.zext()) > max) -> min + )) + } + def wrappingSub(u: UInt, n: UInt, max_plus_one: Int): UInt = { val max = max_plus_one - 1 assert(n <= max.U, "cannot wrapSub when n is larger than max") diff --git a/src/main/scala/gemmini/XactTracker.scala b/src/main/scala/gemmini/XactTracker.scala index af020ed98..9eee539ae 100644 --- a/src/main/scala/gemmini/XactTracker.scala +++ b/src/main/scala/gemmini/XactTracker.scala @@ -13,7 +13,8 @@ class XactTrackerEntry[U <: Data](maxShift: Int, spadWidth: Int, accWidth: Int, val accumulate = Bool() val has_acc_bitwidth = Bool() val scale = UInt(mvin_scale_t_bits.W) - val rows = UInt(16.W) // TODO magic number + val repeats = UInt(16.W) // TODO magic number + val block_stride = UInt(16.W) // TODO magic number val spad_row_offset = UInt(log2Up(spadWidth max accWidth).W) val lg_len_req = UInt(log2Up(log2Up(maxReqBytes+1)+1).W) val bytes_to_read = UInt(log2Up(maxReqBytes+1).W) diff --git a/src/main/scala/gemmini/ZeroWriter.scala b/src/main/scala/gemmini/ZeroWriter.scala new file mode 100644 index 000000000..a8e67df3c --- /dev/null +++ b/src/main/scala/gemmini/ZeroWriter.scala @@ -0,0 +1,70 @@ +package gemmini + +import chisel3._ +import chisel3.util._ + +import Util._ + +class ZeroWriterReq[Tag <: Data](laddr_t: LocalAddr, max_cols: Int, tag_t: Tag) extends Bundle { + val laddr = laddr_t + val cols = UInt(log2Up(max_cols).W) + val block_stride = UInt(16.W) // TODO magic number + val tag = tag_t + + override def cloneType: ZeroWriterReq.this.type = new ZeroWriterReq(laddr_t.cloneType, max_cols, tag_t.cloneType).asInstanceOf[this.type] +} + +class ZeroWriterResp[Tag <: Data](laddr_t: LocalAddr, block_cols: Int, tag_t: Tag) extends Bundle { + val laddr = laddr_t.cloneType + val mask = Vec(block_cols, Bool()) + val last = Bool() + val tag = tag_t + + override def cloneType: ZeroWriterResp.this.type = new ZeroWriterResp(laddr_t, block_cols, tag_t.cloneType).asInstanceOf[this.type] +} + +class ZeroWriter[T <: Data, U <: Data, V <: Data, Tag <: Data](config: GemminiArrayConfig[T, U, V], tag_t: Tag) + extends Module { + import config._ + + val block_cols = meshColumns * tileColumns + val max_cols = (dma_maxbytes / (inputType.getWidth / 8)) max block_cols + + val io = IO(new Bundle { + val req = Flipped(Decoupled(new ZeroWriterReq(local_addr_t, max_cols, tag_t))) + val resp = Decoupled(new ZeroWriterResp(local_addr_t, block_cols, tag_t)) + }) + + val req = Reg(UDValid(new ZeroWriterReq(local_addr_t, max_cols, tag_t))) + + val col_counter = Reg(UInt(log2Up(max_cols).W)) + + io.req.ready := !req.valid + + io.resp.valid := req.valid + io.resp.bits.laddr := req.bits.laddr + req.bits.block_stride * (col_counter / block_cols.U) + io.resp.bits.mask.zipWithIndex.foreach { case (m, i) => m := col_counter + i.U < req.bits.cols } + io.resp.bits.last := col_counter +& block_cols.U >= req.bits.cols + io.resp.bits.tag := req.bits.tag + + when (io.resp.fire()) { + val next_col_counter = floorAdd(col_counter, block_cols.U, req.bits.cols) + + col_counter := next_col_counter + + when (next_col_counter === 0.U) { + req.pop() + io.req.ready := true.B + } + } + + when (io.req.fire()) { + req.push(io.req.bits) + + col_counter := 0.U + } + + when (reset.toBool()) { + req.pop() + } +} From c7a2100d3ad86858d443df84aad67c97176d72f9 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Mon, 8 Feb 2021 06:30:51 -0800 Subject: [PATCH 008/123] Remove popcounts in dma (#55) --- src/main/scala/gemmini/DMA.scala | 72 ++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 23 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index af23998f5..d35564225 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -292,7 +292,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf class StreamWriteRequest(val dataWidth: Int)(implicit p: Parameters) extends CoreBundle { val vaddr = UInt(coreMaxAddrBits.W) val data = UInt(dataWidth.W) - val len = UInt(16.W) // The number of bytes to write // TODO magic number + val len = UInt(log2Up(dataWidth/8+1).W) // The number of bytes to write val status = new MStatus // Pooling variables @@ -329,7 +329,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val req = Reg(new StreamWriteRequest(dataWidth)) - val bytesSent = Reg(UInt(log2Ceil(dataBytes).W)) // TODO this only needs to count up to (dataBytes/aligned_to), right? + val bytesSent = Reg(UInt(log2Ceil(dataBytes+1).W)) // TODO this only needs to count up to (dataBytes/aligned_to), right? val bytesLeft = req.len - bytesSent val xactBusy = RegInit(0.U(nXacts.W)) @@ -349,13 +349,15 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: // Select the size and mask of the TileLink request class Packet extends Bundle { - val size = UInt(log2Ceil(maxBytes).W) - val lg_size = UInt(log2Ceil(log2Ceil(maxBytes)).W) + val size = UInt(log2Ceil(maxBytes+1).W) + val lg_size = UInt(log2Ceil(log2Ceil(maxBytes+1)+1).W) val mask = Vec(maxBeatsPerReq, Vec(beatBytes, Bool())) val vaddr = UInt(vaddrBits.W) val is_full = Bool() - def bytes_written(dummy: Int = 0) = PopCount(mask.flatten) + val bytes_written = UInt(log2Up(dataBytes+1).W) + val bytes_written_per_beat = Vec(maxBeatsPerReq, UInt(log2Up(beatBytes+1).W)) + def total_beats(dummy: Int = 0) = Mux(size < beatBytes.U, 1.U, size / beatBytes.U) } @@ -367,16 +369,12 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val write_packets = write_sizes.map { s => val lg_s = log2Ceil(s) val vaddr_aligned_to_size = if (s == 1) vaddr else Cat(vaddr(vaddrBits-1, lg_s), 0.U(lg_s.W)) + val vaddr_offset = if (s > 1) vaddr(lg_s - 1, 0) else 0.U - val mask = (0 until maxBytes).map { i => - if (s > 1) { - val vaddr_offset = vaddr(lg_s - 1, 0) + val mask = (0 until maxBytes).map { i => i.U >= vaddr_offset && i.U < vaddr_offset +& bytesLeft && (i < s).B } - i.U >= vaddr_offset && - i.U < vaddr_offset +& bytesLeft - } else { - true.B - } && (i < s).B + val bytes_written = { + Mux(vaddr_offset +& bytesLeft > s.U, s.U - vaddr_offset, bytesLeft) } val packet = Wire(new Packet()) @@ -386,10 +384,29 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: packet.vaddr := vaddr_aligned_to_size packet.is_full := mask.take(s).reduce(_ && _) + packet.bytes_written := bytes_written + packet.bytes_written_per_beat.zipWithIndex.foreach { case (b, i) => + val start_of_beat = i * beatBytes + val end_of_beat = (i+1) * beatBytes + + val left_shift = Mux(vaddr_offset >= start_of_beat.U && vaddr_offset < end_of_beat.U, + vaddr_offset - start_of_beat.U, + 0.U) + + val right_shift = Mux(vaddr_offset +& bytesLeft >= start_of_beat.U && vaddr_offset +& bytesLeft < end_of_beat.U, + end_of_beat.U - (vaddr_offset +& bytesLeft), + 0.U) + + val too_early = vaddr_offset >= end_of_beat.U + val too_late = vaddr_offset +& bytesLeft <= start_of_beat.U + + b := Mux(too_early || too_late, 0.U, beatBytes.U - (left_shift +& right_shift)) + } + packet } val best_write_packet = write_packets.reduce { (acc, p) => - Mux(p.bytes_written() > acc.bytes_written(), p, acc) + Mux(p.bytes_written > acc.bytes_written, p, acc) } val write_packet = RegEnableThru(best_write_packet, state === s_writing_new_block) @@ -405,7 +422,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val write_mask = write_packet.mask(beatsSent) val write_shift = PriorityEncoder(write_mask) - val bytes_written_this_beat = PopCount(write_mask) + val bytes_written_this_beat = write_packet.bytes_written_per_beat(beatsSent) // Firing off TileLink write requests val putFull = edge.Put( @@ -430,7 +447,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: } val untranslated_a = Wire(Decoupled(new TLBundleAWithInfo)) - xactBusy_fire := untranslated_a.fire() + xactBusy_fire := untranslated_a.fire() && state === s_writing_new_block untranslated_a.valid := (state === s_writing_new_block || state === s_writing_beats) && !xactBusy.andR() untranslated_a.bits.tl_a := Mux(write_full, putFull, putPartial) untranslated_a.bits.vaddr := write_vaddr @@ -438,9 +455,13 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: // 0 goes to retries, 1 goes to state machine val retry_a = Wire(Decoupled(new TLBundleAWithInfo)) - val tlb_arb = Module(new Arbiter(new TLBundleAWithInfo, 2)) + val shadow_retry_a = Module(new Queue(new TLBundleAWithInfo, 1)) + shadow_retry_a.io.enq.valid := false.B + shadow_retry_a.io.enq.bits := DontCare + val tlb_arb = Module(new Arbiter(new TLBundleAWithInfo, 3)) tlb_arb.io.in(0) <> retry_a - tlb_arb.io.in(1) <> untranslated_a + tlb_arb.io.in(1) <> shadow_retry_a.io.deq + tlb_arb.io.in(2) <> untranslated_a val tlb_q = Module(new Queue(new TLBundleAWithInfo, 1, pipe=true)) tlb_q.io.enq <> tlb_arb.io.out @@ -454,14 +475,19 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val translate_q = Module(new Queue(new TLBundleAWithInfo, 1, pipe=true)) translate_q.io.enq <> tlb_q.io.deq + when (retry_a.valid) { + translate_q.io.enq.valid := false.B + shadow_retry_a.io.enq.valid := tlb_q.io.deq.valid + shadow_retry_a.io.enq.bits := tlb_q.io.deq.bits + } translate_q.io.deq.ready := true.B retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready) retry_a.bits := translate_q.io.deq.bits assert(retry_a.ready) - tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss - tl.a.bits := translate_q.io.deq.bits.tl_a + tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss + tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr tl.d.ready := xactBusy.orR() @@ -470,7 +496,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: when (state === s_writing_new_block) { beatsLeft := write_beats - 1.U - val next_vaddr = req.vaddr + bytes_written_this_beat + val next_vaddr = req.vaddr + write_packet.bytes_written req.vaddr := next_vaddr bytesSent := bytesSent + bytes_written_this_beat @@ -488,9 +514,9 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: beatsLeft := beatsLeft - 1.U bytesSent := bytesSent + bytes_written_this_beat - when (beatsLeft === 0.U) { - val new_page = req.vaddr(pgIdxBits-1, 0) === 0.U + assert(beatsLeft > 0.U) + when (beatsLeft === 1.U) { when (bytes_written_this_beat >= bytesLeft) { // We're done with this request at this point state_machine_ready_for_req := true.B From 8335f1b580271f7ae3635417bbb3ba0b17fc4f23 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Mon, 8 Feb 2021 09:41:23 -0800 Subject: [PATCH 009/123] pipeline scaling parameters (#56) --- src/main/scala/gemmini/Configs.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index bdf4b65df..273c8a4e0 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -56,7 +56,7 @@ object GemminiConfigs { shifter_banks = 1, // TODO add separate parameters for left and up shifter banks dataflow = Dataflow.BOTH, acc_capacity = CapacityInKilobytes(64), - mem_pipeline = 1, + mem_pipeline = 4, hasIm2col = true, //declare im2col block dma_maxbytes = 64, // TODO get this from cacheblockbytes dma_buswidth = 128, // TODO get this from SystemBusKey @@ -123,7 +123,7 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 0, Float(8, 24), + 4, Float(8, 24), identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (elem_t)y);})" )), From 6d183b1dfc89d649bf269dd566dcc32b4066cfd8 Mon Sep 17 00:00:00 2001 From: Seah <54855793+SeahK@users.noreply.github.com> Date: Wed, 10 Feb 2021 19:51:29 -0800 Subject: [PATCH 010/123] Chip config (#58) --- src/main/scala/gemmini/Configs.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 273c8a4e0..7c01ce7d6 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -198,7 +198,18 @@ class DefaultGemminiConfig extends Config((site, here, up) => { case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) }) - +// Default feature for initial Gemmini Chip tape-out experiments +// ToDo: increase & decrease spad/mesh size, single ported SRAM, increase in flight requests +class DefaultGemminiChipConfig extends Config((site, here, up) => { + case BuildRoCC => up(BuildRoCC) ++ Seq( + (p: Parameters) => { + implicit val q = p + val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), dataflow = Dataflow.WS, sp_singleported=false, max_in_flight_reqs=16))) + gemmini + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) /** * Mixin which configures a smaller host processor for the systolic array. This mixin **replaces** the default host rocket (assuming a single core config). From 5b4d23584fe5621ec51667283d57e526b0a60751 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 10 Feb 2021 19:54:30 -0800 Subject: [PATCH 011/123] Group modules to be retimed together (#57) --- src/main/scala/gemmini/AccumulatorMem.scala | 15 ++++++++-- .../gemmini/VectorScalarMultiplier.scala | 29 ++++++++++++------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index e218bd51d..b4ca52b67 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -106,8 +106,13 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve q.io.enq.bits.fromDMA := RegNext(io.read.req.bits.fromDMA) q.io.enq.valid := RegNext(io.read.req.fire()) - val p = Pipeline(q.io.deq, mem_pipeline, Seq.fill(mem_pipeline)((x: PipelinedRdataAndActT) => x) :+ { - x: PipelinedRdataAndActT => + class ScaleModule extends Module { + val io = IO(new Bundle { + val in = Flipped(Decoupled(new PipelinedRdataAndActT)) + val out = Decoupled(new PipelinedRdataAndActT) + }) + io.out <> Pipeline(io.in, mem_pipeline, Seq.fill(mem_pipeline)((x: PipelinedRdataAndActT) => x) :+ { + x: PipelinedRdataAndActT => val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => // val e_scaled = e >> x.shift val e_scaled = scale_args.scale_func(e, x.scale) @@ -123,7 +128,11 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve result.data := activated_rdata result - }) + }) + } + val scale_module = Module(new ScaleModule) + scale_module.io.in <> q.io.deq + val p = scale_module.io.out val q_will_be_empty = (q.io.count +& q.io.enq.fire()) - q.io.deq.fire() === 0.U io.read.req.ready := q_will_be_empty && ( diff --git a/src/main/scala/gemmini/VectorScalarMultiplier.scala b/src/main/scala/gemmini/VectorScalarMultiplier.scala index 4e86f61a8..d33ac0c22 100644 --- a/src/main/scala/gemmini/VectorScalarMultiplier.scala +++ b/src/main/scala/gemmini/VectorScalarMultiplier.scala @@ -42,12 +42,18 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](mvin_scale_args: val req = Reg(UDValid(chiselTypeOf(io.req.bits))) - io.req.ready := !req.valid || (req.bits.repeats === 0.U && io.resp.fire()) - io.resp.valid := req.valid - io.resp.bits.tag := req.bits.tag - io.resp.bits.last := req.bits.repeats === 0.U && req.bits.last - io.resp.bits.row := req.bits.repeats - io.resp.bits.out := (mvin_scale_args match { + val latency = mvin_scale_args match { + case Some(ScaleArguments(_, latency, _, _, _)) => latency + case None => 0 + } + val resp = Wire(Decoupled(new VectorScalarMultiplierResp(block_cols, t, tag_t))) + + io.req.ready := !req.valid || (req.bits.repeats === 0.U && resp.fire()) + resp.valid := req.valid + resp.bits.tag := req.bits.tag + resp.bits.last := req.bits.repeats === 0.U && req.bits.last + resp.bits.row := req.bits.repeats + resp.bits.out := (mvin_scale_args match { case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => req.bits.in.map(x => mvin_scale_func(x, req.bits.scale.asTypeOf(multiplicand_t))) @@ -56,7 +62,7 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](mvin_scale_args: when (io.req.fire()) { req.push(io.req.bits) - }.elsewhen(io.resp.fire()) { + }.elsewhen(resp.fire()) { when (req.bits.repeats === 0.U) { req.pop() }.otherwise { @@ -67,6 +73,10 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](mvin_scale_args: when (reset.toBool()) { req.pop() } + (mvin_scale_args match { + case Some(ScaleArguments(_, latency, _, _, _)) => io.resp <> Pipeline(resp, latency) + case None => io.resp <> resp + }) } object VectorScalarMultiplier { @@ -77,10 +87,7 @@ object VectorScalarMultiplier { val vsm = Module(new VectorScalarMultiplier(scale_args, cols, t, tag_t)) val in = vsm.io.req - val out = scale_args match { - case Some(ScaleArguments(_, latency, _, _, _)) => Pipeline(vsm.io.resp, latency) - case None => vsm.io.resp - } + val out = vsm.io.resp (in, out) } From bed471f460aefdc840fc24d9cf68ebedc306ef0b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 11 Feb 2021 10:15:31 -0800 Subject: [PATCH 012/123] Update ChipConfig --- src/main/scala/gemmini/Configs.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 7c01ce7d6..abcaaf87d 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -204,7 +204,11 @@ class DefaultGemminiChipConfig extends Config((site, here, up) => { case BuildRoCC => up(BuildRoCC) ++ Seq( (p: Parameters) => { implicit val q = p - val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), dataflow = Dataflow.WS, sp_singleported=false, max_in_flight_reqs=16))) + val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig.copy( + sp_capacity=CapacityInKilobytes(64), + acc_capacity=CapacityInKilobytes(32), + dataflow = Dataflow.WS + ))) gemmini } ) From 2f645add683a6169640e3cbbb2ad7368550bb650 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 12 Feb 2021 20:34:48 -0800 Subject: [PATCH 013/123] Add pipeline register after LoopMatmul units (#60) * Add pipeline register between LoopMatmul and ROB * Use a 2-entry queue instead * Monitor all pipeline registers for io.busy --- src/main/scala/gemmini/Controller.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 599e134df..e586a58b7 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -194,9 +194,10 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] // compressed_cmd.ready := false.B // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) + val unrolled_cmd = Queue(loop_cmd) unrolled_cmd.ready := false.B // val cmd_decompressor = Module(new InstDecompressor(rob_entries)) @@ -370,7 +371,7 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] rob_completed_arb.io.out.ready := true.B // Wire up global RoCC signals - io.busy := raw_cmd.valid || loop_conv_unroller_busy || loop_matmul_unroller_busy || rob.io.busy || spad.module.io.busy + io.busy := raw_cmd.valid || loop_conv_unroller_busy || loop_matmul_unroller_busy || rob.io.busy || spad.module.io.busy || unrolled_cmd.valid || loop_cmd.valid io.interrupt := tlb.io.exp.interrupt rob.io.solitary_preload := ex_controller.io.solitary_preload From 21ec295f6a1938d1d456807153f4ab4b9dcb90e7 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 13 Feb 2021 02:07:02 -0800 Subject: [PATCH 014/123] Time-multiplex acc-scale onto fewer FMAs --- src/main/scala/gemmini/AccumulatorMem.scala | 124 +++++++++++++++++--- src/main/scala/gemmini/Configs.scala | 2 +- src/main/scala/gemmini/DSEConfigs.scala | 1 + src/main/scala/gemmini/GemminiConfigs.scala | 1 + src/main/scala/gemmini/Scratchpad.scala | 6 +- 5 files changed, 115 insertions(+), 19 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index b4ca52b67..9af6fe684 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -50,7 +50,7 @@ class AccumulatorMemIO [T <: Data: Arithmetic, U <: Data](n: Int, t: Vec[Vec[T]] override def cloneType: this.type = new AccumulatorMemIO(n, t, rdata, scale_t).asInstanceOf[this.type] } -class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Vec[Vec[T]], mem_pipeline: Int, scale_args: ScaleArguments[T, U], read_small_data: Boolean, read_full_data: Boolean) +class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Vec[Vec[T]], mem_pipeline: Int, scale_args: ScaleArguments[T, U], read_small_data: Boolean, read_full_data: Boolean, num_scale_units: Int) (implicit ev: Arithmetic[T]) extends Module { // TODO Do writes in this module work with matrices of size 2? If we try to read from an address right after writing // to it, then we might not get the written data. We might need some kind of cooldown counter after addresses in the @@ -111,24 +111,114 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve val in = Flipped(Decoupled(new PipelinedRdataAndActT)) val out = Decoupled(new PipelinedRdataAndActT) }) - io.out <> Pipeline(io.in, mem_pipeline, Seq.fill(mem_pipeline)((x: PipelinedRdataAndActT) => x) :+ { - x: PipelinedRdataAndActT => - val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => - // val e_scaled = e >> x.shift - val e_scaled = scale_args.scale_func(e, x.scale) + + if (num_scale_units == -1) { + io.out <> Pipeline(io.in, mem_pipeline, Seq.fill(mem_pipeline)((x: PipelinedRdataAndActT) => x) :+ { + x: PipelinedRdataAndActT => + val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => + // val e_scaled = e >> x.shift + val e_scaled = scale_args.scale_func(e, x.scale) + val e_clipped = e_scaled.clippedToWidthOf(rdataType.head.head) + val e_act = MuxCase(e_clipped, Seq( + (x.act === Activation.RELU) -> e_clipped.relu, + (x.act === Activation.RELU6) -> e_clipped.relu6(x.relu6_shift))) + + e_act + }))) + val result = WireInit(x) + result.data := activated_rdata + + result + }) + } else { + val width = io.in.bits.data.size * io.in.bits.data(0).size + val nEntries = 3 + + val regs = Reg(Vec(nEntries, Valid(new PipelinedRdataAndActT))) + val fired_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + val completed_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + + val outArb = Module(new RRArbiter(new PipelinedRdataAndActT, nEntries)) + for (i <- 0 until nEntries) { + outArb.io.in(i).valid := regs(i).valid && completed_masks(i).reduce(_&&_) + outArb.io.in(i).bits := regs(i).bits + when (outArb.io.in(i).fire()) { regs(i).valid := false.B } + } + io.out <> outArb.io.out + + io.in.ready := !(regs.map(_.valid).reduce(_&&_)) || io.out.fire() + when (io.in.fire()) { + var allocated = false.B + for (i <- 0 until nEntries) { + when (!allocated && (!regs(i).valid || outArb.io.in(i).fire())) { + regs(i).valid := true.B + regs(i).bits := io.in.bits + fired_masks(i).foreach(_ := false.B) + completed_masks(i).foreach(_ := false.B) + } + allocated = allocated || !regs(i).valid || outArb.io.in(i).fire() + } + } + + + + class DataWithIndex extends Bundle { + val scale = io.in.bits.scale.cloneType + val act = io.in.bits.act.cloneType + val relu6_shift = io.in.bits.relu6_shift.cloneType + val data = io.in.bits.data(0)(0).cloneType + val id = UInt(2.W) // TODO hardcoded + val index = UInt() + } + val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } + + for (i <- 0 until nEntries) { + for (w <- 0 until width) { + val input = inputs(i*width+w) + input.valid := regs(i).valid && !fired_masks(i)(w) + input.bits.data := regs(i).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) + input.bits.scale := regs(i).bits.scale + input.bits.act := regs(i).bits.act + input.bits.relu6_shift := regs(i).bits.relu6_shift + input.bits.id := i.U + input.bits.index := w.U + when (input.fire()) { + fired_masks(i)(w) := true.B + } + } + } + for (i <- 0 until num_scale_units) { + val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) + val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) + arb.io.in <> arbIn + arb.io.out.ready := true.B + val arbOut = arb.io.out + val e_scaled = scale_args.scale_func(arbOut.bits.data, arbOut.bits.scale) val e_clipped = e_scaled.clippedToWidthOf(rdataType.head.head) val e_act = MuxCase(e_clipped, Seq( - (x.act === Activation.RELU) -> e_clipped.relu, - (x.act === Activation.RELU6) -> e_clipped.relu6(x.relu6_shift))) - - e_act - }))) - - val result = WireInit(x) - result.data := activated_rdata - - result - }) + (arbOut.bits.act === Activation.RELU) -> e_clipped.relu, + (arbOut.bits.act === Activation.RELU6) -> e_clipped.relu6(arbOut.bits.relu6_shift) + )) + val pipe_in = Wire(Valid(new DataWithIndex)) + pipe_in.valid := arbOut.valid + pipe_in.bits := arbOut.bits + pipe_in.bits.data := e_act + val pipe_out = Pipe(pipe_in, mem_pipeline) + for (j <- 0 until nEntries) { + for (w <- 0 until width) { + if ((j*width+w) % num_scale_units == i) { + when (pipe_out.fire() && pipe_out.bits.id === j.U && pipe_out.bits.index === w.U) { + regs(j).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) := pipe_out.bits.data + completed_masks(j)(w) := true.B + } + } + } + } + } + when (reset.asBool) { + regs.foreach(_.valid := false.B) + } + } } val scale_module = Module(new ScaleModule) scale_module.io.in <> q.io.deq diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index abcaaf87d..80bed4816 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -127,7 +127,7 @@ object GemminiConfigs { identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (elem_t)y);})" )), - + num_acc_scale_units = 4, mvin_scale_acc_args = None, mvin_scale_shared = false, diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index a0d0aa85b..5b532682c 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -52,6 +52,7 @@ object DSEBaseConfig { (t >> u).asSInt() + Mux(r, 1.S, 0.S) }, 0, UInt(8.W)), + num_acc_scale_units = -1, acc_read_full_width = true, acc_read_small_width = true, use_dedicated_tl_port = false, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 889928257..6732e7460 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -39,6 +39,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( mvin_scale_acc_args: Option[ScaleArguments[T, U]], mvin_scale_shared: Boolean, acc_scale_args: ScaleArguments[T, V], + num_acc_scale_units: Int, hasIm2col: Boolean, pe_latency: Int, acc_read_full_width: Boolean, diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 5f11cb0b9..fdfac887e 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -421,7 +421,11 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val acc_row_t = Vec(meshColumns, Vec(tileColumns, accType)) val spad_row_t = Vec(meshColumns, Vec(tileColumns, inputType)) - val banks = Seq.fill(acc_banks) { Module(new AccumulatorMem(acc_bank_entries, acc_row_t, spad_row_t, mem_pipeline, acc_scale_args, acc_read_small_width, acc_read_full_width)) } + val banks = Seq.fill(acc_banks) { Module(new AccumulatorMem( + acc_bank_entries, acc_row_t, spad_row_t, mem_pipeline, + acc_scale_args, acc_read_small_width, acc_read_full_width, + num_acc_scale_units + )) } val bank_ios = VecInit(banks.map(_.io)) // Getting the output of the bank that's about to be issued to the writer From 07171fdb319a7d257b03c545570114c462877beb Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 13 Feb 2021 02:27:30 -0800 Subject: [PATCH 015/123] Add new acc_scale_latency parameter for accumulator scale unit pipelining --- src/main/scala/gemmini/AccumulatorMem.scala | 6 +++--- src/main/scala/gemmini/Configs.scala | 1 + src/main/scala/gemmini/DSEConfigs.scala | 1 + src/main/scala/gemmini/GemminiConfigs.scala | 1 + src/main/scala/gemmini/Scratchpad.scala | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 9af6fe684..dc52cb1a0 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -50,7 +50,7 @@ class AccumulatorMemIO [T <: Data: Arithmetic, U <: Data](n: Int, t: Vec[Vec[T]] override def cloneType: this.type = new AccumulatorMemIO(n, t, rdata, scale_t).asInstanceOf[this.type] } -class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Vec[Vec[T]], mem_pipeline: Int, scale_args: ScaleArguments[T, U], read_small_data: Boolean, read_full_data: Boolean, num_scale_units: Int) +class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Vec[Vec[T]], acc_scale_latency: Int, scale_args: ScaleArguments[T, U], read_small_data: Boolean, read_full_data: Boolean, num_scale_units: Int) (implicit ev: Arithmetic[T]) extends Module { // TODO Do writes in this module work with matrices of size 2? If we try to read from an address right after writing // to it, then we might not get the written data. We might need some kind of cooldown counter after addresses in the @@ -113,7 +113,7 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve }) if (num_scale_units == -1) { - io.out <> Pipeline(io.in, mem_pipeline, Seq.fill(mem_pipeline)((x: PipelinedRdataAndActT) => x) :+ { + io.out <> Pipeline(io.in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: PipelinedRdataAndActT) => x) :+ { x: PipelinedRdataAndActT => val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => // val e_scaled = e >> x.shift @@ -203,7 +203,7 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve pipe_in.valid := arbOut.valid pipe_in.bits := arbOut.bits pipe_in.bits.data := e_act - val pipe_out = Pipe(pipe_in, mem_pipeline) + val pipe_out = Pipe(pipe_in, acc_scale_latency) for (j <- 0 until nEntries) { for (w <- 0 until width) { if ((j*width+w) % num_scale_units == i) { diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 80bed4816..7ae20cd3f 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -128,6 +128,7 @@ object GemminiConfigs { c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (elem_t)y);})" )), num_acc_scale_units = 4, + acc_scale_latency = 3, mvin_scale_acc_args = None, mvin_scale_shared = false, diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 5b532682c..39922b331 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -53,6 +53,7 @@ object DSEBaseConfig { (t >> u).asSInt() + Mux(r, 1.S, 0.S) }, 0, UInt(8.W)), num_acc_scale_units = -1, + acc_scale_latency = 3, acc_read_full_width = true, acc_read_small_width = true, use_dedicated_tl_port = false, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 6732e7460..564d43c24 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -40,6 +40,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( mvin_scale_shared: Boolean, acc_scale_args: ScaleArguments[T, V], num_acc_scale_units: Int, + acc_scale_latency: Int, hasIm2col: Boolean, pe_latency: Int, acc_read_full_width: Boolean, diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index fdfac887e..4ec7d62d6 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -422,7 +422,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val spad_row_t = Vec(meshColumns, Vec(tileColumns, inputType)) val banks = Seq.fill(acc_banks) { Module(new AccumulatorMem( - acc_bank_entries, acc_row_t, spad_row_t, mem_pipeline, + acc_bank_entries, acc_row_t, spad_row_t, acc_scale_latency, acc_scale_args, acc_read_small_width, acc_read_full_width, num_acc_scale_units )) } From 46099962f1c6113d25450f6209f45f24cce07740 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 13 Feb 2021 09:33:51 -0800 Subject: [PATCH 016/123] Accumulator mem must respond in-order --- src/main/scala/gemmini/AccumulatorMem.scala | 25 ++++++++++++--------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index dc52cb1a0..3dfbd2b7b 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -137,27 +137,30 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve val regs = Reg(Vec(nEntries, Valid(new PipelinedRdataAndActT))) val fired_masks = Reg(Vec(nEntries, Vec(width, Bool()))) val completed_masks = Reg(Vec(nEntries, Vec(width, Bool()))) - - val outArb = Module(new RRArbiter(new PipelinedRdataAndActT, nEntries)) - for (i <- 0 until nEntries) { - outArb.io.in(i).valid := regs(i).valid && completed_masks(i).reduce(_&&_) - outArb.io.in(i).bits := regs(i).bits - when (outArb.io.in(i).fire()) { regs(i).valid := false.B } + val head_oh = RegInit(1.U(nEntries.W)) + val tail_oh = RegInit(1.U(nEntries.W)) + io.out.valid := Mux1H(head_oh.asBools, (regs zip completed_masks).map({case (r, c) => r.valid && c.reduce(_&&_)})) + io.out.bits := Mux1H(head_oh.asBools, regs.map(_.bits)) + when (io.out.fire()) { + for (i <- 0 until nEntries) { + when (head_oh(i)) { + regs(i).valid := false.B + } + } + head_oh := (head_oh << 1) | head_oh(nEntries-1) } - io.out <> outArb.io.out - io.in.ready := !(regs.map(_.valid).reduce(_&&_)) || io.out.fire() + io.in.ready := !Mux1H(tail_oh.asBools, regs.map(_.valid)) || (tail_oh === head_oh && io.out.fire()) when (io.in.fire()) { - var allocated = false.B for (i <- 0 until nEntries) { - when (!allocated && (!regs(i).valid || outArb.io.in(i).fire())) { + when (tail_oh(i)) { regs(i).valid := true.B regs(i).bits := io.in.bits fired_masks(i).foreach(_ := false.B) completed_masks(i).foreach(_ := false.B) } - allocated = allocated || !regs(i).valid || outArb.io.in(i).fire() } + tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) } From 279718a3f6130b0036cce78c816e24fa48c86d3f Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 14 Feb 2021 03:10:19 -0800 Subject: [PATCH 017/123] Multiplex single accumulator scale unit between both accumulator banks --- src/main/scala/gemmini/AccumulatorMem.scala | 191 +++--------------- src/main/scala/gemmini/AccumulatorScale.scala | 181 +++++++++++++++++ src/main/scala/gemmini/Controller.scala | 3 +- .../scala/gemmini/ExecuteController.scala | 30 ++- src/main/scala/gemmini/Scratchpad.scala | 78 +++++-- 5 files changed, 290 insertions(+), 193 deletions(-) create mode 100644 src/main/scala/gemmini/AccumulatorScale.scala diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 3dfbd2b7b..5b25c1e06 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -17,19 +17,21 @@ class AccumulatorReadReq[T <: Data](n: Int, shift_width: Int, scale_t: T) extend override def cloneType: this.type = new AccumulatorReadReq(n, shift_width, scale_t.cloneType).asInstanceOf[this.type] } -class AccumulatorReadResp[T <: Data: Arithmetic](rdataType: Vec[Vec[T]], fullDataType: Vec[Vec[T]]) extends Bundle { - val data = rdataType.cloneType - val full_data = fullDataType.cloneType +class AccumulatorReadResp[T <: Data: Arithmetic, U <: Data](fullDataType: Vec[Vec[T]], scale_t: U, shift_width: Int) extends Bundle { + val data = fullDataType.cloneType val fromDMA = Bool() - - override def cloneType: this.type = new AccumulatorReadResp(rdataType.cloneType, fullDataType.cloneType).asInstanceOf[this.type] + val scale = scale_t.cloneType + val relu6_shift = UInt(shift_width.W) + val act = UInt(2.W) + val acc_bank_id = UInt(2.W) // TODO don't hardcode + override def cloneType: this.type = new AccumulatorReadResp(fullDataType.cloneType, scale_t, shift_width).asInstanceOf[this.type] } -class AccumulatorReadIO[T <: Data: Arithmetic, U <: Data](n: Int, shift_width: Int, rdataType: Vec[Vec[T]], fullDataType: Vec[Vec[T]], scale_t: U) extends Bundle { - val req = Decoupled(new AccumulatorReadReq(n, shift_width, scale_t)) - val resp = Flipped(Decoupled(new AccumulatorReadResp(rdataType.cloneType, fullDataType.cloneType))) +class AccumulatorReadIO[T <: Data: Arithmetic, U <: Data](n: Int, shift_width: Int, fullDataType: Vec[Vec[T]], scale_t: U) extends Bundle { + val req = Decoupled(new AccumulatorReadReq[U](n, shift_width, scale_t)) + val resp = Flipped(Decoupled(new AccumulatorReadResp[T, U](fullDataType, scale_t, shift_width))) - override def cloneType: this.type = new AccumulatorReadIO(n, shift_width, rdataType.cloneType, fullDataType.cloneType, scale_t.cloneType).asInstanceOf[this.type] + override def cloneType: this.type = new AccumulatorReadIO(n, shift_width, fullDataType.cloneType, scale_t.cloneType).asInstanceOf[this.type] } class AccumulatorWriteReq[T <: Data: Arithmetic](n: Int, t: Vec[Vec[T]]) extends Bundle { @@ -42,16 +44,17 @@ class AccumulatorWriteReq[T <: Data: Arithmetic](n: Int, t: Vec[Vec[T]]) extends override def cloneType: this.type = new AccumulatorWriteReq(n, t).asInstanceOf[this.type] } -class AccumulatorMemIO [T <: Data: Arithmetic, U <: Data](n: Int, t: Vec[Vec[T]], rdata: Vec[Vec[T]], scale_t: U) extends Bundle { - val read = Flipped(new AccumulatorReadIO(n, log2Ceil(t.head.head.getWidth), rdata, t, scale_t)) +class AccumulatorMemIO [T <: Data: Arithmetic, U <: Data](n: Int, t: Vec[Vec[T]], scale_t: U) extends Bundle { + val read = Flipped(new AccumulatorReadIO(n, log2Ceil(t.head.head.getWidth), t, scale_t)) // val write = Flipped(new AccumulatorWriteIO(n, t)) val write = Flipped(Decoupled(new AccumulatorWriteReq(n, t))) - override def cloneType: this.type = new AccumulatorMemIO(n, t, rdata, scale_t).asInstanceOf[this.type] + override def cloneType: this.type = new AccumulatorMemIO(n, t, scale_t).asInstanceOf[this.type] } -class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Vec[Vec[T]], acc_scale_latency: Int, scale_args: ScaleArguments[T, U], read_small_data: Boolean, read_full_data: Boolean, num_scale_units: Int) - (implicit ev: Arithmetic[T]) extends Module { +class AccumulatorMem[T <: Data, U <: Data]( + n: Int, t: Vec[Vec[T]], scale_args: ScaleArguments[T, U]) + (implicit ev: Arithmetic[T]) extends Module { // TODO Do writes in this module work with matrices of size 2? If we try to read from an address right after writing // to it, then we might not get the written data. We might need some kind of cooldown counter after addresses in the // accumulator have been written to for configurations with such small matrices @@ -64,7 +67,7 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve import ev._ // TODO unify this with TwoPortSyncMemIO - val io = IO(new AccumulatorMemIO(n, t, rdataType, scale_args.multiplicand_t)) + val io = IO(new AccumulatorMemIO(n, t, scale_args.multiplicand_t)) val mem = TwoPortSyncMem(n, t, t.getWidth / 8) // TODO We assume byte-alignment here. Use aligned_to instead @@ -88,144 +91,27 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve mem.io.raddr := Mux(io.write.fire() && io.write.bits.acc, io.write.bits.addr, io.read.req.bits.addr) mem.io.ren := io.read.req.fire() || (io.write.fire() && io.write.bits.acc) - class PipelinedRdataAndActT extends Bundle { - val data = mem.io.rdata.cloneType - val full_data = mem.io.rdata.cloneType - val scale = io.read.req.bits.scale.cloneType - val relu6_shift = io.read.req.bits.relu6_shift.cloneType - val act = io.read.req.bits.act.cloneType - val fromDMA = io.read.req.bits.fromDMA.cloneType - } - - val q = Module(new Queue(new PipelinedRdataAndActT, 1, true, true)) + val q = Module(new Queue(new AccumulatorReadResp(t, scale_args.multiplicand_t, log2Ceil(t.head.head.getWidth)), 1, true, true)) q.io.enq.bits.data := mem.io.rdata - q.io.enq.bits.full_data := mem.io.rdata q.io.enq.bits.scale := RegNext(io.read.req.bits.scale) q.io.enq.bits.relu6_shift := RegNext(io.read.req.bits.relu6_shift) q.io.enq.bits.act := RegNext(io.read.req.bits.act) q.io.enq.bits.fromDMA := RegNext(io.read.req.bits.fromDMA) + q.io.enq.bits.acc_bank_id := DontCare q.io.enq.valid := RegNext(io.read.req.fire()) - class ScaleModule extends Module { - val io = IO(new Bundle { - val in = Flipped(Decoupled(new PipelinedRdataAndActT)) - val out = Decoupled(new PipelinedRdataAndActT) - }) - - if (num_scale_units == -1) { - io.out <> Pipeline(io.in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: PipelinedRdataAndActT) => x) :+ { - x: PipelinedRdataAndActT => - val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => - // val e_scaled = e >> x.shift - val e_scaled = scale_args.scale_func(e, x.scale) - val e_clipped = e_scaled.clippedToWidthOf(rdataType.head.head) - val e_act = MuxCase(e_clipped, Seq( - (x.act === Activation.RELU) -> e_clipped.relu, - (x.act === Activation.RELU6) -> e_clipped.relu6(x.relu6_shift))) - - e_act - }))) - val result = WireInit(x) - result.data := activated_rdata - - result - }) - } else { - val width = io.in.bits.data.size * io.in.bits.data(0).size - val nEntries = 3 - - val regs = Reg(Vec(nEntries, Valid(new PipelinedRdataAndActT))) - val fired_masks = Reg(Vec(nEntries, Vec(width, Bool()))) - val completed_masks = Reg(Vec(nEntries, Vec(width, Bool()))) - val head_oh = RegInit(1.U(nEntries.W)) - val tail_oh = RegInit(1.U(nEntries.W)) - io.out.valid := Mux1H(head_oh.asBools, (regs zip completed_masks).map({case (r, c) => r.valid && c.reduce(_&&_)})) - io.out.bits := Mux1H(head_oh.asBools, regs.map(_.bits)) - when (io.out.fire()) { - for (i <- 0 until nEntries) { - when (head_oh(i)) { - regs(i).valid := false.B - } - } - head_oh := (head_oh << 1) | head_oh(nEntries-1) - } - - io.in.ready := !Mux1H(tail_oh.asBools, regs.map(_.valid)) || (tail_oh === head_oh && io.out.fire()) - when (io.in.fire()) { - for (i <- 0 until nEntries) { - when (tail_oh(i)) { - regs(i).valid := true.B - regs(i).bits := io.in.bits - fired_masks(i).foreach(_ := false.B) - completed_masks(i).foreach(_ := false.B) - } - } - tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) - } - - - - class DataWithIndex extends Bundle { - val scale = io.in.bits.scale.cloneType - val act = io.in.bits.act.cloneType - val relu6_shift = io.in.bits.relu6_shift.cloneType - val data = io.in.bits.data(0)(0).cloneType - val id = UInt(2.W) // TODO hardcoded - val index = UInt() - } - val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } - - for (i <- 0 until nEntries) { - for (w <- 0 until width) { - val input = inputs(i*width+w) - input.valid := regs(i).valid && !fired_masks(i)(w) - input.bits.data := regs(i).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) - input.bits.scale := regs(i).bits.scale - input.bits.act := regs(i).bits.act - input.bits.relu6_shift := regs(i).bits.relu6_shift - input.bits.id := i.U - input.bits.index := w.U - when (input.fire()) { - fired_masks(i)(w) := true.B - } - } - } - for (i <- 0 until num_scale_units) { - val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) - val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) - arb.io.in <> arbIn - arb.io.out.ready := true.B - val arbOut = arb.io.out - val e_scaled = scale_args.scale_func(arbOut.bits.data, arbOut.bits.scale) - val e_clipped = e_scaled.clippedToWidthOf(rdataType.head.head) - val e_act = MuxCase(e_clipped, Seq( - (arbOut.bits.act === Activation.RELU) -> e_clipped.relu, - (arbOut.bits.act === Activation.RELU6) -> e_clipped.relu6(arbOut.bits.relu6_shift) - )) - val pipe_in = Wire(Valid(new DataWithIndex)) - pipe_in.valid := arbOut.valid - pipe_in.bits := arbOut.bits - pipe_in.bits.data := e_act - val pipe_out = Pipe(pipe_in, acc_scale_latency) - for (j <- 0 until nEntries) { - for (w <- 0 until width) { - if ((j*width+w) % num_scale_units == i) { - when (pipe_out.fire() && pipe_out.bits.id === j.U && pipe_out.bits.index === w.U) { - regs(j).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) := pipe_out.bits.data - completed_masks(j)(w) := true.B - } - } - } - } - } - when (reset.asBool) { - regs.foreach(_.valid := false.B) - } - } - } - val scale_module = Module(new ScaleModule) - scale_module.io.in <> q.io.deq - val p = scale_module.io.out + + val p = q.io.deq + + io.read.resp.bits.data := p.bits.data + io.read.resp.bits.fromDMA := p.bits.fromDMA + io.read.resp.bits.relu6_shift := p.bits.relu6_shift + io.read.resp.bits.act := p.bits.act + io.read.resp.bits.scale := p.bits.scale + io.read.resp.bits.acc_bank_id := DontCare // This is set in Scratchpad + io.read.resp.valid := p.valid + p.ready := io.read.resp.ready + val q_will_be_empty = (q.io.count +& q.io.enq.fire()) - q.io.deq.fire() === 0.U io.read.req.ready := q_will_be_empty && ( @@ -235,21 +121,8 @@ class AccumulatorMem[T <: Data, U <: Data](n: Int, t: Vec[Vec[T]], rdataType: Ve !(RegNext(io.write.fire()) && RegNext(io.write.bits.addr) === io.read.req.bits.addr) && !(w_buf_valid && waddr_buf === io.read.req.bits.addr) ) - io.read.resp.bits.data := p.bits.data - io.read.resp.bits.full_data := p.bits.full_data - io.read.resp.bits.fromDMA := p.bits.fromDMA - io.read.resp.valid := p.valid - p.ready := io.read.resp.ready - if (read_small_data) - io.read.resp.bits.data := p.bits.data - else - io.read.resp.bits.data := 0.U.asTypeOf(p.bits.data) // TODO make this DontCare instead - if (read_full_data) - io.read.resp.bits.full_data := p.bits.full_data - else - io.read.resp.bits.full_data := 0.U.asTypeOf(q.io.enq.bits.full_data) // TODO make this DontCare instead // io.write.current_waddr.valid := mem.io.wen // io.write.current_waddr.bits := mem.io.waddr diff --git a/src/main/scala/gemmini/AccumulatorScale.scala b/src/main/scala/gemmini/AccumulatorScale.scala new file mode 100644 index 000000000..415688c46 --- /dev/null +++ b/src/main/scala/gemmini/AccumulatorScale.scala @@ -0,0 +1,181 @@ +package gemmini + +import chisel3._ +import chisel3.util._ + +import Util._ + +class AccumulatorScaleResp[T <: Data: Arithmetic](fullDataType: Vec[Vec[T]], rDataType: Vec[Vec[T]]) extends Bundle { + val full_data = fullDataType.cloneType + val data = rDataType.cloneType + val acc_bank_id = UInt(2.W) + val fromDMA = Bool() + override def cloneType: this.type = new AccumulatorScaleResp(fullDataType, rDataType).asInstanceOf[this.type] +} + +class AccumulatorScaleIO[T <: Data: Arithmetic, U <: Data]( + fullDataType: Vec[Vec[T]], scale_t: U, shift_width: Int, + rDataType: Vec[Vec[T]] +) extends Bundle { + val in = Flipped(Decoupled(new AccumulatorReadResp[T,U](fullDataType, scale_t, shift_width))) + val out = Decoupled(new AccumulatorScaleResp[T](fullDataType, rDataType)) + override def cloneType: this.type = new AccumulatorScaleIO(fullDataType, scale_t, + shift_width, rDataType).asInstanceOf[this.type] +} + +class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( + fullDataType: Vec[Vec[T]], rDataType: Vec[Vec[T]], + scale_t: U, shift_width: Int, + num_scale_units: Int, acc_scale_latency: Int, + read_small_data: Boolean, read_full_data: Boolean, + scale_args: ScaleArguments[T, U])(implicit ev: Arithmetic[T]) extends Module { + + import ev._ + val io = IO(new AccumulatorScaleIO[T,U]( + fullDataType, scale_t, shift_width, rDataType + )(ev)) + + val out = Wire(Decoupled(new AccumulatorScaleResp[T]( + fullDataType, rDataType)(ev))) + + + + if (num_scale_units == -1) { + val pipe_out = Pipeline(io.in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: AccumulatorReadResp[T,U]) => x) :+ { + x: AccumulatorReadResp[T,U] => + val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => + // val e_scaled = e >> x.shiftls + val e_scaled = scale_args.scale_func(e, x.scale) + val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) + val e_act = MuxCase(e_clipped, Seq( + (x.act === Activation.RELU) -> e_clipped.relu, + (x.act === Activation.RELU6) -> e_clipped.relu6(x.relu6_shift))) + + e_act + }))) + val result = WireInit(x) + result.data := activated_rdata + result + }) + out.valid := pipe_out.valid + pipe_out.ready := out.ready + out.bits.full_data := pipe_out.bits.data + out.bits.data := pipe_out.bits.data + out.bits.fromDMA := pipe_out.bits.fromDMA + out.bits.acc_bank_id := pipe_out.bits.acc_bank_id + } else { + val width = io.in.bits.data.size * io.in.bits.data(0).size + val nEntries = 3 + val regs = Reg(Vec(nEntries, Valid(new AccumulatorReadResp[T,U]( + fullDataType, scale_t, shift_width)(ev)))) + val out_regs = Reg(Vec(nEntries, new AccumulatorScaleResp[T]( + fullDataType, rDataType)(ev))) + + val fired_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + val completed_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + val head_oh = RegInit(1.U(nEntries.W)) + val tail_oh = RegInit(1.U(nEntries.W)) + out.valid := Mux1H(head_oh.asBools, (regs zip completed_masks).map({case (r, c) => r.valid && c.reduce(_&&_)})) + out.bits := Mux1H(head_oh.asBools, out_regs) + when (out.fire()) { + for (i <- 0 until nEntries) { + when (head_oh(i)) { + regs(i).valid := false.B + } + } + head_oh := (head_oh << 1) | head_oh(nEntries-1) + } + + io.in.ready := !Mux1H(tail_oh.asBools, regs.map(_.valid)) || (tail_oh === head_oh && out.fire()) + when (io.in.fire()) { + for (i <- 0 until nEntries) { + when (tail_oh(i)) { + regs(i).valid := true.B + regs(i).bits := io.in.bits + out_regs(i).fromDMA := io.in.bits.fromDMA + out_regs(i).acc_bank_id := io.in.bits.acc_bank_id + fired_masks(i).foreach(_ := false.B) + completed_masks(i).foreach(_ := false.B) + } + } + tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) + } + + + class DataWithIndex extends Bundle { + val scale = io.in.bits.scale.cloneType + val act = io.in.bits.act.cloneType + val relu6_shift = io.in.bits.relu6_shift.cloneType + val data = io.in.bits.data(0)(0).cloneType + val full_data = io.in.bits.data(0)(0).cloneType + val id = UInt(2.W) // TODO hardcoded + val index = UInt() + } + val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } + + for (i <- 0 until nEntries) { + for (w <- 0 until width) { + val input = inputs(i*width+w) + input.valid := regs(i).valid && !fired_masks(i)(w) + input.bits.data := regs(i).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) + input.bits.full_data := regs(i).bits.data(w / io.in.bits.data(0).size)(w % io.in.bits.data(0).size) + input.bits.scale := regs(i).bits.scale + input.bits.act := regs(i).bits.act + input.bits.relu6_shift := regs(i).bits.relu6_shift + input.bits.id := i.U + input.bits.index := w.U + when (input.fire()) { + fired_masks(i)(w) := true.B + } + } + } + for (i <- 0 until num_scale_units) { + val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) + val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) + arb.io.in <> arbIn + arb.io.out.ready := true.B + val arbOut = arb.io.out + val e_scaled = scale_args.scale_func(arbOut.bits.data, arbOut.bits.scale) + val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) + val e_act = MuxCase(e_clipped, Seq( + (arbOut.bits.act === Activation.RELU) -> e_clipped.relu, + (arbOut.bits.act === Activation.RELU6) -> e_clipped.relu6(arbOut.bits.relu6_shift) + )) + val pipe_in = Wire(Valid(new DataWithIndex)) + pipe_in.valid := arbOut.valid + pipe_in.bits := arbOut.bits + pipe_in.bits.data := e_act + val pipe_out = Pipe(pipe_in, acc_scale_latency) + for (j <- 0 until nEntries) { + for (w <- 0 until width) { + if ((j*width+w) % num_scale_units == i) { + val id0 = w % io.in.bits.data(0).size + val id1 = w / io.in.bits.data(0).size + when (pipe_out.fire() && pipe_out.bits.id === j.U && pipe_out.bits.index === w.U) { + out_regs(j).data (id1)(id0) := pipe_out.bits.data + out_regs(j).full_data(id1)(id0) := pipe_out.bits.full_data + completed_masks(j)(w) := true.B + } + } + } + } + } + when (reset.asBool) { + regs.foreach(_.valid := false.B) + } + } + + io.out <> out + + if (read_small_data) + io.out.bits.data := out.bits.data + else + io.out.bits.data := DontCare + + if (read_full_data) + io.out.bits.full_data := out.bits.full_data + else + io.out.bits.full_data := DontCare + +} + diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index e586a58b7..5ce573fd1 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -296,7 +296,8 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] spad.module.io.dma.write <> store_controller.io.dma ex_controller.io.srams.read <> spad.module.io.srams.read ex_controller.io.srams.write <> spad.module.io.srams.write - ex_controller.io.acc.read <> spad.module.io.acc.read + spad.module.io.acc.read_req <> ex_controller.io.acc.read_req + ex_controller.io.acc.read_resp <> spad.module.io.acc.read_resp ex_controller.io.acc.write <> spad.module.io.acc.write // Im2Col unit diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index 4f7994ddc..e884aa597 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -27,7 +27,15 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } val acc = new Bundle { - val read = Vec(acc_banks, new AccumulatorReadIO(acc_bank_entries, log2Up(accType.getWidth), Vec(meshColumns, Vec(tileColumns, inputType)), Vec(meshColumns, Vec(tileColumns, accType)), acc_scale_args.multiplicand_t)) + val read_req = Vec(acc_banks, Decoupled(new AccumulatorReadReq( + acc_bank_entries, log2Up(accType.getWidth), acc_scale_args.multiplicand_t + ))) + + val read_resp = Flipped(Vec(acc_banks, Decoupled(new AccumulatorScaleResp( + Vec(meshColumns, Vec(tileColumns, inputType)), + Vec(meshColumns, Vec(tileColumns, accType)) + )))) + // val write = Vec(acc_banks, new AccumulatorWriteIO(acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType)))) val write = Vec(acc_banks, Decoupled(new AccumulatorWriteReq(acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType))))) } @@ -424,7 +432,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val read_d_from_acc = d_valid && d_read_from_acc && dataDBankAcc === i.U && start_inputting_d && !preload_zeros && d_row_is_not_all_zeros //&& !im2col_wire Seq((read_a_from_acc, a_ready), (read_b_from_acc, b_ready), (read_d_from_acc, d_ready)).foreach { case (rd, r) => - when(rd && !io.acc.read(i).req.ready) { + when(rd && !io.acc.read_req(i).ready) { r := false.B } } @@ -448,21 +456,21 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In */ // TODO Remove the ability to read into Mesh from AccumulatorMem completely - io.acc.read(i).req.valid := false.B - io.acc.read(i).req.bits.scale := acc_scale - io.acc.read(i).req.bits.full := false.B - io.acc.read(i).req.bits.relu6_shift := relu6_shift - io.acc.read(i).req.bits.act := activation - io.acc.read(i).req.bits.fromDMA := false.B - io.acc.read(i).req.bits.addr := DontCare + io.acc.read_req(i).valid := false.B + io.acc.read_req(i).bits.scale := acc_scale + io.acc.read_req(i).bits.full := false.B + io.acc.read_req(i).bits.relu6_shift := relu6_shift + io.acc.read_req(i).bits.act := activation + io.acc.read_req(i).bits.fromDMA := false.B + io.acc.read_req(i).bits.addr := DontCare when(im2col_en === false.B){ - io.acc.read(i).req.bits.addr := MuxCase(a_address.acc_row(), + io.acc.read_req(i).bits.addr := MuxCase(a_address.acc_row(), Seq(read_b_from_acc -> b_address.acc_row(), read_d_from_acc -> d_address.acc_row())) } - io.acc.read(i).resp.ready := true.B + io.acc.read_resp(i).ready := true.B } // Im2Col reads diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 4ec7d62d6..42156f893 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -186,8 +186,16 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // Accumulator ports val acc = new Bundle { - val read = Flipped(Vec(acc_banks, new AccumulatorReadIO(acc_bank_entries, log2Up(accType.getWidth), Vec(meshColumns, Vec(tileColumns, inputType)), Vec(meshColumns, Vec(tileColumns, accType)), acc_scale_args.multiplicand_t))) - val write = Flipped(Vec(acc_banks, Decoupled(new AccumulatorWriteReq(acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType)))))) + val read_req = Flipped(Vec(acc_banks, Decoupled(new AccumulatorReadReq( + acc_bank_entries, log2Up(accType.getWidth), acc_scale_args.multiplicand_t + )))) + val read_resp = Vec(acc_banks, Decoupled(new AccumulatorScaleResp( + Vec(meshColumns, Vec(tileColumns, inputType)), + Vec(meshColumns, Vec(tileColumns, accType)) + ))) + val write = Flipped(Vec(acc_banks, Decoupled(new AccumulatorWriteReq( + acc_bank_entries, Vec(meshColumns, Vec(tileColumns, accType)) + )))) } // TLB ports @@ -417,29 +425,61 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, } } + val acc_row_t = Vec(meshColumns, Vec(tileColumns, accType)) + val spad_row_t = Vec(meshColumns, Vec(tileColumns, inputType)) + + val acc_scale_unit = Module(new AccumulatorScale( + acc_row_t, + spad_row_t, + acc_scale_args.multiplicand_t, + log2Up(accType.getWidth), + num_acc_scale_units, + acc_scale_latency, + acc_read_small_width, + acc_read_full_width, + acc_scale_args + )) + val acc_scale_arb = Module(new RRArbiter(new AccumulatorReadResp( + acc_row_t, + acc_scale_args.multiplicand_t, + log2Up(accType.getWidth) + ), acc_banks)) + + acc_scale_unit.io.in <> acc_scale_arb.io.out + val dma_resp_ready = ( + writer.module.io.req.ready && + write_issue_q.io.deq.bits.laddr.is_acc_addr && + !write_issue_q.io.deq.bits.laddr.is_garbage() + ) + acc_scale_unit.io.out.ready := false.B + when (acc_scale_unit.io.out.bits.fromDMA && dma_resp_ready) { + acc_scale_unit.io.out.ready := true.B + writeData.valid := acc_scale_unit.io.out.valid + writeData.bits := acc_scale_unit.io.out.bits.data.asUInt + fullAccWriteData := acc_scale_unit.io.out.bits.full_data.asUInt + } + for (i <- 0 until acc_banks) { + io.acc.read_resp(i).valid := false.B + io.acc.read_resp(i).bits := acc_scale_unit.io.out.bits + when (!acc_scale_unit.io.out.bits.fromDMA && acc_scale_unit.io.out.bits.acc_bank_id === i.U) { + acc_scale_unit.io.out.ready := io.acc.read_resp(i).ready + io.acc.read_resp(i).valid := acc_scale_unit.io.out.valid + } + } + { - val acc_row_t = Vec(meshColumns, Vec(tileColumns, accType)) - val spad_row_t = Vec(meshColumns, Vec(tileColumns, inputType)) val banks = Seq.fill(acc_banks) { Module(new AccumulatorMem( - acc_bank_entries, acc_row_t, spad_row_t, acc_scale_latency, - acc_scale_args, acc_read_small_width, acc_read_full_width, - num_acc_scale_units + acc_bank_entries, acc_row_t, acc_scale_args )) } val bank_ios = VecInit(banks.map(_.io)) // Getting the output of the bank that's about to be issued to the writer val bank_issued_io = bank_ios(write_issue_q.io.deq.bits.laddr.acc_bank()) - when (write_issue_q.io.deq.bits.laddr.is_acc_addr) { - writeData.valid := bank_issued_io.read.resp.valid && bank_issued_io.read.resp.bits.fromDMA - writeData.bits := bank_issued_io.read.resp.bits.data.asUInt() - fullAccWriteData := bank_issued_io.read.resp.bits.full_data.asUInt() - } - // Reading from the Accumulator banks bank_ios.zipWithIndex.foreach { case (bio, i) => - val ex_read_req = io.acc.read(i).req + val ex_read_req = io.acc.read_req(i) val exread = ex_read_req.valid // TODO we tie the write dispatch queue's, and write issue queue's, ready and valid signals together here @@ -472,14 +512,8 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, bio.read.req.bits := DontCare } - val ex_read_resp = io.acc.read(i).resp - val dma_resp_ready = writer.module.io.req.ready && - write_issue_q.io.deq.bits.laddr.is_acc_addr && write_issue_q.io.deq.bits.laddr.acc_bank() === i.U && // I believe we don't need to check that write_issue_q is valid here, because if the accumulator bank's resp is valid, then that means that the write_issue_q's deq should also be valid - !write_issue_q.io.deq.bits.laddr.is_garbage() - - bio.read.resp.ready := Mux(bio.read.resp.bits.fromDMA, dma_resp_ready, ex_read_resp.ready) - ex_read_resp.valid := bio.read.resp.valid // TODO should we AND this with fromDMA? - ex_read_resp.bits := bio.read.resp.bits + acc_scale_arb.io.in(i) <> bio.read.resp + acc_scale_arb.io.in(i).bits.acc_bank_id := i.U } // Writing to the accumulator banks From c9e9b1a3fa8a5bfc9c0af495d80cfe6148119bb8 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sun, 14 Feb 2021 21:02:00 -0800 Subject: [PATCH 018/123] Maintain in-order accesses to AccScale unit --- src/main/scala/gemmini/Scratchpad.scala | 79 ++++++++++++++++++------- 1 file changed, 56 insertions(+), 23 deletions(-) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 42156f893..9b7887872 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -207,17 +207,34 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, }) val write_dispatch_q = Queue(io.dma.write.req) - write_dispatch_q.ready := false.B - + // Write scale queue is necessary to maintain in-order requests to accumulator scale unit + // Writes from main SPAD just flow directly between scale_q and issue_q, while writes + // From acc are ordered + val write_scale_q = Module(new Queue(new ScratchpadMemWriteRequest(local_addr_t), mem_pipeline)) val write_issue_q = Module(new Queue(new ScratchpadMemWriteRequest(local_addr_t), mem_pipeline+1, pipe=true)) val read_issue_q = Module(new Queue(new ScratchpadMemReadRequest(local_addr_t, mvin_scale_t_bits), mem_pipeline+1, pipe=true)) // TODO can't this just be a normal queue? + write_scale_q.io.enq.valid := false.B + write_scale_q.io.enq.bits := write_dispatch_q.bits + write_scale_q.io.deq.ready := false.B + write_issue_q.io.enq.valid := false.B - write_issue_q.io.enq.bits := write_dispatch_q.bits + write_issue_q.io.enq.bits := write_scale_q.io.deq.bits + + + // Garbage can immediately fire between dispatch_q and scale_q + when (write_dispatch_q.bits.laddr.is_garbage()) { + write_scale_q.io.enq <> write_dispatch_q + } + // Non-acc or garbage can immediately fire between scale_q and issue_q + when (write_scale_q.io.deq.bits.laddr.is_garbage() || !write_scale_q.io.deq.bits.laddr.is_acc_addr) { + write_issue_q.io.enq <> write_scale_q.io.deq + } + val writeData = Wire(Valid(UInt((spad_w max acc_w).W))) - writeData.valid := false.B + writeData.valid := write_issue_q.io.deq.bits.laddr.is_garbage() writeData.bits := DontCare val fullAccWriteData = Wire(UInt(acc_w.W)) fullAccWriteData := DontCare @@ -225,8 +242,8 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, write_issue_q.io.deq.bits.laddr.is_acc_addr && write_issue_q.io.deq.bits.laddr.read_full_acc_row val writeData_is_all_zeros = write_issue_q.io.deq.bits.laddr.is_garbage() - writer.module.io.req.valid := write_issue_q.io.deq.valid && (writeData.valid || writeData_is_all_zeros) - write_issue_q.io.deq.ready := writer.module.io.req.ready && (writeData.valid || writeData_is_all_zeros) + writer.module.io.req.valid := write_issue_q.io.deq.valid && writeData.valid + write_issue_q.io.deq.ready := writer.module.io.req.ready && writeData.valid writer.module.io.req.bits.vaddr := write_issue_q.io.deq.bits.vaddr writer.module.io.req.bits.len := Mux(writeData_is_full_width, write_issue_q.io.deq.bits.len * (accType.getWidth / 8).U, @@ -241,6 +258,9 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, io.dma.write.resp.valid := false.B io.dma.write.resp.bits.cmd_id := write_dispatch_q.bits.cmd_id + when (write_dispatch_q.bits.laddr.is_garbage() && write_dispatch_q.fire()) { + io.dma.write.resp.valid := true.B + } read_issue_q.io.enq <> io.dma.read.req @@ -329,7 +349,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, writer.module.io.flush := io.flush reader.module.io.flush := io.flush - io.busy := writer.module.io.busy || reader.module.io.busy || write_issue_q.io.deq.valid + io.busy := writer.module.io.busy || reader.module.io.busy || write_issue_q.io.deq.valid || write_scale_q.io.deq.valid || write_dispatch_q.valid { val banks = Seq.fill(sp_banks) { Module(new ScratchpadBank(sp_bank_entries, spad_w, mem_pipeline, aligned_to, config.sp_singleported)) } @@ -349,11 +369,12 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val exread = ex_read_req.valid // TODO we tie the write dispatch queue's, and write issue queue's, ready and valid signals together here - val dmawrite = write_dispatch_q.valid && write_issue_q.io.enq.ready && + val dmawrite = write_dispatch_q.valid && write_scale_q.io.enq.ready && + !write_dispatch_q.bits.laddr.is_garbage() && !(bio.write.en && config.sp_singleported.B) && !write_dispatch_q.bits.laddr.is_acc_addr && write_dispatch_q.bits.laddr.sp_bank() === i.U - bio.read.req.valid := exread || (dmawrite && !write_dispatch_q.bits.laddr.is_garbage()) + bio.read.req.valid := exread || dmawrite ex_read_req.ready := bio.read.req.ready // The ExecuteController gets priority when reading from SRAMs @@ -364,9 +385,9 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, bio.read.req.bits.addr := write_dispatch_q.bits.laddr.sp_row() bio.read.req.bits.fromDMA := true.B - when (bio.read.req.fire() || write_dispatch_q.bits.laddr.is_garbage()) { + when (bio.read.req.fire()) { write_dispatch_q.ready := true.B - write_issue_q.io.enq.valid := true.B + write_scale_q.io.enq.valid := true.B io.dma.write.resp.valid := true.B } @@ -439,13 +460,9 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, acc_read_full_width, acc_scale_args )) - val acc_scale_arb = Module(new RRArbiter(new AccumulatorReadResp( - acc_row_t, - acc_scale_args.multiplicand_t, - log2Up(accType.getWidth) - ), acc_banks)) - acc_scale_unit.io.in <> acc_scale_arb.io.out + acc_scale_unit.io.in.valid := false.B + acc_scale_unit.io.in.bits := DontCare val dma_resp_ready = ( writer.module.io.req.ready && write_issue_q.io.deq.bits.laddr.is_acc_addr && @@ -483,10 +500,11 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val exread = ex_read_req.valid // TODO we tie the write dispatch queue's, and write issue queue's, ready and valid signals together here - val dmawrite = write_dispatch_q.valid && write_issue_q.io.enq.ready && + val dmawrite = write_dispatch_q.valid && write_scale_q.io.enq.ready && + !write_dispatch_q.bits.laddr.is_garbage() && write_dispatch_q.bits.laddr.is_acc_addr && write_dispatch_q.bits.laddr.acc_bank() === i.U - bio.read.req.valid := exread || (dmawrite && !write_dispatch_q.bits.laddr.is_garbage()) + bio.read.req.valid := exread || dmawrite bio.read.req.bits.scale := ex_read_req.bits.scale bio.read.req.bits.relu6_shift := ex_read_req.bits.relu6_shift bio.read.req.bits.act := ex_read_req.bits.act @@ -502,18 +520,33 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, bio.read.req.bits.full := write_dispatch_q.bits.laddr.read_full_acc_row bio.read.req.bits.fromDMA := true.B - when (bio.read.req.fire() || write_dispatch_q.bits.laddr.is_garbage()) { + when (bio.read.req.fire()) { write_dispatch_q.ready := true.B - write_issue_q.io.enq.valid := true.B + write_scale_q.io.enq.valid := true.B io.dma.write.resp.valid := true.B } }.otherwise { bio.read.req.bits := DontCare } + bio.read.resp.ready := false.B + + + when (write_scale_q.io.deq.valid && + acc_scale_unit.io.in.ready && + bio.read.resp.valid && + write_issue_q.io.enq.ready && + write_scale_q.io.deq.bits.laddr.is_acc_addr && + write_scale_q.io.deq.bits.laddr.acc_bank() === i.U) { + write_scale_q.io.deq.ready := true.B + acc_scale_unit.io.in.valid := true.B + bio.read.resp.ready := true.B + write_issue_q.io.enq.valid := true.B + + acc_scale_unit.io.in.bits := bio.read.resp.bits + acc_scale_unit.io.in.bits.acc_bank_id := i.U + } - acc_scale_arb.io.in(i) <> bio.read.resp - acc_scale_arb.io.in(i).bits.acc_bank_id := i.U } // Writing to the accumulator banks From 3714ad569a224fb3bb105ae902dd8dcb0bfeea32 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Mon, 15 Feb 2021 21:37:15 -0500 Subject: [PATCH 019/123] Fix Firesim Freeze (#62) Fixed deadlock when a bias is loaded in at the same moment that a zero-write occurs. Expanded ports that were too small. --- src/main/scala/gemmini/Controller.scala | 2 +- src/main/scala/gemmini/DMA.scala | 8 +++----- src/main/scala/gemmini/LoopConv.scala | 6 +++--- src/main/scala/gemmini/LoopMatmul.scala | 6 +++--- src/main/scala/gemmini/ROB.scala | 21 ++++++++++++++------- src/main/scala/gemmini/Scratchpad.scala | 5 +---- src/main/scala/gemmini/ZeroWriter.scala | 2 +- 7 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index e586a58b7..3df1e3aca 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -9,7 +9,7 @@ import chisel3.util._ import freechips.rocketchip.config._ import freechips.rocketchip.diplomacy._ import freechips.rocketchip.tile._ -import freechips.rocketchip.tilelink.{TLIdentityNode} +import freechips.rocketchip.tilelink.TLIdentityNode import GemminiISA._ import Util._ diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index d35564225..bbd9b5073 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -7,7 +7,7 @@ import chisel3.experimental.DataMirror import freechips.rocketchip.config.Parameters import freechips.rocketchip.diplomacy.{IdRange, LazyModule, LazyModuleImp} import freechips.rocketchip.tile.{CoreBundle, HasCoreParameters} -import freechips.rocketchip.tilelink.{TLBundleA} +import freechips.rocketchip.tilelink.TLBundleA import testchipip.TLHelper import freechips.rocketchip.rocket.MStatus import freechips.rocketchip.rocket.constants.MemoryOpConstants @@ -221,7 +221,6 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf io.tlb.req.bits.tlb_req.cmd := M_XWR io.tlb.req.bits.status := tlb_q.io.deq.bits.status - val translate_q = Module(new Queue(new TLBundleAWithInfo, 1, pipe=true)) translate_q.io.enq <> tlb_q.io.deq translate_q.io.deq.ready := true.B @@ -230,11 +229,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf retry_a.bits := translate_q.io.deq.bits assert(retry_a.ready) - tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss - tl.a.bits := translate_q.io.deq.bits.tl_a + tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss + tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr - io.reserve.valid := state === s_req_new_block && untranslated_a.ready // TODO decouple "reserve.valid" from "tl.a.ready" io.reserve.entry.shift := read_shift io.reserve.entry.is_acc := req.is_acc diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 0ec656886..e9cb40b06 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -726,9 +726,9 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I val io = IO(new Bundle { val in = Flipped(Decoupled(new RoCCCommand)) val out = Decoupled(new RoCCCommand) - val ld_utilization = Input(UInt(log2Up(rob_size).W)) - val st_utilization = Input(UInt(log2Up(rob_size).W)) - val ex_utilization = Input(UInt(log2Up(rob_size).W)) + val ld_utilization = Input(UInt(log2Up(rob_size+1).W)) + val st_utilization = Input(UInt(log2Up(rob_size+1).W)) + val ex_utilization = Input(UInt(log2Up(rob_size+1).W)) val busy = Output(Bool()) }) diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 9b549c348..5932a51a0 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -606,9 +606,9 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: val io = IO(new Bundle { val in = Flipped(Decoupled(new RoCCCommand)) val out = Decoupled(new RoCCCommand) - val ld_utilization = Input(UInt(log2Up(rob_size).W)) - val st_utilization = Input(UInt(log2Up(rob_size).W)) - val ex_utilization = Input(UInt(log2Up(rob_size).W)) + val ld_utilization = Input(UInt(log2Up(rob_size+1).W)) + val st_utilization = Input(UInt(log2Up(rob_size+1).W)) + val ex_utilization = Input(UInt(log2Up(rob_size+1).W)) val busy = Output(Bool()) }) diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 2c026dc07..040df5940 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -3,12 +3,9 @@ package gemmini import chisel3._ import chisel3.util._ - import freechips.rocketchip.tile.RoCCCommand - import GemminiISA._ import Util._ -//import midas.targetutils.FpgaDebug // TODO unify this class with GemminiCmdWithDeps class ROBIssue[T <: Data](cmd_t: T, rob_entries: Int) extends Bundle { @@ -40,9 +37,9 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val ex = new ROBIssue(cmd_t, rob_entries) } - val ld_utilization = Output(UInt(log2Up(rob_entries).W)) - val st_utilization = Output(UInt(log2Up(rob_entries).W)) - val ex_utilization = Output(UInt(log2Up(rob_entries).W)) + val ld_utilization = Output(UInt(log2Up(rob_entries+1).W)) + val st_utilization = Output(UInt(log2Up(rob_entries+1).W)) + val ex_utilization = Output(UInt(log2Up(rob_entries+1).W)) val busy = Output(Bool()) @@ -317,9 +314,13 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf io.st_utilization := utilization_st_q io.ex_utilization := utilization_ex_q - val packed_deps = VecInit(entries.map(e => Cat(e.bits.deps))) + val packed_deps = VecInit(entries.map(e => Cat(e.bits.deps.reverse))) dontTouch(packed_deps) + val valids = VecInit(entries.map(_.valid)) + val functs = VecInit(entries.map(_.bits.cmd.inst.funct)) + val issueds = VecInit(entries.map(_.bits.issued)) + val pop_count_packed_deps = VecInit(entries.map(e => Mux(e.valid, PopCount(e.bits.deps), 0.U))) val min_pop_count = pop_count_packed_deps.reduce((acc, d) => minOf(acc, d)) // assert(min_pop_count < 2.U) @@ -335,6 +336,12 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf } assert(cycles_since_issue < 10000.U, "pipeline stall") + val instructions_allocated = RegInit(0.U(32.W)) + when (io.alloc.fire()) { + instructions_allocated := instructions_allocated + 1.U + } + dontTouch(instructions_allocated) + val cntr = Counter(10000000) when (cntr.inc()) { printf(p"Utilization: $utilization\n") diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 5f11cb0b9..dfee257d5 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -499,10 +499,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // We need to make sure that we don't try to return a dma read resp from both mvin_scale and mvin_scale_acc // at the same time. mvin_scale always gets priority in this cases - val spad_dmaread_last = mvin_scale_out.valid && mvin_scale_out.bits.last && !mvin_scale_out.bits.tag.is_acc - val spad_zerowrite_last = zero_writer.io.resp.valid && zero_writer.io.resp.bits.last && - !zero_writer.io.resp.bits.laddr.is_acc_addr - val spad_last = spad_dmaread_last || spad_zerowrite_last + val spad_last = mvin_scale_out.valid && mvin_scale_out.bits.last && !mvin_scale_out.bits.tag.is_acc val dmaread = (from_mvin_scale || from_mvin_scale_acc) && dmaread_bank === i.U /* && diff --git a/src/main/scala/gemmini/ZeroWriter.scala b/src/main/scala/gemmini/ZeroWriter.scala index a8e67df3c..c2e97b365 100644 --- a/src/main/scala/gemmini/ZeroWriter.scala +++ b/src/main/scala/gemmini/ZeroWriter.scala @@ -7,7 +7,7 @@ import Util._ class ZeroWriterReq[Tag <: Data](laddr_t: LocalAddr, max_cols: Int, tag_t: Tag) extends Bundle { val laddr = laddr_t - val cols = UInt(log2Up(max_cols).W) + val cols = UInt(log2Up(max_cols+1).W) val block_stride = UInt(16.W) // TODO magic number val tag = tag_t From 344c76c6169d40a48a034f6f6bc6beef9959700c Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 16 Feb 2021 00:48:37 -0800 Subject: [PATCH 020/123] Time-multiplex the VectorScalarMultiplier --- src/main/scala/gemmini/Configs.scala | 2 +- src/main/scala/gemmini/DSEConfigs.scala | 1 + src/main/scala/gemmini/GemminiConfigs.scala | 1 + src/main/scala/gemmini/Scratchpad.scala | 17 +- .../gemmini/VectorScalarMultiplier.scala | 178 ++++++++++++++---- 5 files changed, 155 insertions(+), 44 deletions(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 7ae20cd3f..02343fd32 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -88,7 +88,7 @@ object GemminiConfigs { // Mux(s >= 0.S, ((t >> u).asSInt() + Mux(r, 1.S, 0.S)).asSInt(), (t << (0.S-s).asUInt()).asSInt()) // }, // 0, SInt(8.W), "0")), - + num_mvin_scale_units = 4, mvin_scale_args = Some(ScaleArguments( (t: SInt, f: Float) => { val f_rec = recFNFromFN(f.expWidth, f.sigWidth, f.bits) diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 39922b331..2c5ce390c 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -36,6 +36,7 @@ object DSEBaseConfig { inputType = SInt(8.W), outputType = SInt(19.W), accType = SInt(32.W), + num_mvin_scale_units = -1, mvin_scale_args = None, mvin_scale_acc_args = None, mvin_scale_shared = false, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 564d43c24..5007229d8 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -37,6 +37,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( accType: T, mvin_scale_args: Option[ScaleArguments[T, U]], mvin_scale_acc_args: Option[ScaleArguments[T, U]], + num_mvin_scale_units: Int, mvin_scale_shared: Boolean, acc_scale_args: ScaleArguments[T, V], num_acc_scale_units: Int, diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 9b7887872..a3a731258 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -294,9 +294,20 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, reader.module.io.req.bits.status := read_issue_q.io.deq.bits.status reader.module.io.req.bits.cmd_id := read_issue_q.io.deq.bits.cmd_id - val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier(config.mvin_scale_args, config.inputType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), is_acc = false) - val (mvin_scale_acc_in, mvin_scale_acc_out) = if (mvin_scale_shared) (mvin_scale_in, mvin_scale_out) else - VectorScalarMultiplier(config.mvin_scale_acc_args, config.accType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), is_acc = true) + val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier( + config.mvin_scale_args, + config.inputType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), + num_mvin_scale_units, + is_acc = false + ) + val (mvin_scale_acc_in, mvin_scale_acc_out) = if (mvin_scale_shared) (mvin_scale_in, mvin_scale_out) else ( + VectorScalarMultiplier( + config.mvin_scale_acc_args, + config.accType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), + num_mvin_scale_units, + is_acc = true + ) + ) mvin_scale_in.valid := reader.module.io.resp.valid && (mvin_scale_shared.B || !reader.module.io.resp.bits.is_acc || (reader.module.io.resp.bits.is_acc && !reader.module.io.resp.bits.has_acc_bitwidth)) diff --git a/src/main/scala/gemmini/VectorScalarMultiplier.scala b/src/main/scala/gemmini/VectorScalarMultiplier.scala index d33ac0c22..14a32cd35 100644 --- a/src/main/scala/gemmini/VectorScalarMultiplier.scala +++ b/src/main/scala/gemmini/VectorScalarMultiplier.scala @@ -24,11 +24,10 @@ class VectorScalarMultiplierResp[T <: Data, Tag <: Data](block_cols: Int, t: T, override def cloneType: VectorScalarMultiplierResp.this.type = new VectorScalarMultiplierResp(block_cols, t, tag_t).asInstanceOf[this.type] } -// Currently, this class only supports multiplications of scratchpad inputs, rather than accumulator inputs -// class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](config: GemminiArrayConfig[T, U], tag_t: Tag) extends Module { - // import config._ - // val block_cols = meshColumns * tileColumns -class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](mvin_scale_args: Option[ScaleArguments[T, U]], block_cols: Int, t: T, tag_t: Tag) extends Module { +class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( + mvin_scale_args: Option[ScaleArguments[T, U]], block_cols: Int, t: T, tag_t: Tag, + num_scale_units: Int +) extends Module { val u = mvin_scale_args match { case Some(ScaleArguments(_, _, multiplicand_t, _, _)) => multiplicand_t @@ -40,55 +39,154 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data](mvin_scale_args: val resp = Decoupled(new VectorScalarMultiplierResp(block_cols, t, tag_t)) }) - val req = Reg(UDValid(chiselTypeOf(io.req.bits))) - + val width = block_cols val latency = mvin_scale_args match { case Some(ScaleArguments(_, latency, _, _, _)) => latency case None => 0 } - val resp = Wire(Decoupled(new VectorScalarMultiplierResp(block_cols, t, tag_t))) - - io.req.ready := !req.valid || (req.bits.repeats === 0.U && resp.fire()) - resp.valid := req.valid - resp.bits.tag := req.bits.tag - resp.bits.last := req.bits.repeats === 0.U && req.bits.last - resp.bits.row := req.bits.repeats - resp.bits.out := (mvin_scale_args match { - case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => - req.bits.in.map(x => mvin_scale_func(x, req.bits.scale.asTypeOf(multiplicand_t))) - - case None => req.bits.in - }) + + val in = Reg(Valid(new VectorScalarMultiplierReq(block_cols, t, u, tag_t))) + val in_fire = WireInit(false.B) + io.req.ready := !in.valid || (in.bits.repeats === 0.U && in_fire) when (io.req.fire()) { - req.push(io.req.bits) - }.elsewhen(resp.fire()) { - when (req.bits.repeats === 0.U) { - req.pop() - }.otherwise { - req.bits.repeats := req.bits.repeats - 1.U + in.valid := io.req.valid + in.bits := io.req.bits + } .elsewhen (in_fire) { + when (in.bits.repeats === 0.U) { + in.valid := false.B } + in.bits.repeats := in.bits.repeats - 1.U } + when (reset.asBool) { + in.valid := false.B + } + + + if (num_scale_units == -1) { + val pipe = Module(new Pipeline( + new VectorScalarMultiplierResp(block_cols, t, tag_t), + latency + )()) + io.resp <> pipe.io.out + in_fire := pipe.io.in.fire() + + pipe.io.in.valid := in.valid + pipe.io.in.bits.tag := in.bits.tag + pipe.io.in.bits.last := in.bits.repeats === 0.U && in.bits.last + pipe.io.in.bits.row := in.bits.repeats + pipe.io.in.bits.out := (mvin_scale_args match { + case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => + in.bits.in.map(x => mvin_scale_func(x, in.bits.scale.asTypeOf(multiplicand_t))) + case None => in.bits.in + }) + } else { + val nEntries = 3 + val regs = Reg(Vec(nEntries, Valid(new VectorScalarMultiplierReq(block_cols, t, u, tag_t)))) + val out_regs = Reg(Vec(nEntries, new VectorScalarMultiplierResp(block_cols, t, tag_t))) + + val fired_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + val completed_masks = Reg(Vec(nEntries, Vec(width, Bool()))) + val head_oh = RegInit(1.U(nEntries.W)) + val tail_oh = RegInit(1.U(nEntries.W)) + + io.resp.valid := Mux1H(head_oh.asBools, (regs zip completed_masks).map({case (r,c) => r.valid && c.reduce(_&&_)})) + io.resp.bits := Mux1H(head_oh.asBools, out_regs) + when (io.resp.fire()) { + for (i <- 0 until nEntries) { + when (head_oh(i)) { + regs(i).valid := false.B + } + } + head_oh := (head_oh << 1) | head_oh(nEntries-1) + } + in_fire := (in.valid && + (!Mux1H(tail_oh.asBools, regs.map(_.valid)) || (tail_oh === head_oh && io.resp.fire())) + ) + when (in_fire) { + for (i <- 0 until nEntries) { + when (tail_oh(i)) { + regs(i).valid := true.B + regs(i).bits := in.bits + out_regs(i).tag := in.bits.tag + out_regs(i).last := in.bits.repeats === 0.U && in.bits.last + out_regs(i).row := in.bits.repeats + fired_masks(i).foreach(_ := false.B) + completed_masks(i).foreach(_ := false.B) + } + } + tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) + } + + class DataWithIndex extends Bundle { + val data = in.bits.in(0).cloneType + val scale = u.cloneType + val id = UInt(2.W) // TODO hardcoded + val index = UInt() + } + val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } + for (i <- 0 until nEntries) { + for (w <- 0 until width) { + val input = inputs(i*width+w) + input.valid := regs(i).valid && !fired_masks(i)(w) + input.bits.data := regs(i).bits.in(w) + input.bits.scale := regs(i).bits.scale.asTypeOf(u) + input.bits.id := i.U + input.bits.index := w.U + when (input.fire()) { + fired_masks(i)(w) := true.B + } + } + } + for (i <- 0 until num_scale_units) { + val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) + val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) + arb.io.in <> arbIn + arb.io.out.ready := true.B + val arbOut = arb.io.out + val e_scaled = mvin_scale_args match { + case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => + mvin_scale_func(arb.io.out.bits.data, arb.io.out.bits.scale.asTypeOf(multiplicand_t)) + case None => arb.io.out.bits.data + } + + val pipe_in = Wire(Valid(new DataWithIndex)) + pipe_in.valid := arbOut.valid + pipe_in.bits := arbOut.bits + pipe_in.bits.data := e_scaled + val pipe_out = Pipe(pipe_in, latency) + for (j <- 0 until nEntries) { + for (w <- 0 until width) { + if ((j*width+w) % num_scale_units == i) { + when (pipe_out.fire() && pipe_out.bits.id === j.U && pipe_out.bits.index === w.U) { + out_regs(j).out(w) := pipe_out.bits.data + completed_masks(j)(w) := true.B + } + } + } + } + } + when (reset.asBool) { + regs.foreach(_.valid := false.B) + } + - when (reset.toBool()) { - req.pop() } - (mvin_scale_args match { - case Some(ScaleArguments(_, latency, _, _, _)) => io.resp <> Pipeline(resp, latency) - case None => io.resp <> resp - }) + + } object VectorScalarMultiplier { // Returns the input and output IO of the module (together with the pipeline) - def apply[T <: Data, U <: Data, Tag <: Data](scale_args: Option[ScaleArguments[T, U]], t: T, cols: Int, tag_t: Tag, is_acc: Boolean, is_mvin: Boolean=true) = { + def apply[T <: Data, U <: Data, Tag <: Data]( + scale_args: Option[ScaleArguments[T, U]], + t: T, cols: Int, tag_t: Tag, + num_multipliers: Int, + is_acc: Boolean, + is_mvin: Boolean=true + ) = { assert(!is_acc || is_mvin) - - val vsm = Module(new VectorScalarMultiplier(scale_args, cols, t, tag_t)) - - val in = vsm.io.req - val out = vsm.io.resp - - (in, out) + val vsm = Module(new VectorScalarMultiplier(scale_args, cols, t, tag_t, num_multipliers)) + (vsm.io.req, vsm.io.resp) } } From 1a6ed243874c8306e747d0f80b7773d3e2e1a3cc Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Tue, 16 Feb 2021 16:43:22 -0500 Subject: [PATCH 021/123] Add FP configs and fix them (#64) * Add FP configs and fix zero-writer connections, as well as Tiler dimensions * Bump gemmini-rocc-tests * Removed unusued DMAWriteCommandTracker * Implement gt for floats Tested with FP32 4-by-4 config --- software/gemmini-rocc-tests | 2 +- src/main/scala/gemmini/Arithmetic.scala | 20 ++- src/main/scala/gemmini/ConfigsFP.scala | 143 ++++++++++++++++++ ...dTracker.scala => DMACommandTracker.scala} | 2 +- .../gemmini/DMAWriteCommandTracker.scala | 9 -- src/main/scala/gemmini/GemminiConfigs.scala | 12 +- src/main/scala/gemmini/Im2Col.scala | 2 +- src/main/scala/gemmini/LoadController.scala | 2 +- src/main/scala/gemmini/Scratchpad.scala | 9 +- src/main/scala/gemmini/StoreController.scala | 2 +- 10 files changed, 181 insertions(+), 22 deletions(-) create mode 100644 src/main/scala/gemmini/ConfigsFP.scala rename src/main/scala/gemmini/{DMAReadCommandTracker.scala => DMACommandTracker.scala} (97%) delete mode 100644 src/main/scala/gemmini/DMAWriteCommandTracker.scala diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index dab14715e..dcbc4cdbe 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit dab14715ea5a8ba9049feb65d98022119b7a72f2 +Subproject commit dcbc4cdbe2873355a6ae96b16e11a68e6a926f0d diff --git a/src/main/scala/gemmini/Arithmetic.scala b/src/main/scala/gemmini/Arithmetic.scala index 0fcac90e3..ffb245745 100644 --- a/src/main/scala/gemmini/Arithmetic.scala +++ b/src/main/scala/gemmini/Arithmetic.scala @@ -271,7 +271,25 @@ object Arithmetic { */ } - override def >(t: Float): Bool = true.B // TODO + override def >(t: Float): Bool = { + // Recode all operands + val t_rec = recFNFromFN(t.expWidth, t.sigWidth, t.bits) + val self_rec = recFNFromFN(self.expWidth, self.sigWidth, self.bits) + + // Resize t to self's width + val t_resizer = Module(new RecFNToRecFN(t.expWidth, t.sigWidth, self.expWidth, self.sigWidth)) + t_resizer.io.in := t_rec + t_resizer.io.roundingMode := consts.round_near_even + t_resizer.io.detectTininess := consts.tininess_afterRounding + val t_rec_resized = t_resizer.io.out + + val comparator = Module(new CompareRecFN(self.expWidth, self.sigWidth)) + comparator.io.a := self_rec + comparator.io.b := t_rec_resized + comparator.io.signaling := false.B + + comparator.io.gt + } override def withWidthOf(t: Float): Float = { val self_rec = recFNFromFN(self.expWidth, self.sigWidth, self.bits) diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala new file mode 100644 index 000000000..9c5aa6593 --- /dev/null +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -0,0 +1,143 @@ +package gemmini + +import chisel3._ +import freechips.rocketchip.config.{Config, Parameters} +import freechips.rocketchip.diplomacy.{LazyModule, ValName} +import freechips.rocketchip.subsystem._ +import freechips.rocketchip.tile.{BuildRoCC, OpcodeSet} + +// ----------------------------- +// Floating Point Config Mixins +// ----------------------------- + + +object GemminiFPConfigs { + import Arithmetic.FloatArithmetic._ + val defaultFPConfig = GemminiArrayConfig[Float, Float, Float]( + tileRows = 1, + tileColumns = 1, + meshRows = 4, + meshColumns = 4, + + ld_queue_length = 8, + st_queue_length = 2, + ex_queue_length = 8, + + rob_entries = 16, + + hasIm2col = false, + + sp_banks = 4, + sp_singleported = true, + acc_banks = 1, + sp_capacity = CapacityInKilobytes(256), + shifter_banks = 1, // TODO add separate parameters for left and up shifter banks + dataflow = Dataflow.BOTH, + acc_capacity = CapacityInKilobytes(64), + mem_pipeline = 1, + + dma_maxbytes = 64, // TODO get this from cacheblockbytes + dma_buswidth = 128, // TODO get this from SystemBusKey + aligned_to = 1, + tlb_size = 4, + use_tlb_register_filter = true, + max_in_flight_reqs = 16, + use_dedicated_tl_port = false, + + inputType = Float(8, 24), + outputType = Float(8, 24), + accType = Float(8, 24), + + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_shared = false, + acc_scale_args = ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", + c_str = "((x) * (scale))" + ), + acc_read_full_width = true, + acc_read_small_width = true, + + pe_latency = 1, + ) + + //FP32 Single Precision Configuration + val FP32DefaultConfig = defaultFPConfig.copy(inputType = Float(8, 24), outputType = Float(8, 24), accType = Float(8, 24), + pe_latency = 2, + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + ) + + //FP16 Half Precision Configuration + val FP16DefaultConfig = defaultFPConfig.copy(inputType = Float(5, 11), outputType = Float(5, 11), accType = Float(8, 24), + pe_latency = 2, + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(5, 11), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(5, 11), identity = "1.0", c_str="((x) * (scale))")), + ) + + //Bfloat16 Brain-half Precision Configuration + val BF16DefaultConfig = defaultFPConfig.copy(inputType = Float(8, 8), outputType = Float(8, 8), accType = Float(8, 24), + pe_latency = 2, + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + ) + + //Bfloat16 Brain-half Precision Configuration 8x8 array + val BF16Default8Config = defaultFPConfig.copy(inputType = Float(8, 8), outputType = Float(8, 8), accType = Float(8, 24), + meshRows = 8, meshColumns = 8, + pe_latency = 2, + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + ) + +} + + +//===========FP32 Default Config========= +class GemminiFP32DefaultConfig extends Config((site, here, up) => { + case BuildRoCC => Seq( + (p: Parameters) => { + implicit val q = p + implicit val v = implicitly[ValName] + LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.FP32DefaultConfig)) + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) + + +//===========FP16 Default Config========= +class GemminiFP16DefaultConfig extends Config((site, here, up) => { + case BuildRoCC => Seq( + (p: Parameters) => { + implicit val q = p + implicit val v = implicitly[ValName] + LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.FP16DefaultConfig)) + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) + +//===========BFLOAT16 Default Config========= +class GemminiBF16DefaultConfig extends Config((site, here, up) => { + case BuildRoCC => Seq( + (p: Parameters) => { + implicit val q = p + implicit val v = implicitly[ValName] + LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.BF16DefaultConfig)) + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) + +//===========BFLOAT16 Default Config 8x8========= +class GemminiBF16Default8Config extends Config((site, here, up) => { + case BuildRoCC => Seq( + (p: Parameters) => { + implicit val q = p + implicit val v = implicitly[ValName] + LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.BF16Default8Config)) + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) + diff --git a/src/main/scala/gemmini/DMAReadCommandTracker.scala b/src/main/scala/gemmini/DMACommandTracker.scala similarity index 97% rename from src/main/scala/gemmini/DMAReadCommandTracker.scala rename to src/main/scala/gemmini/DMACommandTracker.scala index a7eecfda6..2632f7530 100644 --- a/src/main/scala/gemmini/DMAReadCommandTracker.scala +++ b/src/main/scala/gemmini/DMACommandTracker.scala @@ -6,7 +6,7 @@ import chisel3.util._ // This module is meant to go inside the Load controller, where it can track which commands are currently // in flight and which are completed -class DMAReadCommandTracker[T <: Data](val nCmds: Int, val maxBytes: Int, tag_t: => T) extends Module { +class DMACommandTracker[T <: Data](val nCmds: Int, val maxBytes: Int, tag_t: => T) extends Module { def cmd_id_t = UInt((log2Ceil(nCmds) max 1).W) val io = IO(new Bundle { diff --git a/src/main/scala/gemmini/DMAWriteCommandTracker.scala b/src/main/scala/gemmini/DMAWriteCommandTracker.scala deleted file mode 100644 index 30765a446..000000000 --- a/src/main/scala/gemmini/DMAWriteCommandTracker.scala +++ /dev/null @@ -1,9 +0,0 @@ -package gemmini - -import chisel3._ -import chisel3.util._ - -object DMAWriteCommandTracker { - def apply[T <: Data](nCmds: Int, nRows: Int, tag_t: => T) = Module(new DMAReadCommandTracker(nCmds = nCmds, - maxBytes = nRows, tag_t = tag_t)) -} diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 889928257..373edd77f 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -112,13 +112,15 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( // cisc-gemmini hardware-specific compile-time global constants //========================================================================== + val cisc_dim = (meshRows * tileRows) / 2 + val ITYPE_BITS = inputType.getWidth - val ITYPE_BYTES = (inputType.getWidth+7) / 8 + val ITYPE_BYTES = (inputType.getWidth+cisc_dim-1) / cisc_dim val LOG2_ITYPE_BYTES = if(ITYPE_BYTES <= 1) 0 else log2Up(ITYPE_BYTES) val OTYPE_BITS = accType.getWidth val LOG2_OTYPE_BITS = log2Up(OTYPE_BITS) - val OTYPE_BYTES = (accType.getWidth+7) / 8 + val OTYPE_BYTES = (accType.getWidth+cisc_dim-1) / cisc_dim val LOG2_OTYPE_BYTES = if(OTYPE_BYTES <= 1) 0 else log2Up(OTYPE_BYTES) val SP_BANKS = sp_banks @@ -135,12 +137,12 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( val LOG2_MNK_BYTES = log2Up(MNK_BYTES) val MNK_BYTES_PER_TILE_ROW = MNK_BYTES * DIM val LOG2_MNK_BYTES_PER_TILE_ROW = log2Up(MNK_BYTES_PER_TILE_ROW) - val TILE_IDX = MNK_BYTES / (DIM / 8) + val TILE_IDX = MNK_BYTES / (DIM / cisc_dim) val LOG2_TILE_IDX = log2Up(TILE_IDX) //-------------------------------------------------------------------------- - val I_TILE_BYTE_WIDTH = DIM * ((inputType.getWidth+7) / 8) - val O_TILE_BYTE_WIDTH = DIM * ((accType.getWidth+7) / 8) + val I_TILE_BYTE_WIDTH = DIM * ((inputType.getWidth+cisc_dim-1) / cisc_dim) + val O_TILE_BYTE_WIDTH = DIM * ((accType.getWidth+cisc_dim-1) / cisc_dim) val I_TILE_BYTE_WIDTH_LOG2 = log2Up(I_TILE_BYTE_WIDTH) val O_TILE_BYTE_WIDTH_LOG2 = log2Up(O_TILE_BYTE_WIDTH) require(pow(2,I_TILE_BYTE_WIDTH_LOG2) == I_TILE_BYTE_WIDTH, diff --git a/src/main/scala/gemmini/Im2Col.scala b/src/main/scala/gemmini/Im2Col.scala index 52039d4d0..4970334da 100644 --- a/src/main/scala/gemmini/Im2Col.scala +++ b/src/main/scala/gemmini/Im2Col.scala @@ -415,7 +415,7 @@ class Im2Col[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V when(i.U < channel){ im2col_data(i) := sram_req_output(i) }.otherwise{ - im2col_data(i) := 0.S //when channel < 16, pad with 0 + im2col_data(i) := 0.U.asTypeOf(inputType) //when channel < 16, pad with 0 } } } diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index 65146b2be..d9221f5b2 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -77,7 +77,7 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig (block_cols * config.accType.getWidth / 8) val maxBytesInMatRequest = block_rows * maxBytesInRowRequest - val cmd_tracker = Module(new DMAReadCommandTracker(nCmds, maxBytesInMatRequest, deps_t)) + val cmd_tracker = Module(new DMACommandTracker(nCmds, maxBytesInMatRequest, deps_t)) io.busy := cmd.valid || cmd_tracker.io.busy diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index dfee257d5..2dbaf6c77 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -406,7 +406,12 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, }.elsewhen (zerowrite) { bio.write.addr := zero_writer.io.resp.bits.laddr.sp_row() bio.write.data := 0.U - bio.write.mask := zero_writer.io.resp.bits.mask + bio.write.mask := { + val n = inputType.getWidth / 8 + val mask = zero_writer.io.resp.bits.mask + val expanded = VecInit(mask.flatMap(e => Seq.fill(n)(e))) + expanded + } zero_writer.io.resp.ready := true.B // TODO we combinationally couple valid and ready signals }.otherwise { @@ -547,7 +552,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, }.elsewhen (zerowrite && bio.write.fire()) { bio.write.bits.data := 0.U.asTypeOf(acc_row_t) bio.write.bits.mask := { - val n = accType.getWidth / inputType.getWidth + val n = accType.getWidth / 8 val mask = zero_writer.io.resp.bits.mask val expanded = VecInit(mask.flatMap(e => Seq.fill(n)(e))) expanded diff --git a/src/main/scala/gemmini/StoreController.scala b/src/main/scala/gemmini/StoreController.scala index 83609ffb7..561061674 100644 --- a/src/main/scala/gemmini/StoreController.scala +++ b/src/main/scala/gemmini/StoreController.scala @@ -111,7 +111,7 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm ((config.sp_banks * config.sp_bank_entries) max (config.acc_banks * config.acc_bank_entries)) - val cmd_tracker = Module(new DMAReadCommandTracker(nCmds, cmd_tracker_max_rows, deps_t)) + val cmd_tracker = Module(new DMACommandTracker(nCmds, cmd_tracker_max_rows, deps_t)) // DMA IO wiring io.dma.req.valid := (control_state === waiting_for_command && cmd.valid && DoStore && cmd_tracker.io.alloc.ready) || From 63e2bfe0b6097888887404ca6fa03f551be4506f Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 16 Feb 2021 23:01:38 -0800 Subject: [PATCH 022/123] Short circuit past vector-scalar-multiplier for identity-scales --- src/main/scala/gemmini/Arithmetic.scala | 4 ++++ .../scala/gemmini/VectorScalarMultiplier.scala | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/main/scala/gemmini/Arithmetic.scala b/src/main/scala/gemmini/Arithmetic.scala index ffb245745..9170b8346 100644 --- a/src/main/scala/gemmini/Arithmetic.scala +++ b/src/main/scala/gemmini/Arithmetic.scala @@ -17,6 +17,7 @@ abstract class ArithmeticOps[T <: Data](self: T) { def +(t: T): T def >>(u: UInt): T // This is a rounding shift! Rounds away from 0 def >(t: T): Bool + def identity: T def withWidthOf(t: T): T def clippedToWidthOf(t: T): T // Like "withWidthOf", except that it saturates def relu: T @@ -62,6 +63,7 @@ object Arithmetic { } override def zero: UInt = 0.U + override def identity: UInt = 1.U } } @@ -111,6 +113,7 @@ object Arithmetic { } override def zero: SInt = 0.S + override def identity: SInt = 1.S } } @@ -393,6 +396,7 @@ object Arithmetic { } override def zero: Float = 0.U.asTypeOf(self) + override def identity: Float = Cat(0.U(2.W), ~(0.U((self.expWidth-1).W)), 0.U((self.sigWidth-1).W)).asTypeOf(self) } } } diff --git a/src/main/scala/gemmini/VectorScalarMultiplier.scala b/src/main/scala/gemmini/VectorScalarMultiplier.scala index 14a32cd35..cb8be5e97 100644 --- a/src/main/scala/gemmini/VectorScalarMultiplier.scala +++ b/src/main/scala/gemmini/VectorScalarMultiplier.scala @@ -29,9 +29,9 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( num_scale_units: Int ) extends Module { - val u = mvin_scale_args match { - case Some(ScaleArguments(_, _, multiplicand_t, _, _)) => multiplicand_t - case None => Bool() // TODO make this a 0-width UInt + val (u, always_identity) = mvin_scale_args match { + case Some(ScaleArguments(_, _, multiplicand_t, _, _)) => (multiplicand_t, false) + case None => (Bool(), true) // TODO make this a 0-width UInt } val io = IO(new Bundle { @@ -111,8 +111,15 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( out_regs(i).tag := in.bits.tag out_regs(i).last := in.bits.repeats === 0.U && in.bits.last out_regs(i).row := in.bits.repeats - fired_masks(i).foreach(_ := false.B) - completed_masks(i).foreach(_ := false.B) + out_regs(i).out := in.bits.in + val identity = (u match { + case u: UInt => Arithmetic.UIntArithmetic.cast(u).identity + case s: SInt => Arithmetic.SIntArithmetic.cast(s).identity + case f: Float => Arithmetic.FloatArithmetic.cast(f).identity + case b: Bool => 1.U(1.W) + }) + fired_masks(i).foreach(_ := in.bits.scale.asUInt === identity.asUInt || always_identity.B) + completed_masks(i).foreach(_ := in.bits.scale.asUInt === identity.asUInt || always_identity.B) } } tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) From cf191ba3e3c2f858dd9e36b163791192363dcb05 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 16 Feb 2021 23:26:36 -0800 Subject: [PATCH 023/123] Bank the accumulator memories into single-ported mems --- src/main/scala/gemmini/AccumulatorMem.scala | 128 +++++++++++++++++--- src/main/scala/gemmini/Configs.scala | 2 + src/main/scala/gemmini/ConfigsFP.scala | 2 + src/main/scala/gemmini/DSEConfigs.scala | 2 + src/main/scala/gemmini/GemminiConfigs.scala | 2 + src/main/scala/gemmini/Scratchpad.scala | 3 +- 6 files changed, 123 insertions(+), 16 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 5b25c1e06..85836a7f8 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -53,7 +53,9 @@ class AccumulatorMemIO [T <: Data: Arithmetic, U <: Data](n: Int, t: Vec[Vec[T]] } class AccumulatorMem[T <: Data, U <: Data]( - n: Int, t: Vec[Vec[T]], scale_args: ScaleArguments[T, U]) + n: Int, t: Vec[Vec[T]], scale_args: ScaleArguments[T, U], + acc_singleported: Boolean, num_acc_sub_banks: Int +) (implicit ev: Arithmetic[T]) extends Module { // TODO Do writes in this module work with matrices of size 2? If we try to read from an address right after writing // to it, then we might not get the written data. We might need some kind of cooldown counter after addresses in the @@ -69,7 +71,6 @@ class AccumulatorMem[T <: Data, U <: Data]( // TODO unify this with TwoPortSyncMemIO val io = IO(new AccumulatorMemIO(n, t, scale_args.multiplicand_t)) - val mem = TwoPortSyncMem(n, t, t.getWidth / 8) // TODO We assume byte-alignment here. Use aligned_to instead // For any write operation, we spend 2 cycles reading the existing address out, buffering it in a register, and then // accumulating on top of it (if necessary) @@ -78,21 +79,117 @@ class AccumulatorMem[T <: Data, U <: Data]( val acc_buf = ShiftRegister(io.write.bits.acc, 2) val mask_buf = ShiftRegister(io.write.bits.mask, 2) val w_buf_valid = ShiftRegister(io.write.fire(), 2) - - val w_sum = VecInit((RegNext(mem.io.rdata) zip wdata_buf).map { case (rv, wv) => + val rdata = Wire(t) + rdata := DontCare + val block_read_req = WireInit(false.B) + val w_sum = VecInit((RegNext(rdata) zip wdata_buf).map { case (rv, wv) => VecInit((rv zip wv).map(t => t._1 + t._2)) }) - mem.io.waddr := waddr_buf - mem.io.wen := w_buf_valid - mem.io.wdata := Mux(acc_buf, w_sum, wdata_buf) - mem.io.mask := mask_buf - - mem.io.raddr := Mux(io.write.fire() && io.write.bits.acc, io.write.bits.addr, io.read.req.bits.addr) - mem.io.ren := io.read.req.fire() || (io.write.fire() && io.write.bits.acc) + if (!acc_singleported) { + val mem = TwoPortSyncMem(n, t, t.getWidth / 8) // TODO We assume byte-alignment here. Use aligned_to instead + mem.io.waddr := waddr_buf + mem.io.wen := w_buf_valid + mem.io.wdata := Mux(acc_buf, w_sum, wdata_buf) + mem.io.mask := mask_buf + rdata := mem.io.rdata + mem.io.raddr := Mux(io.write.fire() && io.write.bits.acc, io.write.bits.addr, io.read.req.bits.addr) + mem.io.ren := io.read.req.fire() || (io.write.fire() && io.write.bits.acc) + } else { + val mask_len = t.getWidth / 8 + val mask_elem = UInt((t.getWidth / mask_len).W) + val reads = Wire(Vec(2, Decoupled(UInt()))) + reads(0).valid := io.write.valid && io.write.bits.acc + reads(0).bits := io.write.bits.addr + reads(0).ready := true.B + reads(1).valid := io.read.req.valid + reads(1).bits := io.read.req.bits.addr + reads(1).ready := true.B + block_read_req := !reads(1).ready + for (i <- 0 until num_acc_sub_banks) { + def isThisBank(addr: UInt) = addr(log2Ceil(num_acc_sub_banks)-1,0) === i.U + def getBankIdx(addr: UInt) = addr >> log2Ceil(num_acc_sub_banks) + val mem = SyncReadMem(n / num_acc_sub_banks, Vec(mask_len, mask_elem)) + + val ren = WireInit(false.B) + val raddr = WireInit(getBankIdx(reads(0).bits)) + val nEntries = 3 + // Writes coming 2 cycles after read leads to bad bank behavior + // Add another buffer here + class W_Q_Entry[T <: Data](mask_len: Int, mask_elem: T) extends Bundle { + val valid = Bool() + val data = Vec(mask_len, mask_elem) + val mask = Vec(mask_len, Bool()) + val addr = UInt(log2Ceil(n/num_acc_sub_banks).W) + override def cloneType: this.type = new W_Q_Entry(mask_len, mask_elem).asInstanceOf[this.type] + } + val w_q = Reg(Vec(nEntries, new W_Q_Entry(mask_len, mask_elem))) + for (e <- w_q) { + when (e.valid) { + assert(!(io.write.valid && io.write.bits.acc && isThisBank(io.write.bits.addr) && getBankIdx(io.write.bits.addr) === e.addr)) + when (io.read.req.valid && isThisBank(io.read.req.bits.addr) && getBankIdx(io.read.req.bits.addr) === e.addr) { + reads(1).ready := false.B + } + } + } + val w_q_head = RegInit(1.U(nEntries.W)) + val w_q_tail = RegInit(1.U(nEntries.W)) + when (reset.asBool) { + w_q.foreach(_.valid := false.B) + } + val wen = WireInit(false.B) + val wdata = Mux1H(w_q_head.asBools, w_q.map(_.data)) + val wmask = Mux1H(w_q_head.asBools, w_q.map(_.mask)) + val waddr = Mux1H(w_q_head.asBools, w_q.map(_.addr)) + when (wen) { + w_q_head := w_q_head << 1 | w_q_head(nEntries-1) + for (i <- 0 until nEntries) { + when (w_q_head(i)) { + w_q(i).valid := false.B + } + } + } + when (w_buf_valid && isThisBank(waddr_buf)) { + assert(!((w_q_tail.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_))) + w_q_tail := w_q_tail << 1 | w_q_tail(nEntries-1) + for (i <- 0 until nEntries) { + when (w_q_tail(i)) { + w_q(i).valid := true.B + w_q(i).data := Mux(acc_buf, w_sum, wdata_buf).asTypeOf(Vec(mask_len, mask_elem)) + w_q(i).mask := mask_buf + w_q(i).addr := getBankIdx(waddr_buf) + } + } + + } + val bank_rdata = mem.read(raddr, ren && !wen).asTypeOf(t) + when (RegNext(ren)) { + rdata := bank_rdata + } + when (wen) { + mem.write(waddr, wdata, wmask) + } + // Three requestors, 1 slot + // Priority is incoming reads for RMW > writes from RMW > incoming reads + when (reads(0).valid && isThisBank(reads(0).bits)) { + ren := true.B + when (isThisBank(reads(1).bits)) { + reads(1).ready := false.B + } + } .elsewhen ((w_q_head.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_)) { + wen := true.B + when (isThisBank(reads(1).bits)) { + reads(1).ready := false.B + } + } .otherwise { + ren := isThisBank(reads(1).bits) + raddr := getBankIdx(reads(1).bits) + } + } + } val q = Module(new Queue(new AccumulatorReadResp(t, scale_args.multiplicand_t, log2Ceil(t.head.head.getWidth)), 1, true, true)) - q.io.enq.bits.data := mem.io.rdata + q.io.enq.bits.data := rdata q.io.enq.bits.scale := RegNext(io.read.req.bits.scale) q.io.enq.bits.relu6_shift := RegNext(io.read.req.bits.relu6_shift) q.io.enq.bits.act := RegNext(io.read.req.bits.act) @@ -119,14 +216,15 @@ class AccumulatorMem[T <: Data, U <: Data]( !(io.write.fire() && io.write.bits.acc) && // Make sure we aren't reading something that is still being written !(RegNext(io.write.fire()) && RegNext(io.write.bits.addr) === io.read.req.bits.addr) && - !(w_buf_valid && waddr_buf === io.read.req.bits.addr) - ) + !(w_buf_valid && waddr_buf === io.read.req.bits.addr) && + !block_read_req + ) // io.write.current_waddr.valid := mem.io.wen // io.write.current_waddr.bits := mem.io.waddr - io.write.ready := !io.write.bits.acc || (!(io.write.bits.addr === mem.io.waddr && mem.io.wen) && + io.write.ready := !io.write.bits.acc || (!(io.write.bits.addr === waddr_buf && w_buf_valid) && !(io.write.bits.addr === RegNext(io.write.bits.addr) && RegNext(io.write.fire()))) // assert(!(io.read.req.valid && io.write.en && io.write.acc), "reading and accumulating simultaneously is not supported") diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 9194aa57d..7d95f30e6 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -51,6 +51,8 @@ object GemminiConfigs { sp_banks = 4, sp_singleported = true, acc_banks = 2, + acc_singleported = true, + num_acc_sub_banks = 2, sp_capacity = CapacityInKilobytes(256), shifter_banks = 1, // TODO add separate parameters for left and up shifter banks dataflow = Dataflow.BOTH, diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 0abcd7ca0..0d9801b70 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -30,6 +30,8 @@ object GemminiFPConfigs { sp_banks = 4, sp_singleported = true, acc_banks = 1, + acc_singleported = false, + num_acc_sub_banks = -1, sp_capacity = CapacityInKilobytes(256), shifter_banks = 1, // TODO add separate parameters for left and up shifter banks dataflow = Dataflow.BOTH, diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 2c5ce390c..1d4dd7de6 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -23,6 +23,8 @@ object DSEBaseConfig { rob_entries = 8, sp_banks = 4, // TODO support one-bank designs acc_banks = 1, + acc_singleported = false, + num_acc_sub_banks = -1, sp_capacity = CapacityInKilobytes(64), sp_singleported = false, shifter_banks = 1, // TODO add separate parameters for left and up shifter banks diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index d0c7e9962..67db67626 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -25,6 +25,8 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( sp_singleported: Boolean, sp_capacity: GemminiMemCapacity, acc_banks: Int, + acc_singleported: Boolean, + num_acc_sub_banks: Int, acc_capacity: GemminiMemCapacity, shifter_banks: Int, dataflow: Dataflow.Value, diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index aa6510f77..9a1410e56 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -503,7 +503,8 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, { val banks = Seq.fill(acc_banks) { Module(new AccumulatorMem( - acc_bank_entries, acc_row_t, acc_scale_args + acc_bank_entries, acc_row_t, acc_scale_args, + acc_singleported, num_acc_sub_banks )) } val bank_ios = VecInit(banks.map(_.io)) From 96b59a47c5e43fbc80f34ab798c91b7ae7f53ea8 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Feb 2021 01:34:12 -0800 Subject: [PATCH 024/123] Add pipeline register after accumulatorscale register --- src/main/scala/gemmini/AccumulatorScale.scala | 7 ++++++- src/main/scala/gemmini/Configs.scala | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorScale.scala b/src/main/scala/gemmini/AccumulatorScale.scala index 415688c46..c54a6236d 100644 --- a/src/main/scala/gemmini/AccumulatorScale.scala +++ b/src/main/scala/gemmini/AccumulatorScale.scala @@ -134,7 +134,12 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) arb.io.in <> arbIn arb.io.out.ready := true.B - val arbOut = arb.io.out + val arbOut = Reg(Valid(new DataWithIndex)) + arbOut.valid := arb.io.out.valid + arbOut.bits := arb.io.out.bits + when (reset.asBool) { + arbOut.valid := false.B + } val e_scaled = scale_args.scale_func(arbOut.bits.data, arbOut.bits.scale) val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) val e_act = MuxCase(e_clipped, Seq( diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 7d95f30e6..3abdb8ca5 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -114,7 +114,7 @@ object GemminiConfigs { mvin_scale_shared = false, num_acc_scale_units = 4, - acc_scale_latency = 3, + acc_scale_latency = 4, acc_scale_args = ScaleArguments( (t: SInt, f: Float) => { val f_rec = recFNFromFN(f.expWidth, f.sigWidth, f.bits) From 353b9e5785788c332a003d47542359b695a238dc Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Feb 2021 10:51:54 -0800 Subject: [PATCH 025/123] Allow parallel read/writes from different sub-banks of accumulator --- src/main/scala/gemmini/AccumulatorMem.scala | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 85836a7f8..1334dc4ce 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -79,10 +79,12 @@ class AccumulatorMem[T <: Data, U <: Data]( val acc_buf = ShiftRegister(io.write.bits.acc, 2) val mask_buf = ShiftRegister(io.write.bits.mask, 2) val w_buf_valid = ShiftRegister(io.write.fire(), 2) - val rdata = Wire(t) - rdata := DontCare + val acc_rdata = Wire(t) + acc_rdata := DontCare + val read_rdata = Wire(t) + read_rdata := DontCare val block_read_req = WireInit(false.B) - val w_sum = VecInit((RegNext(rdata) zip wdata_buf).map { case (rv, wv) => + val w_sum = VecInit((RegNext(acc_rdata) zip wdata_buf).map { case (rv, wv) => VecInit((rv zip wv).map(t => t._1 + t._2)) }) @@ -92,7 +94,8 @@ class AccumulatorMem[T <: Data, U <: Data]( mem.io.wen := w_buf_valid mem.io.wdata := Mux(acc_buf, w_sum, wdata_buf) mem.io.mask := mask_buf - rdata := mem.io.rdata + acc_rdata := mem.io.rdata + read_rdata := mem.io.rdata mem.io.raddr := Mux(io.write.fire() && io.write.bits.acc, io.write.bits.addr, io.read.req.bits.addr) mem.io.ren := io.read.req.fire() || (io.write.fire() && io.write.bits.acc) } else { @@ -163,8 +166,10 @@ class AccumulatorMem[T <: Data, U <: Data]( } val bank_rdata = mem.read(raddr, ren && !wen).asTypeOf(t) - when (RegNext(ren)) { - rdata := bank_rdata + when (RegNext(ren && reads(0).valid && isThisBank(reads(0).bits))) { + acc_rdata := bank_rdata + } .elsewhen (RegNext(ren)) { + read_rdata := bank_rdata } when (wen) { mem.write(waddr, wdata, wmask) @@ -189,7 +194,7 @@ class AccumulatorMem[T <: Data, U <: Data]( } val q = Module(new Queue(new AccumulatorReadResp(t, scale_args.multiplicand_t, log2Ceil(t.head.head.getWidth)), 1, true, true)) - q.io.enq.bits.data := rdata + q.io.enq.bits.data := read_rdata q.io.enq.bits.scale := RegNext(io.read.req.bits.scale) q.io.enq.bits.relu6_shift := RegNext(io.read.req.bits.relu6_shift) q.io.enq.bits.act := RegNext(io.read.req.bits.act) From 3ef8907b25a0b4c3742128aa85dbeaa6478de773 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Feb 2021 16:11:09 -0800 Subject: [PATCH 026/123] Allow back-to-back accumulator writes if their masks do not overlap --- src/main/scala/gemmini/AccumulatorMem.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 1334dc4ce..2f50c33df 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -129,7 +129,11 @@ class AccumulatorMem[T <: Data, U <: Data]( val w_q = Reg(Vec(nEntries, new W_Q_Entry(mask_len, mask_elem))) for (e <- w_q) { when (e.valid) { - assert(!(io.write.valid && io.write.bits.acc && isThisBank(io.write.bits.addr) && getBankIdx(io.write.bits.addr) === e.addr)) + assert(!( + io.write.valid && io.write.bits.acc && + isThisBank(io.write.bits.addr) && getBankIdx(io.write.bits.addr) === e.addr && + ((io.write.bits.mask.asUInt & e.mask.asUInt) =/= 0.U) + )) when (io.read.req.valid && isThisBank(io.read.req.bits.addr) && getBankIdx(io.read.req.bits.addr) === e.addr) { reads(1).ready := false.B } From ff4825d14f1132b1b2009874d5b0bc129b0cda24 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Feb 2021 21:22:49 -0800 Subject: [PATCH 027/123] Clean up configs for scale-multplexing --- src/main/scala/gemmini/AccumulatorScale.scala | 4 +-- src/main/scala/gemmini/Configs.scala | 7 ++--- src/main/scala/gemmini/ConfigsFP.scala | 25 ++++++++---------- src/main/scala/gemmini/DSEConfigs.scala | 5 +--- src/main/scala/gemmini/GemminiConfigs.scala | 11 +++----- src/main/scala/gemmini/Scratchpad.scala | 4 --- .../gemmini/VectorScalarMultiplier.scala | 26 +++++++++---------- 7 files changed, 33 insertions(+), 49 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorScale.scala b/src/main/scala/gemmini/AccumulatorScale.scala index c54a6236d..3c9a173e1 100644 --- a/src/main/scala/gemmini/AccumulatorScale.scala +++ b/src/main/scala/gemmini/AccumulatorScale.scala @@ -26,7 +26,6 @@ class AccumulatorScaleIO[T <: Data: Arithmetic, U <: Data]( class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( fullDataType: Vec[Vec[T]], rDataType: Vec[Vec[T]], scale_t: U, shift_width: Int, - num_scale_units: Int, acc_scale_latency: Int, read_small_data: Boolean, read_full_data: Boolean, scale_args: ScaleArguments[T, U])(implicit ev: Arithmetic[T]) extends Module { @@ -38,7 +37,8 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( val out = Wire(Decoupled(new AccumulatorScaleResp[T]( fullDataType, rDataType)(ev))) - + val num_scale_units = scale_args.num_scale_units + val acc_scale_latency = scale_args.latency if (num_scale_units == -1) { val pipe_out = Pipeline(io.in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: AccumulatorReadResp[T,U]) => x) :+ { diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 3abdb8ca5..1de78cb34 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -71,7 +71,6 @@ object GemminiConfigs { outputType = SInt(20.W), accType = SInt(32.W), - num_mvin_scale_units = 4, mvin_scale_args = Some(ScaleArguments( (t: SInt, f: Float) => { val f_rec = recFNFromFN(f.expWidth, f.sigWidth, f.bits) @@ -106,15 +105,13 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 4, Float(8, 24), + 5, Float(8, 24), 4, identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (elem_t)y);})" )), mvin_scale_acc_args = None, mvin_scale_shared = false, - num_acc_scale_units = 4, - acc_scale_latency = 4, acc_scale_args = ScaleArguments( (t: SInt, f: Float) => { val f_rec = recFNFromFN(f.expWidth, f.sigWidth, f.bits) @@ -149,7 +146,7 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 0, Float(8, 24), + 4, Float(8, 24), 4, identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (acc_t)y);})" ), diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 0d9801b70..09c2db0af 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -50,14 +50,11 @@ object GemminiFPConfigs { outputType = Float(8, 24), accType = Float(8, 24), - num_mvin_scale_units = -1, // Don't multiplex - mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), - mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), mvin_scale_shared = false, - num_acc_scale_units = -1, // Don't multiplex - acc_scale_latency = 3, - acc_scale_args = ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", + acc_scale_args = ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str = "((x) * (scale))" ), acc_read_full_width = true, @@ -69,30 +66,30 @@ object GemminiFPConfigs { //FP32 Single Precision Configuration val FP32DefaultConfig = defaultFPConfig.copy(inputType = Float(8, 24), outputType = Float(8, 24), accType = Float(8, 24), pe_latency = 2, - mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), - mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), ) //FP16 Half Precision Configuration val FP16DefaultConfig = defaultFPConfig.copy(inputType = Float(5, 11), outputType = Float(5, 11), accType = Float(8, 24), pe_latency = 2, - mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(5, 11), identity = "1.0", c_str="((x) * (scale))")), - mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(5, 11), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(5, 11), -1, identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(5, 11), -1, identity = "1.0", c_str="((x) * (scale))")), ) //Bfloat16 Brain-half Precision Configuration val BF16DefaultConfig = defaultFPConfig.copy(inputType = Float(8, 8), outputType = Float(8, 8), accType = Float(8, 24), pe_latency = 2, - mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), - mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), ) //Bfloat16 Brain-half Precision Configuration 8x8 array val BF16Default8Config = defaultFPConfig.copy(inputType = Float(8, 8), outputType = Float(8, 8), accType = Float(8, 24), meshRows = 8, meshColumns = 8, pe_latency = 2, - mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), - mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 0, Float(8, 24), identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), + mvin_scale_acc_args = Some(ScaleArguments((t: Float, u: Float) => t * u, 4, Float(8, 24), -1, identity = "1.0", c_str="((x) * (scale))")), ) } diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 1d4dd7de6..54cfd2222 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -38,7 +38,6 @@ object DSEBaseConfig { inputType = SInt(8.W), outputType = SInt(19.W), accType = SInt(32.W), - num_mvin_scale_units = -1, mvin_scale_args = None, mvin_scale_acc_args = None, mvin_scale_shared = false, @@ -54,9 +53,7 @@ object DSEBaseConfig { val r = (point_five & (zeros | ones_digit)).asBool() (t >> u).asSInt() + Mux(r, 1.S, 0.S) - }, 0, UInt(8.W)), - num_acc_scale_units = -1, - acc_scale_latency = 3, + }, 0, UInt(8.W), -1), acc_read_full_width = true, acc_read_small_width = true, use_dedicated_tl_port = false, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 67db67626..2bfb22faa 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -10,6 +10,7 @@ case class CapacityInKilobytes(kilobytes: Int) extends GemminiMemCapacity case class CapacityInMatrices(matrices: Int) extends GemminiMemCapacity case class ScaleArguments[T <: Data, U <: Data](scale_func: (T, U) => T, latency: Int, multiplicand_t: U, + num_scale_units: Int, identity: String="0", c_str: String="ROUNDING_RIGHT_SHIFT(x, scale)") case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( @@ -39,11 +40,8 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( accType: T, mvin_scale_args: Option[ScaleArguments[T, U]], mvin_scale_acc_args: Option[ScaleArguments[T, U]], - num_mvin_scale_units: Int, mvin_scale_shared: Boolean, acc_scale_args: ScaleArguments[T, V], - num_acc_scale_units: Int, - acc_scale_latency: Int, hasIm2col: Boolean, pe_latency: Int, acc_read_full_width: Boolean, @@ -71,12 +69,12 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( val local_addr_t = new LocalAddr(sp_banks, sp_bank_entries, acc_banks, acc_bank_entries) val mvin_scale_t = mvin_scale_args match { - case Some(ScaleArguments(_, _, t, _, _)) => t + case Some(ScaleArguments(_, _, t, _, _, _)) => t case None => Bool() // TODO replace this with UInt(0.W) } val mvin_scale_acc_t = mvin_scale_acc_args match { - case Some(ScaleArguments(_, _, t, _, _)) => t + case Some(ScaleArguments(_, _, t, _, _, _)) => t case None => Bool() // TODO replace this with UInt(0.W) } @@ -230,7 +228,6 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( // assert(Set(8, 16, 32, 64).contains(outputType.getWidth)) assert(Set(8, 16, 32, 64).contains(accType.getWidth)) - assert(acc_scale_args.latency == 0, "Accumulator's scale latency must be 0 cycles") val header = new StringBuilder() header ++= s"#ifndef $guard\n" @@ -305,7 +302,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( header ++= s"#define row_align_acc(blocks) __attribute__((aligned(blocks*DIM*sizeof(acc_t))))\n\n" val mvin_scale_identity = mvin_scale_args match { - case Some(ScaleArguments(_, _, _, identity, _)) => identity + case Some(ScaleArguments(_, _, _, _, identity, _)) => identity case None => "0" } header ++= s"#define MVIN_SCALE_IDENTITY $mvin_scale_identity\n\n" diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 9a1410e56..a9372ff77 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -297,14 +297,12 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier( config.mvin_scale_args, config.inputType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), - num_mvin_scale_units, is_acc = false ) val (mvin_scale_acc_in, mvin_scale_acc_out) = if (mvin_scale_shared) (mvin_scale_in, mvin_scale_out) else ( VectorScalarMultiplier( config.mvin_scale_acc_args, config.accType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), - num_mvin_scale_units, is_acc = true ) ) @@ -470,8 +468,6 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, spad_row_t, acc_scale_args.multiplicand_t, log2Up(accType.getWidth), - num_acc_scale_units, - acc_scale_latency, acc_read_small_width, acc_read_full_width, acc_scale_args diff --git a/src/main/scala/gemmini/VectorScalarMultiplier.scala b/src/main/scala/gemmini/VectorScalarMultiplier.scala index cb8be5e97..e8d3adafd 100644 --- a/src/main/scala/gemmini/VectorScalarMultiplier.scala +++ b/src/main/scala/gemmini/VectorScalarMultiplier.scala @@ -25,13 +25,12 @@ class VectorScalarMultiplierResp[T <: Data, Tag <: Data](block_cols: Int, t: T, } class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( - mvin_scale_args: Option[ScaleArguments[T, U]], block_cols: Int, t: T, tag_t: Tag, - num_scale_units: Int + mvin_scale_args: Option[ScaleArguments[T, U]], block_cols: Int, t: T, tag_t: Tag ) extends Module { - val (u, always_identity) = mvin_scale_args match { - case Some(ScaleArguments(_, _, multiplicand_t, _, _)) => (multiplicand_t, false) - case None => (Bool(), true) // TODO make this a 0-width UInt + val (u, num_scale_units, always_identity) = mvin_scale_args match { + case Some(ScaleArguments(_, _, multiplicand_t, num_scale_units, _, _)) => (multiplicand_t, num_scale_units, false) + case None => (Bool(), -1, true) // TODO make this a 0-width UInt } val io = IO(new Bundle { @@ -41,7 +40,7 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( val width = block_cols val latency = mvin_scale_args match { - case Some(ScaleArguments(_, latency, _, _, _)) => latency + case Some(ScaleArguments(_, latency, _, _, _, _)) => latency case None => 0 } @@ -76,7 +75,7 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( pipe.io.in.bits.last := in.bits.repeats === 0.U && in.bits.last pipe.io.in.bits.row := in.bits.repeats pipe.io.in.bits.out := (mvin_scale_args match { - case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => + case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _, _)) => in.bits.in.map(x => mvin_scale_func(x, in.bits.scale.asTypeOf(multiplicand_t))) case None => in.bits.in }) @@ -150,11 +149,13 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) arb.io.in <> arbIn arb.io.out.ready := true.B - val arbOut = arb.io.out + val arbOut = Reg(Valid(new DataWithIndex)) + arbOut.valid := arb.io.out.valid + arbOut.bits := arb.io.out.bits val e_scaled = mvin_scale_args match { - case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _)) => - mvin_scale_func(arb.io.out.bits.data, arb.io.out.bits.scale.asTypeOf(multiplicand_t)) - case None => arb.io.out.bits.data + case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _, _)) => + mvin_scale_func(arbOut.bits.data, arbOut.bits.scale.asTypeOf(multiplicand_t)) + case None => arbOut.bits.data } val pipe_in = Wire(Valid(new DataWithIndex)) @@ -188,12 +189,11 @@ object VectorScalarMultiplier { def apply[T <: Data, U <: Data, Tag <: Data]( scale_args: Option[ScaleArguments[T, U]], t: T, cols: Int, tag_t: Tag, - num_multipliers: Int, is_acc: Boolean, is_mvin: Boolean=true ) = { assert(!is_acc || is_mvin) - val vsm = Module(new VectorScalarMultiplier(scale_args, cols, t, tag_t, num_multipliers)) + val vsm = Module(new VectorScalarMultiplier(scale_args, cols, t, tag_t)) (vsm.io.req, vsm.io.resp) } } From 2d1c508d6ce7cff7387f175caecda00976412c25 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 18 Feb 2021 01:46:06 -0800 Subject: [PATCH 028/123] debugging --- src/main/scala/gemmini/AccumulatorMem.scala | 5 +++- src/main/scala/gemmini/Scratchpad.scala | 27 +++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 2f50c33df..1eaddd15f 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -87,6 +87,8 @@ class AccumulatorMem[T <: Data, U <: Data]( val w_sum = VecInit((RegNext(acc_rdata) zip wdata_buf).map { case (rv, wv) => VecInit((rv zip wv).map(t => t._1 + t._2)) }) + val counter = RegInit(0.U(32.W)) + counter := counter + 1.U if (!acc_singleported) { val mem = TwoPortSyncMem(n, t, t.getWidth / 8) // TODO We assume byte-alignment here. Use aligned_to instead @@ -156,8 +158,9 @@ class AccumulatorMem[T <: Data, U <: Data]( } } } + when (w_buf_valid && isThisBank(waddr_buf)) { - assert(!((w_q_tail.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_))) + assert(!RegNext(((w_q_tail.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_)))) w_q_tail := w_q_tail << 1 | w_q_tail(nEntries-1) for (i <- 0 until nEntries) { when (w_q_tail(i)) { diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index a9372ff77..12bea49bf 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -594,8 +594,18 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, val zerowrite = zero_writer.io.resp.valid && zero_writer.io.resp.bits.laddr.is_acc_addr && zero_writer.io.resp.bits.laddr.acc_bank() === i.U && !((mvin_scale_out.valid && mvin_scale_out.bits.last) || (mvin_scale_acc_out.valid && mvin_scale_acc_out.bits.last)) - - bio.write.valid := exwrite || ((dmaread || zerowrite) && !spad_last) + val consecutive_write_block = RegInit(false.B) + val consecutive_write_sub_bank = RegInit(0.U(log2Ceil(num_acc_sub_banks).W)) + when (bio.write.fire() && bio.write.bits.acc && + (bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) === consecutive_write_sub_bank)) { + consecutive_write_block := true.B + } .elsewhen (bio.write.fire() && bio.write.bits.acc) { + consecutive_write_block := false.B + consecutive_write_sub_bank := bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) + } .otherwise { + consecutive_write_block := false.B + } + bio.write.valid := false.B bio.write.bits.acc := MuxCase(zero_writer.io.resp.bits.laddr.accumulate, Seq(exwrite -> io.acc.write(i).bits.acc, @@ -607,9 +617,11 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, (from_mvin_scale || from_mvin_scale_acc) -> dmaread_row)) when (exwrite) { + bio.write.valid := true.B bio.write.bits.data := io.acc.write(i).bits.data bio.write.bits.mask := io.acc.write(i).bits.mask - }.elsewhen (dmaread && bio.write.fire()) { + }.elsewhen (dmaread && !spad_last && !consecutive_write_block) { + bio.write.valid := true.B bio.write.bits.data := Mux(from_mvin_scale, VecInit(mvin_scale_out.bits.out.map(e => e.withWidthOf(accType))).asTypeOf(acc_row_t), mvin_scale_acc_out.bits.out.asTypeOf(acc_row_t)) @@ -624,11 +636,12 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, mvin_scale_acc_out.bits.tag.mask) when(from_mvin_scale) { - mvin_scale_out.ready := true.B + mvin_scale_out.ready := bio.write.ready }.otherwise { - mvin_scale_acc_out.ready := true.B + mvin_scale_acc_out.ready := bio.write.ready } - }.elsewhen (zerowrite && bio.write.fire()) { + }.elsewhen (zerowrite && !spad_last && !consecutive_write_block) { + bio.write.valid := true.B bio.write.bits.data := 0.U.asTypeOf(acc_row_t) bio.write.bits.mask := { val n = accType.getWidth / 8 @@ -637,7 +650,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, expanded } - zero_writer.io.resp.ready := true.B + zero_writer.io.resp.ready := bio.write.ready }.otherwise { bio.write.bits.data := DontCare bio.write.bits.mask := DontCare From adf41e23f3ef5c7b28b8ea15fd7943d15b524718 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 18 Feb 2021 17:48:06 -0800 Subject: [PATCH 029/123] Move scale units into ScalePipe modules for better retiming --- src/main/scala/gemmini/AccumulatorScale.scala | 67 ++++++++++++------- .../gemmini/VectorScalarMultiplier.scala | 49 +++++++++----- 2 files changed, 73 insertions(+), 43 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorScale.scala b/src/main/scala/gemmini/AccumulatorScale.scala index 3c9a173e1..20e7fc41d 100644 --- a/src/main/scala/gemmini/AccumulatorScale.scala +++ b/src/main/scala/gemmini/AccumulatorScale.scala @@ -23,6 +23,40 @@ class AccumulatorScaleIO[T <: Data: Arithmetic, U <: Data]( shift_width, rDataType).asInstanceOf[this.type] } +class AccScaleDataWithIndex[T <: Data: Arithmetic, U <: Data](t: T, u: U, scale_args: ScaleArguments[T, U]) extends Bundle { + val shift_width = log2Ceil(t.getWidth) + + val scale = u.cloneType + val act = UInt(2.W) + val relu6_shift = UInt(shift_width.W) + val data = t.cloneType + val full_data = t.cloneType + val id = UInt(2.W) // TODO hardcoded + val index = UInt() + override def cloneType: this.type = new AccScaleDataWithIndex(t, u, scale_args: ScaleArguments[T, U]).asInstanceOf[this.type] +} + +class AccScalePipe[T <: Data : Arithmetic, U <: Data](t: T, rDataType: Vec[Vec[T]], scale_args: ScaleArguments[T, U])(implicit ev: Arithmetic[T]) extends Module { + val u = scale_args.multiplicand_t + val io = IO(new Bundle { + val in = Input(Valid(new AccScaleDataWithIndex(t, u, scale_args)(ev))) + val out = Output(Valid(new AccScaleDataWithIndex(t, u, scale_args)(ev))) + }) + import ev._ + val latency = scale_args.latency + val out = WireInit(io.in) + + val e_scaled = scale_args.scale_func(io.in.bits.data, io.in.bits.scale) + val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) + val e_act = MuxCase(e_clipped, Seq( + (io.in.bits.act === Activation.RELU) -> e_clipped.relu, + (io.in.bits.act === Activation.RELU6) -> e_clipped.relu6(io.in.bits.relu6_shift))) + + out.bits.data := e_act + io.out := Pipe(out, latency) +} + + class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( fullDataType: Vec[Vec[T]], rDataType: Vec[Vec[T]], scale_t: U, shift_width: Int, @@ -33,7 +67,7 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( val io = IO(new AccumulatorScaleIO[T,U]( fullDataType, scale_t, shift_width, rDataType )(ev)) - + val t = io.in.bits.data(0)(0).cloneType val out = Wire(Decoupled(new AccumulatorScaleResp[T]( fullDataType, rDataType)(ev))) @@ -101,17 +135,7 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) } - - class DataWithIndex extends Bundle { - val scale = io.in.bits.scale.cloneType - val act = io.in.bits.act.cloneType - val relu6_shift = io.in.bits.relu6_shift.cloneType - val data = io.in.bits.data(0)(0).cloneType - val full_data = io.in.bits.data(0)(0).cloneType - val id = UInt(2.W) // TODO hardcoded - val index = UInt() - } - val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } + val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new AccScaleDataWithIndex(t, scale_t, scale_args)(ev))) } for (i <- 0 until nEntries) { for (w <- 0 until width) { @@ -131,26 +155,19 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( } for (i <- 0 until num_scale_units) { val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) - val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) + val arb = Module(new RRArbiter(new AccScaleDataWithIndex(t, scale_t, scale_args)(ev), arbIn.length)) arb.io.in <> arbIn arb.io.out.ready := true.B - val arbOut = Reg(Valid(new DataWithIndex)) + val arbOut = Reg(Valid(new AccScaleDataWithIndex(t, scale_t, scale_args)(ev))) arbOut.valid := arb.io.out.valid arbOut.bits := arb.io.out.bits when (reset.asBool) { arbOut.valid := false.B } - val e_scaled = scale_args.scale_func(arbOut.bits.data, arbOut.bits.scale) - val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) - val e_act = MuxCase(e_clipped, Seq( - (arbOut.bits.act === Activation.RELU) -> e_clipped.relu, - (arbOut.bits.act === Activation.RELU6) -> e_clipped.relu6(arbOut.bits.relu6_shift) - )) - val pipe_in = Wire(Valid(new DataWithIndex)) - pipe_in.valid := arbOut.valid - pipe_in.bits := arbOut.bits - pipe_in.bits.data := e_act - val pipe_out = Pipe(pipe_in, acc_scale_latency) + val pipe = Module(new AccScalePipe(t, rDataType, scale_args)(ev, ev)) + pipe.io.in := arbOut + val pipe_out = pipe.io.out + for (j <- 0 until nEntries) { for (w <- 0 until width) { if ((j*width+w) % num_scale_units == i) { diff --git a/src/main/scala/gemmini/VectorScalarMultiplier.scala b/src/main/scala/gemmini/VectorScalarMultiplier.scala index e8d3adafd..d1cefcb3f 100644 --- a/src/main/scala/gemmini/VectorScalarMultiplier.scala +++ b/src/main/scala/gemmini/VectorScalarMultiplier.scala @@ -24,6 +24,26 @@ class VectorScalarMultiplierResp[T <: Data, Tag <: Data](block_cols: Int, t: T, override def cloneType: VectorScalarMultiplierResp.this.type = new VectorScalarMultiplierResp(block_cols, t, tag_t).asInstanceOf[this.type] } +class DataWithIndex[T <: Data, U <: Data](t: T, u: U) extends Bundle { + val data = t.cloneType + val scale = u.cloneType + val id = UInt(2.W) // TODO hardcoded + val index = UInt() + override def cloneType: DataWithIndex.this.type = new DataWithIndex(t, u).asInstanceOf[this.type] +} + +class ScalePipe[T <: Data, U <: Data](t: T, mvin_scale_args: ScaleArguments[T, U]) extends Module { + val u = mvin_scale_args.multiplicand_t + val io = IO(new Bundle { + val in = Input(Valid(new DataWithIndex(t, u))) + val out = Output(Valid(new DataWithIndex(t, u))) + }) + val latency = mvin_scale_args.latency + val out = WireInit(io.in) + out.bits.data := mvin_scale_args.scale_func(io.in.bits.data, io.in.bits.scale.asTypeOf(u)) + io.out := Pipe(out, latency) +} + class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( mvin_scale_args: Option[ScaleArguments[T, U]], block_cols: Int, t: T, tag_t: Tag ) extends Module { @@ -124,13 +144,9 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( tail_oh := (tail_oh << 1) | tail_oh(nEntries-1) } - class DataWithIndex extends Bundle { - val data = in.bits.in(0).cloneType - val scale = u.cloneType - val id = UInt(2.W) // TODO hardcoded - val index = UInt() - } - val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex)) } + + + val inputs = Seq.fill(width*nEntries) { Wire(Decoupled(new DataWithIndex(t, u))) } for (i <- 0 until nEntries) { for (w <- 0 until width) { val input = inputs(i*width+w) @@ -146,23 +162,20 @@ class VectorScalarMultiplier[T <: Data, U <: Data, Tag <: Data]( } for (i <- 0 until num_scale_units) { val arbIn = inputs.zipWithIndex.filter({ case (_, w) => w % num_scale_units == i }).map(_._1) - val arb = Module(new RRArbiter(new DataWithIndex, arbIn.length)) + val arb = Module(new RRArbiter(new DataWithIndex(t, u), arbIn.length)) arb.io.in <> arbIn arb.io.out.ready := true.B - val arbOut = Reg(Valid(new DataWithIndex)) + val arbOut = Reg(Valid(new DataWithIndex(t, u))) arbOut.valid := arb.io.out.valid arbOut.bits := arb.io.out.bits - val e_scaled = mvin_scale_args match { - case Some(ScaleArguments(mvin_scale_func, _, multiplicand_t, _, _, _)) => - mvin_scale_func(arbOut.bits.data, arbOut.bits.scale.asTypeOf(multiplicand_t)) - case None => arbOut.bits.data + when (reset.asBool) { + arbOut.valid := false.B } - val pipe_in = Wire(Valid(new DataWithIndex)) - pipe_in.valid := arbOut.valid - pipe_in.bits := arbOut.bits - pipe_in.bits.data := e_scaled - val pipe_out = Pipe(pipe_in, latency) + + val pipe = Module(new ScalePipe(t, mvin_scale_args.get)) + pipe.io.in := arbOut + val pipe_out = pipe.io.out for (j <- 0 until nEntries) { for (w <- 0 until width) { if ((j*width+w) % num_scale_units == i) { From e1d512f19d215b0b170d0828dc2785fd264790e5 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 19 Feb 2021 00:03:53 -0800 Subject: [PATCH 030/123] Bump acc-scale pipelining to 5 --- src/main/scala/gemmini/Configs.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 1de78cb34..8acb737a5 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -146,7 +146,7 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 4, Float(8, 24), 4, + 5, Float(8, 24), 4, identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (acc_t)y);})" ), From a356ec6dd6a72ee17c12e52b3dd1cec08080ef76 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Fri, 19 Feb 2021 12:42:00 -0500 Subject: [PATCH 031/123] Fix tlb req cmd for DMA reads (#68) --- src/main/scala/gemmini/DMA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index bbd9b5073..c12b28ac6 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -218,7 +218,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf io.tlb.req.bits.tlb_req.vaddr := tlb_q.io.deq.bits.vaddr io.tlb.req.bits.tlb_req.passthrough := false.B io.tlb.req.bits.tlb_req.size := 0.U // send_size - io.tlb.req.bits.tlb_req.cmd := M_XWR + io.tlb.req.bits.tlb_req.cmd := M_XRD io.tlb.req.bits.status := tlb_q.io.deq.bits.status val translate_q = Module(new Queue(new TLBundleAWithInfo, 1, pipe=true)) From 1f704d60c218ce0a99f6d7cbd587728cd86279b3 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 24 Feb 2021 17:50:11 +0000 Subject: [PATCH 032/123] Merge bad_dataflow asserts into a single assert --- src/main/scala/gemmini/Mesh.scala | 2 ++ src/main/scala/gemmini/PE.scala | 6 +++++- src/main/scala/gemmini/Tile.scala | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/Mesh.scala b/src/main/scala/gemmini/Mesh.scala index 5f50c992e..22ece6f35 100644 --- a/src/main/scala/gemmini/Mesh.scala +++ b/src/main/scala/gemmini/Mesh.scala @@ -57,6 +57,7 @@ class Mesh[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, } } // Chain control signals (pipeline across each column) + assert(!(mesh.map(_.map(_.io.bad_dataflow).reduce(_||_)).reduce(_||_))) for (c <- 0 until meshColumns) { meshT(c).foldLeft((io.in_control(c), io.in_valid(c))) { case ((in_ctrl, valid), tile) => @@ -68,6 +69,7 @@ class Mesh[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, (tile.io.out_control, tile.io.out_valid) } } + // Chain in_valid (pipeline across each column) for (c <- 0 until meshColumns) { meshT(c).foldLeft(io.in_valid(c)) { diff --git a/src/main/scala/gemmini/PE.scala b/src/main/scala/gemmini/PE.scala index b912ad34e..7c17cc394 100644 --- a/src/main/scala/gemmini/PE.scala +++ b/src/main/scala/gemmini/PE.scala @@ -34,6 +34,8 @@ class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, val in_valid = Input(Bool()) val out_valid = Output(Bool()) + + val bad_dataflow = Output(Bool()) }) val cType = if (df == Dataflow.WS) inputType else accType @@ -66,6 +68,7 @@ class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, val COMPUTE = 0.U(1.W) val PROPAGATE = 1.U(1.W) + io.bad_dataflow := false.B when ((df == Dataflow.OS).B || ((df == Dataflow.BOTH).B && dataflow === OUTPUT_STATIONARY)) { when(prop === PROPAGATE) { io.out_c := (c1 >> shift_offset).clippedToWidthOf(outputType) @@ -89,7 +92,8 @@ class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, c2 := d } }.otherwise { - assert(false.B, "unknown dataflow") + io.bad_dataflow := true.B + //assert(false.B, "unknown dataflow") io.out_c := DontCare io.out_b := DontCare } diff --git a/src/main/scala/gemmini/Tile.scala b/src/main/scala/gemmini/Tile.scala index 69b606b81..1a2bfe745 100644 --- a/src/main/scala/gemmini/Tile.scala +++ b/src/main/scala/gemmini/Tile.scala @@ -25,6 +25,8 @@ class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: val in_valid = Input(Vec(columns, Bool())) val out_valid = Output(Vec(columns, Bool())) + + val bad_dataflow = Output(Bool()) }) val tile = Seq.fill(rows, columns)(Module(new PE(inputType, outputType, accType, df, pe_latency))) @@ -83,6 +85,7 @@ class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: io.out_control(c) := tile(rows-1)(c).io.out_control io.out_valid(c) := tile(rows-1)(c).io.out_valid } + io.bad_dataflow := tile.map(_.map(_.io.bad_dataflow).reduce(_||_)).reduce(_||_) // Drive the Tile's right IO for (r <- 0 until rows) { From 9a92fa07e7432376331e5f013cc00695d6ac783a Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 Feb 2021 22:19:55 -0800 Subject: [PATCH 033/123] Include used opcode in gemmini_params.h | clean up configs --- SPIKE.hash | 2 +- software/gemmini-rocc-tests | 2 +- src/main/scala/gemmini/Configs.scala | 38 ++++++++------------- src/main/scala/gemmini/ConfigsFP.scala | 9 ++--- src/main/scala/gemmini/Controller.scala | 4 +-- src/main/scala/gemmini/DSEConfigs.scala | 25 +++++++------- src/main/scala/gemmini/GemminiConfigs.scala | 9 +++++ 7 files changed, 46 insertions(+), 43 deletions(-) diff --git a/SPIKE.hash b/SPIKE.hash index 3ab5b634f..10f560bc1 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -a4ed25a96fdb47642b39d893b7e1ca36d07700aa +cd7d15b889844f730fa8e6d5688555ec584f876d diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index dcbc4cdbe..5ef85f78a 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit dcbc4cdbe2873355a6ae96b16e11a68e6a926f0d +Subproject commit 5ef85f78aea8a41ed7fe0bf1b8b2d10daefa5827 diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index abcaaf87d..21a982e94 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -39,6 +39,7 @@ object GemminiConfigs { val defaultConfig = GemminiArrayConfig[SInt, Float, Float]( // val defaultConfig = GemminiArrayConfig[Float, Float]( + opcodes = OpcodeSet.custom3, tileRows = 1, tileColumns = 1, // meshRows = 4, @@ -180,6 +181,11 @@ object GemminiConfigs { use_tlb_register_filter = true, max_in_flight_reqs = 16, ) + + val chipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(64), acc_capacity=CapacityInKilobytes(32), dataflow=Dataflow.WS) + val largeChipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), dataflow=Dataflow.WS, + meshRows=32, meshColumns=32 + ) } /** @@ -187,33 +193,19 @@ object GemminiConfigs { Also sets the system bus width to 128 bits (instead of the deafult 64 bits) to allow for the default 16x16 8-bit systolic array to be attached. */ -class DefaultGemminiConfig extends Config((site, here, up) => { +class DefaultGemminiConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( + gemminiConfig: GemminiArrayConfig[T,U,V] = GemminiConfigs.defaultConfig +) extends Config((site, here, up) => { case BuildRoCC => up(BuildRoCC) ++ Seq( - (p: Parameters) => { - implicit val q = p - val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig)) - gemmini + (p: Parameters) => { + implicit val q = p + val gemmini = LazyModule(new Gemmini(gemminiConfig)) + gemmini } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) }) -// Default feature for initial Gemmini Chip tape-out experiments -// ToDo: increase & decrease spad/mesh size, single ported SRAM, increase in flight requests -class DefaultGemminiChipConfig extends Config((site, here, up) => { - case BuildRoCC => up(BuildRoCC) ++ Seq( - (p: Parameters) => { - implicit val q = p - val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig.copy( - sp_capacity=CapacityInKilobytes(64), - acc_capacity=CapacityInKilobytes(32), - dataflow = Dataflow.WS - ))) - gemmini - } - ) - case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) -}) /** * Mixin which configures a smaller host processor for the systolic array. This mixin **replaces** the default host rocket (assuming a single core config). @@ -247,7 +239,7 @@ class GemminiHostMiniCore extends Config((site, here, up) => { (up(RocketTilesKey, site).length - 1 -> Seq((p: Parameters) => { implicit val q = p - val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig)) + val gemmini = LazyModule(new Gemmini(GemminiConfigs.defaultConfig)) gemmini })) }) @@ -286,7 +278,7 @@ class WithGemminiHostMiniCore extends Config((site, here, up) => { (up(RocketTilesKey, site).length -> Seq((p: Parameters) => { implicit val q = p - val gemmini = LazyModule(new Gemmini(OpcodeSet.custom3, GemminiConfigs.defaultConfig)) + val gemmini = LazyModule(new Gemmini(GemminiConfigs.defaultConfig)) gemmini })) }) diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 9c5aa6593..4a210f800 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -14,6 +14,7 @@ import freechips.rocketchip.tile.{BuildRoCC, OpcodeSet} object GemminiFPConfigs { import Arithmetic.FloatArithmetic._ val defaultFPConfig = GemminiArrayConfig[Float, Float, Float]( + opcodes = OpcodeSet.custom2, tileRows = 1, tileColumns = 1, meshRows = 4, @@ -98,7 +99,7 @@ class GemminiFP32DefaultConfig extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.FP32DefaultConfig)) + LazyModule(new Gemmini(GemminiFPConfigs.FP32DefaultConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -111,7 +112,7 @@ class GemminiFP16DefaultConfig extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.FP16DefaultConfig)) + LazyModule(new Gemmini(GemminiFPConfigs.FP16DefaultConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -123,7 +124,7 @@ class GemminiBF16DefaultConfig extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.BF16DefaultConfig)) + LazyModule(new Gemmini(GemminiFPConfigs.BF16DefaultConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -135,7 +136,7 @@ class GemminiBF16Default8Config extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, GemminiFPConfigs.BF16Default8Config)) + LazyModule(new Gemmini(GemminiFPConfigs.BF16Default8Config)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 3df1e3aca..e501a6800 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -94,10 +94,10 @@ class LocalAddr(sp_banks: Int, sp_bank_entries: Int, acc_banks: Int, acc_bank_en override def cloneType: LocalAddr.this.type = new LocalAddr(sp_banks, sp_bank_entries, acc_banks, acc_bank_entries).asInstanceOf[this.type] } -class Gemmini[T <: Data : Arithmetic, U <: Data, V <: Data](opcodes: OpcodeSet, val config: GemminiArrayConfig[T, U, V]) +class Gemmini[T <: Data : Arithmetic, U <: Data, V <: Data](val config: GemminiArrayConfig[T, U, V]) (implicit p: Parameters) extends LazyRoCC ( - opcodes = OpcodeSet.custom3, + opcodes = config.opcodes, nPTWPorts = 1) { Files.write(Paths.get(config.headerFilePath), config.generateHeader().getBytes(StandardCharsets.UTF_8)) diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index a0d0aa85b..850f0cdf4 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -13,6 +13,7 @@ import freechips.rocketchip.tile.{BuildRoCC, OpcodeSet} object DSEBaseConfig { val baseConfig = GemminiArrayConfig[SInt, Bool, UInt]( + opcodes = OpcodeSet.custom3, tileRows = 1, tileColumns = 1, meshRows = 16, @@ -91,7 +92,7 @@ class GemminiParamsDSE1 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.baseConfig)) + LazyModule(new Gemmini(DSEConfigs.baseConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -103,7 +104,7 @@ class GemminiParamsDSE2 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.wsOnlyConfig)) + LazyModule(new Gemmini(DSEConfigs.wsOnlyConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -115,7 +116,7 @@ class GemminiParamsDSE3 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.bothDataflowsConfig)) + LazyModule(new Gemmini(DSEConfigs.bothDataflowsConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -127,7 +128,7 @@ class GemminiParamsDSE4 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.highBitwidthConfig)) + LazyModule(new Gemmini(DSEConfigs.highBitwidthConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -139,7 +140,7 @@ class GemminiParamsDSE5 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.largerDimConfig)) + LazyModule(new Gemmini(DSEConfigs.largerDimConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -151,7 +152,7 @@ class GemminiParamsDSE6 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.fullyCombinationalConfig)) + LazyModule(new Gemmini(DSEConfigs.fullyCombinationalConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -163,7 +164,7 @@ class GemminiParamsDSE7 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.moreMemoryConfig)) + LazyModule(new Gemmini(DSEConfigs.moreMemoryConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -175,7 +176,7 @@ class GemminiParamsDSE8 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.moreBanksConfig)) + LazyModule(new Gemmini(DSEConfigs.moreBanksConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -187,7 +188,7 @@ class GemminiParamsDSE10 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.narrowerBusConfig)) + LazyModule(new Gemmini(DSEConfigs.narrowerBusConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 8) @@ -199,7 +200,7 @@ class GemminiParamsPnR16 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.pnr16Config)) + LazyModule(new Gemmini(DSEConfigs.pnr16Config)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -211,7 +212,7 @@ class GemminiParamsPnR32 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.pnr32Config)) + LazyModule(new Gemmini(DSEConfigs.pnr32Config)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) @@ -223,7 +224,7 @@ class GemminiParamsDSE11 extends Config((site, here, up) => { (p: Parameters) => { implicit val q = p implicit val v = implicitly[ValName] - LazyModule(new Gemmini(OpcodeSet.custom3, DSEConfigs.baseConfig)) + LazyModule(new Gemmini(DSEConfigs.baseConfig)) } ) case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 373edd77f..ada4efadb 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -4,6 +4,7 @@ package gemmini import scala.math.{pow,sqrt} import chisel3._ import chisel3.util._ +import freechips.rocketchip.tile._ sealed abstract trait GemminiMemCapacity case class CapacityInKilobytes(kilobytes: Int) extends GemminiMemCapacity @@ -13,6 +14,7 @@ case class ScaleArguments[T <: Data, U <: Data](scale_func: (T, U) => T, latency identity: String="0", c_str: String="ROUNDING_RIGHT_SHIFT(x, scale)") case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( + opcodes: OpcodeSet, tileRows: Int, tileColumns: Int, meshRows: Int, @@ -234,6 +236,13 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( header ++= s"#include \n" header ++= s"#include \n\n" + val opcodeid = Seq( + OpcodeSet.custom0, OpcodeSet.custom1, OpcodeSet.custom2, OpcodeSet.custom3 + ).indexWhere(o => o.opcodes(0).litValue == opcodes.opcodes(0).litValue) + println(opcodeid, opcodes.opcodes) + require (opcodeid != -1 && opcodes.opcodes.size == 1) + header ++= s"#define XCUSTOM_ACC $opcodeid\n" + header ++= s"#define DIM ${tileColumns*meshColumns}\n" header ++= s"#define ADDR_LEN 32\n" header ++= s"#define BANK_NUM $sp_banks\n" From be80e3919d903878fe23c466b023818cd4b59458 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 25 Feb 2021 23:02:26 -0800 Subject: [PATCH 034/123] Bump Chipyard --- CHIPYARD.hash | 2 +- src/main/scala/gemmini/ConfigsFP.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHIPYARD.hash b/CHIPYARD.hash index 7c2445815..70a1842b7 100644 --- a/CHIPYARD.hash +++ b/CHIPYARD.hash @@ -1 +1 @@ -6b0d57d60690cc223013ea228b687b519b716c50 +1e2f778a6705033d67ccbcc932e66083e4646f15 diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 4a210f800..5dfd3153c 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -14,7 +14,7 @@ import freechips.rocketchip.tile.{BuildRoCC, OpcodeSet} object GemminiFPConfigs { import Arithmetic.FloatArithmetic._ val defaultFPConfig = GemminiArrayConfig[Float, Float, Float]( - opcodes = OpcodeSet.custom2, + opcodes = OpcodeSet.custom3, tileRows = 1, tileColumns = 1, meshRows = 4, From b29a14bddcf28929bb1b831de76bda8771f2250c Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Fri, 26 Feb 2021 17:01:18 -0800 Subject: [PATCH 035/123] Bump Spike --- SPIKE.hash | 2 +- software/gemmini-rocc-tests | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/SPIKE.hash b/SPIKE.hash index 10f560bc1..30fadc6c9 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -cd7d15b889844f730fa8e6d5688555ec584f876d +86265d02e8abea3b367114393d6b0661fd35b156 diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 5ef85f78a..1d6e1cd66 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 5ef85f78aea8a41ed7fe0bf1b8b2d10daefa5827 +Subproject commit 1d6e1cd66d586a4c1b073f524dd013f64e3f05f1 From 98374db9f706a88972097af63dd0271fafec4c25 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Tue, 2 Mar 2021 23:56:37 -0800 Subject: [PATCH 036/123] Fix support for dualported unbanked accumulators --- src/main/scala/gemmini/Scratchpad.scala | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 12bea49bf..77ac7e877 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -595,15 +595,17 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, zero_writer.io.resp.bits.laddr.acc_bank() === i.U && !((mvin_scale_out.valid && mvin_scale_out.bits.last) || (mvin_scale_acc_out.valid && mvin_scale_acc_out.bits.last)) val consecutive_write_block = RegInit(false.B) - val consecutive_write_sub_bank = RegInit(0.U(log2Ceil(num_acc_sub_banks).W)) - when (bio.write.fire() && bio.write.bits.acc && - (bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) === consecutive_write_sub_bank)) { - consecutive_write_block := true.B - } .elsewhen (bio.write.fire() && bio.write.bits.acc) { - consecutive_write_block := false.B - consecutive_write_sub_bank := bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) - } .otherwise { - consecutive_write_block := false.B + if (acc_singleported) { + val consecutive_write_sub_bank = RegInit(0.U((1 max log2Ceil(num_acc_sub_banks)).W)) + when (bio.write.fire() && bio.write.bits.acc && + (bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) === consecutive_write_sub_bank)) { + consecutive_write_block := true.B + } .elsewhen (bio.write.fire() && bio.write.bits.acc) { + consecutive_write_block := false.B + consecutive_write_sub_bank := bio.write.bits.addr(log2Ceil(num_acc_sub_banks)-1,0) + } .otherwise { + consecutive_write_block := false.B + } } bio.write.valid := false.B From 961f625f88d7dfc049b0d8c72d29e4b94d890679 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 6 Mar 2021 21:14:40 -0800 Subject: [PATCH 037/123] Scale LoopMatmul with rob size --- src/main/scala/gemmini/Controller.scala | 2 +- src/main/scala/gemmini/LoopMatmul.scala | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 3df1e3aca..e71bde223 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -195,7 +195,7 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, + meshRows*tileRows, coreMaxAddrBits, rob_entries, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) val unrolled_cmd = Queue(loop_cmd) unrolled_cmd.ready := false.B diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 5932a51a0..bbf97d392 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -888,9 +888,13 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: object LoopMatmul { def apply(in: DecoupledIO[RoCCCommand], ld_utilization: UInt, st_utilization: UInt, ex_utilization: UInt, - block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: Int, max_exs: Int, max_sts: Int, + block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_addr: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, dma_max_bytes: Int) (implicit p: Parameters): Tuple2[DecoupledIO[RoCCCommand], Bool] = { + + val max_lds = rob_size * 1 / 4 + val max_exs = rob_size * 3 / 4 + val max_sts = rob_size * 1 / 8 val mod = Module(new LoopMatmul(block_size, coreMaxAddrBits, rob_size, max_lds, max_exs, max_sts, max_addr, max_acc_addr, input_w, acc_w, dma_max_bytes)) mod.io.in <> in From 347c48b940e109c7c81a8428f6f01be5bdf3a2e0 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 6 Mar 2021 21:30:13 -0800 Subject: [PATCH 038/123] Move max_lds/exs/sts calculation to Controller.scala --- src/main/scala/gemmini/Controller.scala | 6 +++++- src/main/scala/gemmini/LoopMatmul.scala | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index e71bde223..09fe48b18 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -194,8 +194,12 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] // compressed_cmd.ready := false.B // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + + val max_lds = rob_entries * 1 / 4 + val max_exs = rob_entries * 3 / 4 + val max_sts = rob_entries * 1 / 8 val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - meshRows*tileRows, coreMaxAddrBits, rob_entries, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, + meshRows*tileRows, coreMaxAddrBits, rob_entries, max_lds, max_exs, max_sts, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) val unrolled_cmd = Queue(loop_cmd) unrolled_cmd.ready := false.B diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index bbf97d392..5932a51a0 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -888,13 +888,9 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: object LoopMatmul { def apply(in: DecoupledIO[RoCCCommand], ld_utilization: UInt, st_utilization: UInt, ex_utilization: UInt, - block_size: Int, coreMaxAddrBits: Int, rob_size: Int, + block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: Int, max_exs: Int, max_sts: Int, max_addr: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, dma_max_bytes: Int) (implicit p: Parameters): Tuple2[DecoupledIO[RoCCCommand], Bool] = { - - val max_lds = rob_size * 1 / 4 - val max_exs = rob_size * 3 / 4 - val max_sts = rob_size * 1 / 8 val mod = Module(new LoopMatmul(block_size, coreMaxAddrBits, rob_size, max_lds, max_exs, max_sts, max_addr, max_acc_addr, input_w, acc_w, dma_max_bytes)) mod.io.in <> in From a7b0d8c79b7e4835ddc3cefc47d04e61ab9c8df3 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 10 Mar 2021 22:32:14 +0400 Subject: [PATCH 039/123] Fix conv FSM freeze (#69) Made these changes to fix conv FSM: * Fix a_stride updates in ROB * Make the ROB's block_mvin raw checks less cautious * Stop GARBAGE_ADDRs from being considered as hazards in ROB * Fix a typo in `LoopConv/LdInput` Made these changes to improve performance: * Replaced downsampling layers with a series of strided matmuls * Improved conv tiling factor selection * Added block-mvout for resadds * Add config options for the ExecuteController to only read data from the spad, and only write data to the accumulator. This eliminates RAW-hazard checking delays in the ExecuteController. Performance: * ResNet50: 153709356 cycles (40% utilization) --- SPIKE.hash | 2 +- software/gemmini-rocc-tests | 2 +- src/main/scala/gemmini/Configs.scala | 9 +- src/main/scala/gemmini/ConfigsFP.scala | 20 ++ src/main/scala/gemmini/Controller.scala | 86 +-------- src/main/scala/gemmini/DMA.scala | 42 +++-- src/main/scala/gemmini/DSEConfigs.scala | 5 + .../scala/gemmini/ExecuteController.scala | 177 +++++++++++------- src/main/scala/gemmini/FrontendTLB.scala | 1 - src/main/scala/gemmini/GemminiConfigs.scala | 29 ++- src/main/scala/gemmini/Im2Col.scala | 1 + src/main/scala/gemmini/LocalAddr.scala | 83 ++++++++ src/main/scala/gemmini/LoopConv.scala | 3 +- src/main/scala/gemmini/LoopMatmul.scala | 22 ++- src/main/scala/gemmini/ROB.scala | 113 +++++++---- src/main/scala/gemmini/Scratchpad.scala | 4 + src/main/scala/gemmini/StoreController.scala | 43 +++-- 17 files changed, 413 insertions(+), 229 deletions(-) create mode 100644 src/main/scala/gemmini/LocalAddr.scala diff --git a/SPIKE.hash b/SPIKE.hash index 30fadc6c9..ccafc0dbd 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -86265d02e8abea3b367114393d6b0661fd35b156 +9b0082a416a4f1967fda434c7129953fad77b2af diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 1d6e1cd66..463e3eebd 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 1d6e1cd66d586a4c1b073f524dd013f64e3f05f1 +Subproject commit 463e3eebdfd96bb2874a42556ce0337688ef817a diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 21a982e94..fcdd05cee 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -58,7 +58,7 @@ object GemminiConfigs { dataflow = Dataflow.BOTH, acc_capacity = CapacityInKilobytes(64), mem_pipeline = 4, - hasIm2col = true, //declare im2col block + hasIm2col = false, //declare im2col block dma_maxbytes = 64, // TODO get this from cacheblockbytes dma_buswidth = 128, // TODO get this from SystemBusKey aligned_to = 1, @@ -177,6 +177,11 @@ object GemminiConfigs { use_dedicated_tl_port = false, pe_latency = 0, + ex_read_from_spad = true, + ex_read_from_acc = true, + ex_write_to_spad = true, + ex_write_to_acc = true, + tlb_size = 4, use_tlb_register_filter = true, max_in_flight_reqs = 16, @@ -186,6 +191,8 @@ object GemminiConfigs { val largeChipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), dataflow=Dataflow.WS, meshRows=32, meshColumns=32 ) + + val highPerfConfig = defaultConfig.copy(dataflow=Dataflow.WS, acc_read_full_width = false, ex_read_from_acc = false, ex_write_to_spad = false, max_in_flight_reqs = 64) } /** diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 5dfd3153c..65b747d7d 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -59,6 +59,11 @@ object GemminiFPConfigs { acc_read_small_width = true, pe_latency = 1, + + ex_read_from_spad = true, + ex_read_from_acc = true, + ex_write_to_spad = true, + ex_write_to_acc = true, ) //FP32 Single Precision Configuration @@ -130,6 +135,21 @@ class GemminiBF16DefaultConfig extends Config((site, here, up) => { case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) }) +class GemminiBF16DefaultHighPerfConfig extends Config((site, here, up) => { + case BuildRoCC => Seq( + (p: Parameters) => { + implicit val q = p + implicit val v = implicitly[ValName] + val gemmini = LazyModule(new Gemmini(GemminiFPConfigs.BF16DefaultConfig.copy( + ex_read_from_acc = false, + ex_write_to_spad = false, + ))) + gemmini + } + ) + case SystemBusKey => up(SystemBusKey).copy(beatBytes = 16) +}) + //===========BFLOAT16 Default Config 8x8========= class GemminiBF16Default8Config extends Config((site, here, up) => { case BuildRoCC => Seq( diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 83d781d24..75d59c755 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -20,80 +20,6 @@ class GemminiCmd(rob_entries: Int)(implicit p: Parameters) extends Bundle { override def cloneType: this.type = new GemminiCmd(rob_entries).asInstanceOf[this.type] } - -class LocalAddr(sp_banks: Int, sp_bank_entries: Int, acc_banks: Int, acc_bank_entries: Int) extends Bundle { - private val localAddrBits = 32 // TODO magic number - - private val spAddrBits = log2Ceil(sp_banks * sp_bank_entries) - private val accAddrBits = log2Ceil(acc_banks * acc_bank_entries) - private val maxAddrBits = spAddrBits max accAddrBits - - private val spBankBits = log2Up(sp_banks) - private val spBankRowBits = log2Up(sp_bank_entries) - - private val accBankBits = log2Up(acc_banks) - private val accBankRowBits = log2Up(acc_bank_entries) - - val is_acc_addr = Bool() - val accumulate = Bool() - val read_full_acc_row = Bool() - val garbage = UInt(((localAddrBits - maxAddrBits - 4) max 0).W) - val garbage_bit = if (localAddrBits - maxAddrBits >= 4) UInt(1.W) else UInt(0.W) - val data = UInt(maxAddrBits.W) - - def sp_bank(dummy: Int = 0) = if (spAddrBits == spBankRowBits) 0.U else data(spAddrBits - 1, spBankRowBits) - def sp_row(dummy: Int = 0) = data(spBankRowBits - 1, 0) - def acc_bank(dummy: Int = 0) = if (accAddrBits == accBankRowBits) 0.U else data(accAddrBits - 1, accBankRowBits) - def acc_row(dummy: Int = 0) = data(accBankRowBits - 1, 0) - - def full_sp_addr(dummy: Int = 0) = data(spAddrBits - 1, 0) - def full_acc_addr(dummy: Int = 0) = data(accAddrBits - 1, 0) - - def is_same_address(other: LocalAddr): Bool = is_acc_addr === other.is_acc_addr && data === other.data - def is_same_address(other: UInt): Bool = is_same_address(other.asTypeOf(this)) - def is_garbage(dummy: Int = 0) = is_acc_addr && accumulate && read_full_acc_row && data.andR() && - (if (garbage_bit.getWidth > 0) garbage_bit.asBool() else true.B) - - def +(other: UInt) = { - require(isPow2(sp_bank_entries)) // TODO remove this requirement - require(isPow2(acc_bank_entries)) // TODO remove this requirement - - val result = WireInit(this) - result.data := data + other - result - } - - def <=(other: LocalAddr) = - is_acc_addr === other.is_acc_addr && - Mux(is_acc_addr, full_acc_addr() <= other.full_acc_addr(), full_sp_addr() <= other.full_sp_addr()) - - def >(other: LocalAddr) = - is_acc_addr === other.is_acc_addr && - Mux(is_acc_addr, full_acc_addr() > other.full_acc_addr(), full_sp_addr() > other.full_sp_addr()) - - def add_with_overflow(other: UInt): Tuple2[LocalAddr, Bool] = { - require(isPow2(sp_bank_entries)) // TODO remove this requirement - require(isPow2(acc_bank_entries)) // TODO remove this requirement - - val sum = data +& other - - val result = WireInit(this) - result.data := sum(data.getWidth-1, 0) - - (result, sum(data.getWidth)) - } - - def make_this_garbage(dummy: Int = 0): Unit = { - is_acc_addr := true.B - accumulate := true.B - read_full_acc_row := true.B - garbage_bit := 1.U - data := ~(0.U(maxAddrBits.W)) - } - - override def cloneType: LocalAddr.this.type = new LocalAddr(sp_banks, sp_bank_entries, acc_banks, acc_bank_entries).asInstanceOf[this.type] -} - class Gemmini[T <: Data : Arithmetic, U <: Data, V <: Data](val config: GemminiArrayConfig[T, U, V]) (implicit p: Parameters) extends LazyRoCC ( @@ -184,23 +110,21 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] val raw_cmd = Queue(io.cmd) // TODO replace 4,12,2 with parameters based on ROB size - val loop_conv_unroller_busy = false.B - /*val (unrolled_cmd_after_conv, loop_conv_unroller_busy) = LoopConv(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + val (conv_cmd, loop_conv_unroller_busy) = LoopConv(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) - unrolled_cmd_after_conv.ready := false.B*/ // val (compressed_cmd, compressor_busy) = InstCompressor(unrolled_cmd) // compressed_cmd.ready := false.B // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - val max_lds = rob_entries * 1 / 4 val max_exs = rob_entries * 3 / 4 val max_sts = rob_entries * 1 / 8 - val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(conv_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, max_lds, max_exs, max_sts, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) + val unrolled_cmd = Queue(loop_cmd) unrolled_cmd.ready := false.B @@ -375,12 +299,12 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] rob_completed_arb.io.out.ready := true.B // Wire up global RoCC signals - io.busy := raw_cmd.valid || loop_conv_unroller_busy || loop_matmul_unroller_busy || rob.io.busy || spad.module.io.busy || unrolled_cmd.valid || loop_cmd.valid + io.busy := raw_cmd.valid || loop_conv_unroller_busy || loop_matmul_unroller_busy || rob.io.busy || spad.module.io.busy || unrolled_cmd.valid || loop_cmd.valid || conv_cmd.valid io.interrupt := tlb.io.exp.interrupt rob.io.solitary_preload := ex_controller.io.solitary_preload - assert(!io.interrupt, "Interrupt handlers have not been written yet") + // assert(!io.interrupt, "Interrupt handlers have not been written yet") // Cycle counters val ld_cycles = RegInit(0.U(34.W)) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index c12b28ac6..7af1751da 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -287,10 +287,11 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } } -class StreamWriteRequest(val dataWidth: Int)(implicit p: Parameters) extends CoreBundle { +class StreamWriteRequest(val dataWidth: Int, val maxBytes: Int)(implicit p: Parameters) extends CoreBundle { val vaddr = UInt(coreMaxAddrBits.W) val data = UInt(dataWidth.W) - val len = UInt(log2Up(dataWidth/8+1).W) // The number of bytes to write + val len = UInt(log2Up((dataWidth/8 max maxBytes)+1).W) // The number of bytes to write + val block = UInt(8.W) // TODO magic number val status = new MStatus // Pooling variables @@ -312,11 +313,13 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val beatBytes = beatBits / 8 val lgBeatBytes = log2Ceil(beatBytes) val maxBeatsPerReq = maxBytes / beatBytes + val inputTypeRowBytes = block_cols * inputType.getWidth / 8 + val maxBlocks = maxBytes / inputTypeRowBytes require(beatBytes > 0) val io = IO(new Bundle { - val req = Flipped(Decoupled(new StreamWriteRequest(dataWidth))) + val req = Flipped(Decoupled(new StreamWriteRequest(dataWidth, maxBytes))) val tlb = new FrontendTLBIO val busy = Output(Bool()) val flush = Input(Bool()) @@ -325,9 +328,14 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val (s_idle :: s_writing_new_block :: s_writing_beats :: Nil) = Enum(3) val state = RegInit(s_idle) - val req = Reg(new StreamWriteRequest(dataWidth)) + val req = Reg(new StreamWriteRequest(dataWidth, maxBytes)) - val bytesSent = Reg(UInt(log2Ceil(dataBytes+1).W)) // TODO this only needs to count up to (dataBytes/aligned_to), right? + // TODO use the same register to hold data_blocks and data_single_block, so that this Mux here is not necessary + val data_blocks = Reg(Vec(maxBlocks, UInt((inputTypeRowBytes * 8).W))) + val data_single_block = Reg(UInt(dataWidth.W)) // For data that's just one-block-wide + val data = Mux(req.block === 0.U, data_single_block, data_blocks.asUInt()) + + val bytesSent = Reg(UInt(log2Ceil((dataBytes max maxBytes)+1).W)) // TODO this only needs to count up to (dataBytes/aligned_to), right? val bytesLeft = req.len - bytesSent val xactBusy = RegInit(0.U(nXacts.W)) @@ -427,14 +435,14 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: fromSource = RegEnableThru(xactId, state === s_writing_new_block), toAddress = 0.U, lgSize = lg_write_size, - data = (req.data >> (bytesSent * 8.U)).asUInt() + data = (data >> (bytesSent * 8.U)).asUInt() )._2 val putPartial = edge.Put( fromSource = RegEnableThru(xactId, state === s_writing_new_block), toAddress = 0.U, lgSize = lg_write_size, - data = ((req.data >> (bytesSent * 8.U)) << (write_shift * 8.U)).asUInt(), + data = ((data >> (bytesSent * 8.U)) << (write_shift * 8.U)).asUInt(), mask = write_mask.asUInt() )._2 @@ -464,7 +472,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val tlb_q = Module(new Queue(new TLBundleAWithInfo, 1, pipe=true)) tlb_q.io.enq <> tlb_arb.io.out - io.tlb.req.valid := tlb_q.io.deq.valid + io.tlb.req.valid := tlb_q.io.deq.fire() io.tlb.req.bits.tlb_req.vaddr := tlb_q.io.deq.bits.vaddr io.tlb.req.bits.tlb_req.passthrough := false.B io.tlb.req.bits.tlb_req.size := 0.U // send_size @@ -478,15 +486,15 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: shadow_retry_a.io.enq.valid := tlb_q.io.deq.valid shadow_retry_a.io.enq.bits := tlb_q.io.deq.bits } - translate_q.io.deq.ready := true.B + translate_q.io.deq.ready := tl.a.ready || io.tlb.resp.miss - retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready) + retry_a.valid := translate_q.io.deq.valid && io.tlb.resp.miss retry_a.bits := translate_q.io.deq.bits - assert(retry_a.ready) + assert(!(retry_a.valid && !retry_a.ready)) tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss tl.a.bits := translate_q.io.deq.bits.tl_a - tl.a.bits.address := io.tlb.resp.paddr + tl.a.bits.address := RegEnableThru(io.tlb.resp.paddr, RegNext(io.tlb.req.fire())) tl.d.ready := xactBusy.orR() @@ -531,17 +539,23 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val pooled = { val cols = dataWidth / inputType.getWidth val v1 = io.req.bits.data.asTypeOf(Vec(cols, inputType)) - val v2 = req.data.asTypeOf(Vec(cols, inputType)) + val v2 = data_single_block.asTypeOf(Vec(cols, inputType)) val m = v1.zip(v2) VecInit(m.zipWithIndex.map{case ((x, y), i) => if (i < block_cols) maxOf(x, y) else y}).asUInt() } req := io.req.bits - req.data := Mux(io.req.bits.pool_en, pooled, io.req.bits.data) + req.len := io.req.bits.block * inputTypeRowBytes.U + io.req.bits.len + + data_single_block := Mux(io.req.bits.pool_en, pooled, io.req.bits.data) + data_blocks(io.req.bits.block) := io.req.bits.data bytesSent := 0.U state := Mux(io.req.bits.store_en, s_writing_new_block, s_idle) + + assert(io.req.bits.len <= (block_cols * inputType.getWidth / 8).U || io.req.bits.block === 0.U, "DMA can't write multiple blocks to main memory when writing full accumulator output") + assert(!io.req.bits.pool_en || io.req.bits.block === 0.U, "Can't pool with block-mvout") } } } diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index 850f0cdf4..363f8af97 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -58,6 +58,11 @@ object DSEBaseConfig { use_dedicated_tl_port = false, pe_latency = 0, + ex_read_from_spad = true, + ex_read_from_acc = true, + ex_write_to_spad = true, + ex_write_to_acc = true, + tlb_size = 4, use_tlb_register_filter = true, max_in_flight_reqs = 16, diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index 4f7994ddc..1fac6d488 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -131,7 +131,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In //val row_turn_counter = RegInit(row_turn) im2col_en := Mux(weight_stride === 0.U, false.B, true.B) - // SRAM addresses of matmul operands val a_address_rs1 = rs1s(a_address_place).asTypeOf(local_addr_t) val b_address_rs2 = rs2s(b_address_place).asTypeOf(local_addr_t) @@ -211,6 +210,8 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In !is_garbage && (mul_raw_haz || pre_raw_haz) }.reduce(_ || _) + val raw_hazards_are_impossible = !ex_read_from_acc && !ex_write_to_spad // Special case where RAW hazards are impossible + val matmul_in_progress = mesh.io.tags_in_progress.map(_.rob_id.valid).reduce(_ || _) io.busy := cmd.valid(0) || matmul_in_progress @@ -242,9 +243,9 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val dataBBankAcc = b_address.acc_bank() val dataDBankAcc = d_address.acc_bank() - val a_read_from_acc = a_address_rs1.is_acc_addr - val b_read_from_acc = b_address_rs2.is_acc_addr - val d_read_from_acc = d_address_rs1.is_acc_addr + val a_read_from_acc = ex_read_from_acc.B && a_address_rs1.is_acc_addr + val b_read_from_acc = ex_read_from_acc.B && b_address_rs2.is_acc_addr + val d_read_from_acc = ex_read_from_acc.B && d_address_rs1.is_acc_addr val start_inputting_a = WireInit(false.B) val start_inputting_b = WireInit(false.B) @@ -322,7 +323,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In !must_wait_for.reduce(_ || _) } - val a_fire = a_valid && a_ready val b_fire = b_valid && b_ready val d_fire = d_valid && d_ready @@ -352,7 +352,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In d_fire_started := true.B } - when(performing_mul_pre && !cntl_ready && !mul_pre_counter_lock){ mul_pre_counter_count := d_fire_counter //store 2 }.elsewhen(!performing_mul_pre){ @@ -402,19 +401,26 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } } - io.srams.read(i).req.valid := read_a || read_b || read_d - io.srams.read(i).req.bits.fromDMA := false.B - io.srams.read(i).req.bits.addr := MuxCase(a_address_rs1.sp_row() + a_fire_counter, - Seq(read_b -> (b_address_rs2.sp_row() + b_fire_counter), - read_d -> (d_address_rs1.sp_row() + block_size.U - 1.U - d_fire_counter_mulpre))) - - when(im2col_en === false.B){ - io.srams.read(i).req.bits.addr := MuxCase(a_address.sp_row(), - Seq(read_b -> b_address.sp_row(), - read_d -> d_address.sp_row())) + if (ex_read_from_spad) { + io.srams.read(i).req.valid := (read_a || read_b || read_d) && cntl_ready + io.srams.read(i).req.bits.fromDMA := false.B + io.srams.read(i).req.bits.addr := MuxCase(a_address_rs1.sp_row() + a_fire_counter, + Seq(read_b -> (b_address_rs2.sp_row() + b_fire_counter), + read_d -> (d_address_rs1.sp_row() + block_size.U - 1.U - d_fire_counter_mulpre))) + + // TODO this just overrides the previous line. Should we erase the previous line? + when(im2col_en === false.B) { + io.srams.read(i).req.bits.addr := MuxCase(a_address.sp_row(), + Seq(read_b -> b_address.sp_row(), + read_d -> d_address.sp_row())) + } + } else { + io.srams.read(i).req.valid := false.B + io.srams.read(i).req.bits.fromDMA := false.B + io.srams.read(i).req.bits.addr := DontCare } - io.srams.read(i).resp.ready := true.B + io.srams.read(i).resp.ready := false.B } // Accumulator read @@ -429,40 +435,34 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } } - /* - io.acc.read(i).req.valid := read_a_from_acc || read_b_from_acc || read_d_from_acc - io.acc.read(i).req.bits.scale := acc_scale - io.acc.read(i).req.bits.full := false.B - io.acc.read(i).req.bits.relu6_shift := relu6_shift - io.acc.read(i).req.bits.act := activation - io.acc.read(i).req.bits.fromDMA := false.B - io.acc.read(i).req.bits.addr := MuxCase(a_address_rs1.acc_row() + a_fire_counter, - Seq(read_b_from_acc -> (b_address_rs2.acc_row() + b_fire_counter), - read_d_from_acc -> (d_address_rs1.acc_row() + block_size.U - 1.U - d_fire_counter))) - - when(im2col_en === false.B){ - io.acc.read(i).req.bits.addr := MuxCase(a_address.acc_row(), - Seq(read_b_from_acc -> b_address.acc_row(), - read_d_from_acc -> d_address.acc_row())) - } - */ - - // TODO Remove the ability to read into Mesh from AccumulatorMem completely - io.acc.read(i).req.valid := false.B - io.acc.read(i).req.bits.scale := acc_scale - io.acc.read(i).req.bits.full := false.B - io.acc.read(i).req.bits.relu6_shift := relu6_shift - io.acc.read(i).req.bits.act := activation - io.acc.read(i).req.bits.fromDMA := false.B - io.acc.read(i).req.bits.addr := DontCare - - when(im2col_en === false.B){ - io.acc.read(i).req.bits.addr := MuxCase(a_address.acc_row(), - Seq(read_b_from_acc -> b_address.acc_row(), - read_d_from_acc -> d_address.acc_row())) + if (ex_read_from_acc) { + io.acc.read(i).req.valid := read_a_from_acc || read_b_from_acc || read_d_from_acc + io.acc.read(i).req.bits.scale := acc_scale + io.acc.read(i).req.bits.full := false.B + io.acc.read(i).req.bits.relu6_shift := relu6_shift + io.acc.read(i).req.bits.act := activation + io.acc.read(i).req.bits.fromDMA := false.B + io.acc.read(i).req.bits.addr := MuxCase(a_address_rs1.acc_row() + a_fire_counter, + Seq(read_b_from_acc -> (b_address_rs2.acc_row() + b_fire_counter), + read_d_from_acc -> (d_address_rs1.acc_row() + block_size.U - 1.U - d_fire_counter))) + + // TODO this just overrides the previous line. Should we erase the previous line? + when(im2col_en === false.B){ + io.acc.read(i).req.bits.addr := MuxCase(a_address.acc_row(), + Seq(read_b_from_acc -> b_address.acc_row(), + read_d_from_acc -> d_address.acc_row())) + } + } else { + io.acc.read(i).req.valid := false.B + io.acc.read(i).req.bits.scale := acc_scale + io.acc.read(i).req.bits.full := false.B + io.acc.read(i).req.bits.relu6_shift := relu6_shift + io.acc.read(i).req.bits.act := activation + io.acc.read(i).req.bits.fromDMA := false.B + io.acc.read(i).req.bits.addr := DontCare } - io.acc.read(i).resp.ready := true.B + io.acc.read(i).resp.ready := false.B } // Im2Col reads @@ -494,7 +494,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In io.im2col.resp.ready := mesh.io.a.ready } - // FSM logic switch (control_state) { is(waiting_for_cmd) { @@ -540,7 +539,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } // Preload - .elsewhen(DoPreloads(0) && cmd.valid(1) && !raw_hazard_pre) { + .elsewhen(DoPreloads(0) && cmd.valid(1) && (raw_hazards_are_impossible.B || !raw_hazard_pre)) { perform_single_preload := true.B performing_single_preload := true.B @@ -555,7 +554,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } // Overlap compute and preload - .elsewhen(DoComputes(0) && cmd.valid(1) && DoPreloads(1) && cmd.valid(2) && !raw_hazard_mulpre) { + .elsewhen(DoComputes(0) && cmd.valid(1) && DoPreloads(1) && (raw_hazards_are_impossible.B || (cmd.valid(2) && !raw_hazard_mulpre))) { perform_mul_pre := true.B performing_mul_pre := true.B @@ -750,11 +749,11 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.enq.bits.im2colling := im2col_wire && im2col_en //im2col_wire val readData = VecInit(io.srams.read.map(_.resp.bits.data)) - val accReadData = readData // VecInit(io.acc.read.map(_.resp.bits.data.asUInt())) // TODO remove ability to read from AccumulatorMem + val accReadData = if (ex_read_from_acc) VecInit(io.acc.read.map(_.resp.bits.data.asUInt())) else readData val im2ColData = io.im2col.resp.bits.a_im2col.asUInt() - val readValid = VecInit(io.srams.read.map(bank => bank.resp.valid && !bank.resp.bits.fromDMA)) - val accReadValid = false.B // VecInit(io.acc.read.map(bank => bank.resp.valid && !bank.resp.bits.fromDMA)) // TODO remove ability to read from AccumulatorMem + val readValid = VecInit(io.srams.read.map(bank => ex_read_from_spad.B && bank.resp.valid && !bank.resp.bits.fromDMA)) + val accReadValid = VecInit(io.acc.read.map(bank => ex_read_from_acc.B && bank.resp.valid && !bank.resp.bits.fromDMA)) val im2ColValid = io.im2col.resp.valid mesh_cntl_signals_q.io.deq.ready := (!cntl.a_fire || mesh.io.a.fire() || !mesh.io.a.ready) && @@ -785,6 +784,37 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val dataB = VecInit(dataB_unpadded.asTypeOf(Vec(block_size, inputType)).zipWithIndex.map { case (d, i) => Mux(i.U < cntl.b_unpadded_cols, d, inputType.zero)}) val dataD = VecInit(dataD_unpadded.asTypeOf(Vec(block_size, inputType)).zipWithIndex.map { case (d, i) => Mux(i.U < cntl.d_unpadded_cols, d, inputType.zero)}) + // Pop responses off the scratchpad io ports + when (mesh_cntl_signals_q.io.deq.fire()) { + when (cntl.a_fire && mesh.io.a.fire() && !cntl.a_garbage && cntl.a_unpadded_cols > 0.U && !cntl.im2colling) { + when (cntl.a_read_from_acc) { + io.acc.read(cntl.a_bank_acc).resp.ready := !io.acc.read(cntl.a_bank_acc).resp.bits.fromDMA + }.otherwise { + io.srams.read(cntl.a_bank).resp.ready := !io.srams.read(cntl.a_bank).resp.bits.fromDMA + } + } + + when (cntl.b_fire && mesh.io.b.fire() && !cntl.b_garbage && !cntl.accumulate_zeros && cntl.b_unpadded_cols > 0.U) { + when (cntl.b_read_from_acc) { + io.acc.read(cntl.b_bank_acc).resp.ready := !io.acc.read(cntl.b_bank_acc).resp.bits.fromDMA + }.otherwise { + io.srams.read(cntl.b_bank).resp.ready := !io.srams.read(cntl.b_bank).resp.bits.fromDMA + } + } + + when (cntl.d_fire && mesh.io.d.fire() && !cntl.d_garbage && !cntl.preload_zeros && cntl.d_unpadded_cols > 0.U) { + when (cntl.d_read_from_acc) { + io.acc.read(cntl.d_bank_acc).resp.ready := !io.acc.read(cntl.d_bank_acc).resp.bits.fromDMA + }.otherwise { + io.srams.read(cntl.d_bank).resp.ready := !io.srams.read(cntl.d_bank).resp.bits.fromDMA + } + } + } + + for (acc_r <- io.acc.read) { + acc_r.resp.ready := true.B + } + when (cntl_valid) { // Default inputs mesh.io.a.valid := cntl.a_fire && dataA_valid @@ -803,14 +833,11 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } when (cntl_valid && cntl.perform_single_preload) { - // mesh.io.a.bits := Mux(cntl.dataflow === Dataflow.WS.id.U, 0.U, dataA.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.a.bits := Mux(a_should_be_fed_into_transposer, dataA.asUInt, 0.U).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) - // mesh.io.b.bits := 0.U.asTypeOf(Vec(meshColumns, Vec(tileColumns, inputType))) mesh.io.b.bits := Mux(b_should_be_fed_into_transposer, dataB.asUInt, 0.U).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) } when (cntl_valid && cntl.perform_single_mul) { - // mesh.io.a.bits := Mux(cntl.dataflow === Dataflow.OS.id.U, 0.U, dataA.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.a.bits := Mux(a_should_be_fed_into_transposer, 0.U, dataA.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.b.bits := Mux(b_should_be_fed_into_transposer, 0.U, dataB.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.tag_in.bits.addr.make_this_garbage() @@ -846,20 +873,34 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In e_act }))) - io.srams.write(i).en := start_array_outputting && w_bank === i.U && !write_to_acc && !is_garbage_addr && write_this_row - io.srams.write(i).addr := w_row - io.srams.write(i).data := activated_wdata.asUInt() - // io.srams.write(i).mask := VecInit(Seq.fill(io.srams.write(0).mask.length)(true.B)) - io.srams.write(i).mask := w_mask.flatMap(b => Seq.fill(inputType.getWidth / (aligned_to * 8))(b)) + if (ex_write_to_spad) { + io.srams.write(i).en := start_array_outputting && w_bank === i.U && !write_to_acc && !is_garbage_addr && write_this_row + io.srams.write(i).addr := w_row + io.srams.write(i).data := activated_wdata.asUInt() + io.srams.write(i).mask := w_mask.flatMap(b => Seq.fill(inputType.getWidth / (aligned_to * 8))(b)) + } else { + io.srams.write(i).en := false.B + io.srams.write(i).addr := DontCare + io.srams.write(i).data := DontCare + io.srams.write(i).mask := DontCare + } } // Write to accumulator for (i <- 0 until acc_banks) { - io.acc.write(i).valid := start_array_outputting && w_bank === i.U && write_to_acc && !is_garbage_addr && write_this_row - io.acc.write(i).bits.addr := w_row - io.acc.write(i).bits.data := VecInit(mesh.io.out.bits.map(v => VecInit(v.map(e => e.withWidthOf(accType))))) - io.acc.write(i).bits.acc := w_address.accumulate - io.acc.write(i).bits.mask := w_mask.flatMap(b => Seq.fill(accType.getWidth / (aligned_to * 8))(b)) + if (ex_write_to_acc) { + io.acc.write(i).valid := start_array_outputting && w_bank === i.U && write_to_acc && !is_garbage_addr && write_this_row + io.acc.write(i).bits.addr := w_row + io.acc.write(i).bits.data := VecInit(mesh.io.out.bits.map(v => VecInit(v.map(e => e.withWidthOf(accType))))) + io.acc.write(i).bits.acc := w_address.accumulate + io.acc.write(i).bits.mask := w_mask.flatMap(b => Seq.fill(accType.getWidth / (aligned_to * 8))(b)) + } else { + io.acc.write(i).valid := false.B + io.acc.write(i).bits.addr := DontCare + io.acc.write(i).bits.data := DontCare + io.acc.write(i).bits.acc := DontCare + io.acc.write(i).bits.mask := DontCare + } assert(!(io.acc.write(i).valid && !io.acc.write(i).ready), "Execute controller write to AccumulatorMem was skipped") } diff --git a/src/main/scala/gemmini/FrontendTLB.scala b/src/main/scala/gemmini/FrontendTLB.scala index 0980d811d..734168167 100644 --- a/src/main/scala/gemmini/FrontendTLB.scala +++ b/src/main/scala/gemmini/FrontendTLB.scala @@ -96,7 +96,6 @@ class FrontendTLB(nClients: Int, entries: Int, maxSize: Int) val l0_tlb_hit = last_translated_valid && ((client.req.bits.tlb_req.vaddr >> pgIdxBits) === (last_translated_vpn >> pgIdxBits)) val l0_tlb_paddr = Cat(last_translated_ppn >> pgIdxBits, client.req.bits.tlb_req.vaddr(pgIdxBits-1,0)) - when (req.fire() && !tlb.io.resp.miss) { last_translated_valid := true.B last_translated_vpn := req.bits.tlb_req.vaddr diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index ada4efadb..f8c4f4bea 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -53,6 +53,11 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( use_tlb_register_filter: Boolean, max_in_flight_reqs: Int, + ex_read_from_spad: Boolean, + ex_read_from_acc: Boolean, + ex_write_to_spad: Boolean, + ex_write_to_acc: Boolean, + headerFileName: String = "gemmini_params.h" ) { val sp_width = meshColumns * tileColumns * inputType.getWidth @@ -86,7 +91,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( val mvin_cols_bits = log2Up(((dma_maxbytes / (inputType.getWidth / 8)) max (meshColumns * tileColumns)) + 1) val mvin_rows_bits = log2Up(meshRows * tileRows + 1) - val mvout_cols_bits = log2Up(meshColumns * tileColumns + 1) + val mvout_cols_bits = log2Up(((dma_maxbytes / (inputType.getWidth / 8)) max (meshColumns * tileColumns)) + 1) val mvout_rows_bits = log2Up(meshRows * tileRows + 1) val load_states = 3 @@ -193,7 +198,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( (dt.expWidth, dt.sigWidth) match { case (8, 24) => (scala.Float.MinValue.toString, scala.Float.MaxValue.toString) case (11, 53) => (scala.Double.MinValue.toString, scala.Double.MaxValue.toString) - case _ => throw new IllegalArgumentException(s"Only single- and double-precision IEEE754 floating point types are currently supported") + case _ => (((Range(-1,-(dt.sigWidth),-1).map(-Math.pow(2, _)).foldLeft(-1.0)(_ + _)) * Math.pow(2, Math.pow(2, dt.expWidth - 1) - 1)).toString, ((Range(-1,-(dt.sigWidth),-1).map(Math.pow(2, _)).foldLeft(1.0)(_ + _)) * Math.pow(2, Math.pow(2, dt.expWidth - 1) - 1)).toString) } case _ => throw new IllegalArgumentException(s"Data type $dataType is unknown") } @@ -207,7 +212,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( (dt.expWidth, dt.sigWidth) match { case (8, 24) => "float" case (11, 53) => "double" - case _ => throw new IllegalArgumentException(s"Only single- and double-precision IEEE754 floating point types are currently supported") + case _ => s"uint" + (Math.pow(2, Math.ceil(Math.log(dt.expWidth + dt.sigWidth)/Math.log(2.0)))).toInt.toString + s"_t" } case _ => throw new IllegalArgumentException(s"Data type $dataType is unknown") } @@ -267,8 +272,15 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( // Datatype of the systolic array val limits = limitsOfDataType(inputType) header ++= s"typedef ${c_type(inputType)} elem_t;\n" - header ++= s"static const elem_t elem_t_max = ${limits._2};\n" - header ++= s"static const elem_t elem_t_min = ${limits._1};\n" + if (inputType.isInstanceOf[Float] && !((inputType.asInstanceOf[Float].expWidth, inputType.asInstanceOf[Float].sigWidth) == (8, 24) || (inputType.asInstanceOf[Float].expWidth, inputType.asInstanceOf[Float].sigWidth) == (11, 53))) + { + header ++= "#define ELEM_T_IS_LOWPREC_FLOAT\n" + header ++= s"static const float elem_t_max = ${limits._2};\n" + header ++= s"static const float elem_t_min = ${limits._1};\n" + } else { + header ++= s"static const elem_t elem_t_max = ${limits._2};\n" + header ++= s"static const elem_t elem_t_min = ${limits._1};\n" + } header ++= s"typedef ${c_type(accType)} acc_t;\n" header ++= s"typedef ${full_c_type(inputType)} full_t;\n\n" @@ -346,6 +358,13 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( |""".stripMargin header ++= "\n" + header ++= """// Rounding right shift equation: https://riscv.github.io/documents/riscv-v-spec/#_vector_fixed_point_rounding_mode_register_vxrm +#define ROUNDING_RIGHT_SHIFT_BITS(x, shift) \ +((shift) > 0 ? (((x) >> (shift)) + \ + (((shift) == 0 ? 0 : (((x) >> ((shift)-1)) & 1)) & \ + ((((shift) <= 1 ? 0 : ((x) & ((1 << ((shift)-1)) - 1))) != 0) | (((x) >> (shift)) & 1)))) : ((x) << (-(shift))))""" + header ++= "\n\n" + header ++= """#define ACC_SCALE(x, scale) \ """ header ++= s" ${acc_scale_args.c_str}" diff --git a/src/main/scala/gemmini/Im2Col.scala b/src/main/scala/gemmini/Im2Col.scala index 4970334da..f264ad320 100644 --- a/src/main/scala/gemmini/Im2Col.scala +++ b/src/main/scala/gemmini/Im2Col.scala @@ -446,5 +446,6 @@ class Im2Col[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, U, V io.resp.valid := false.B io.req.ready := true.B io.sram_reads.foreach(_.req.valid := false.B) + io.sram_reads.foreach(_.resp.ready := false.B) } } diff --git a/src/main/scala/gemmini/LocalAddr.scala b/src/main/scala/gemmini/LocalAddr.scala new file mode 100644 index 000000000..6520b7f9c --- /dev/null +++ b/src/main/scala/gemmini/LocalAddr.scala @@ -0,0 +1,83 @@ +package gemmini + +import chisel3._ +import chisel3.util._ + +class LocalAddr(sp_banks: Int, sp_bank_entries: Int, acc_banks: Int, acc_bank_entries: Int) extends Bundle { + private val localAddrBits = 32 // TODO magic number + + private val spAddrBits = log2Ceil(sp_banks * sp_bank_entries) + private val accAddrBits = log2Ceil(acc_banks * acc_bank_entries) + private val maxAddrBits = spAddrBits max accAddrBits + + private val spBankBits = log2Up(sp_banks) + private val spBankRowBits = log2Up(sp_bank_entries) + + private val accBankBits = log2Up(acc_banks) + private val accBankRowBits = log2Up(acc_bank_entries) + + val is_acc_addr = Bool() + val accumulate = Bool() + val read_full_acc_row = Bool() + val garbage = UInt(((localAddrBits - maxAddrBits - 4) max 0).W) + val garbage_bit = if (localAddrBits - maxAddrBits >= 4) UInt(1.W) else UInt(0.W) + val data = UInt(maxAddrBits.W) + + def sp_bank(dummy: Int = 0) = if (spAddrBits == spBankRowBits) 0.U else data(spAddrBits - 1, spBankRowBits) + def sp_row(dummy: Int = 0) = data(spBankRowBits - 1, 0) + def acc_bank(dummy: Int = 0) = if (accAddrBits == accBankRowBits) 0.U else data(accAddrBits - 1, accBankRowBits) + def acc_row(dummy: Int = 0) = data(accBankRowBits - 1, 0) + + def full_sp_addr(dummy: Int = 0) = data(spAddrBits - 1, 0) + def full_acc_addr(dummy: Int = 0) = data(accAddrBits - 1, 0) + + def is_same_address(other: LocalAddr): Bool = is_acc_addr === other.is_acc_addr && data === other.data + def is_same_address(other: UInt): Bool = is_same_address(other.asTypeOf(this)) + def is_garbage(dummy: Int = 0) = is_acc_addr && accumulate && read_full_acc_row && data.andR() && + (if (garbage_bit.getWidth > 0) garbage_bit.asBool() else true.B) + + def +(other: UInt) = { + require(isPow2(sp_bank_entries)) // TODO remove this requirement + require(isPow2(acc_bank_entries)) // TODO remove this requirement + + val result = WireInit(this) + result.data := data + other + result + } + + def <=(other: LocalAddr) = + is_acc_addr === other.is_acc_addr && + Mux(is_acc_addr, full_acc_addr() <= other.full_acc_addr(), full_sp_addr() <= other.full_sp_addr()) + + def <(other: LocalAddr) = + is_acc_addr === other.is_acc_addr && + Mux(is_acc_addr, full_acc_addr() < other.full_acc_addr(), full_sp_addr() < other.full_sp_addr()) + + def >(other: LocalAddr) = + is_acc_addr === other.is_acc_addr && + Mux(is_acc_addr, full_acc_addr() > other.full_acc_addr(), full_sp_addr() > other.full_sp_addr()) + + def add_with_overflow(other: UInt): Tuple2[LocalAddr, Bool] = { + require(isPow2(sp_bank_entries)) // TODO remove this requirement + require(isPow2(acc_bank_entries)) // TODO remove this requirement + + val sum = data +& other + + val overflow = Mux(is_acc_addr, sum(accAddrBits), sum(spAddrBits)) + + val result = WireInit(this) + result.data := sum(maxAddrBits - 1, 0) + + (result, overflow) + } + + def make_this_garbage(dummy: Int = 0): Unit = { + is_acc_addr := true.B + accumulate := true.B + read_full_acc_row := true.B + garbage_bit := 1.U + data := ~(0.U(maxAddrBits.W)) + } + + override def cloneType: LocalAddr.this.type = new LocalAddr(sp_banks, sp_bank_entries, acc_banks, acc_bank_entries).asInstanceOf[this.type] +} diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index e9cb40b06..83f34fcf9 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -233,7 +233,7 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw (icol >= icols_unpadded.zext()) -> Mux(icols_unpadded.zext() +& rpad.zext() -& icol > block_size.S, block_size.S, icols_unpadded.zext() +& rpad.zext() -& icol) ) ) - val K = Mux(ochs.zext() -& ich > max_ichs_per_mvin, max_ichs_per_mvin, ochs.zext() -& ich) + val K = Mux(ichs.zext() -& ich > max_ichs_per_mvin, max_ichs_per_mvin, ichs.zext() -& ich) // Commands val config_cmd = Wire(new RoCCCommand) @@ -735,7 +735,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I // Create states val concurrent_loops = 2 val loops = Reg(Vec(concurrent_loops, new LoopConvState(block_size, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, coreMaxAddrBits, max_addr, max_acc_addr))) - // val head_loop_id = Reg(UInt(log2Up(concurrent_loops).W)) val head_loop_id = RegInit(0.U(log2Up(concurrent_loops).W)) val tail_loop_id = (~head_loop_id).asUInt() // This is the loop that we always try to configure if available val head_loop = loops(head_loop_id) diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 5932a51a0..74db8914d 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -448,7 +448,7 @@ class LoopMatmulStCReq(val block_size: Int, val coreMaxAddrBits: Int, val iterat val loop_id = UInt(log2Up(concurrent_loops).W) } -class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, concurrent_loops: Int) +class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, max_block_len: Int, concurrent_loops: Int) (implicit p: Parameters) extends Module { val io = IO(new Bundle { val req = Flipped(Decoupled(new LoopMatmulStCReq(block_size, coreMaxAddrBits, iterator_bitwidth, max_acc_addr, concurrent_loops))) @@ -476,6 +476,8 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val req = Reg(new LoopMatmulStCReq(block_size, coreMaxAddrBits, iterator_bitwidth, max_acc_addr, concurrent_loops)) + val max_blocks = Mux(req.full_c, 1.U, Mux(req.max_j <= max_block_len.U, req.max_j, max_block_len.U)) + val j = Reg(UInt(iterator_bitwidth.W)) val i = Reg(UInt(iterator_bitwidth.W)) @@ -484,7 +486,8 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val dram_addr = Mux(req.full_c, req.dram_addr + (i * req.dram_stride + j) * block_size.U * (acc_w/8).U, req.dram_addr + (i * req.dram_stride + j) * block_size.U * (input_w/8).U) val sp_addr = acc_addr_start + (i * req.max_j + j) * block_size.U - val cols = block_size.U - Mux(j + 1.U >= req.max_j, req.pad_j, 0.U) + val blocks = Mux(j + max_blocks <= req.max_j, max_blocks, req.max_j-j) + val cols = (blocks * block_size.U) - Mux(j + blocks >= req.max_j, req.pad_j, 0.U) val rows = block_size.U - Mux(i === req.max_i-1.U, req.pad_i, 0.U) val mvout_cmd = Wire(new RoCCCommand) @@ -499,7 +502,11 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In io.idle := state === idle // The order here is k, j, i - val ex_ahead = io.ex_completed || (io.ex_k === req.max_k - 1.U && (io.ex_j > j || (io.ex_j === j && io.ex_i > i))) + // val ex_ahead = io.ex_completed || (io.ex_k === req.max_k - 1.U && (io.ex_j > j || (io.ex_j === j && io.ex_i > i))) + val ex_ahead = io.ex_completed || + (io.ex_k === req.max_k - 1.U && + (io.ex_j >= j + blocks || + ((io.ex_j === j + blocks - 1.U) && io.ex_i > i))) io.cmd.valid := state =/= idle && !io.rob_overloaded && ex_ahead && req.dram_addr =/= 0.U io.cmd.bits := mvout_cmd @@ -511,7 +518,7 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In }.elsewhen (io.cmd.fire()) { // The order here is k, j, i val next_i = floorAdd(i, 1.U, req.max_i) - val next_j = floorAdd(j, 1.U, req.max_j, next_i === 0.U) + val next_j = floorAdd(j, max_blocks, req.max_j, next_i === 0.U) i := next_i j := next_j @@ -630,7 +637,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: val ldB = Module(new LoopMatmulLdB(block_size, coreMaxAddrBits, iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) val ldD = Module(new LoopMatmulLdD(block_size, coreMaxAddrBits, iterator_bitwidth, max_acc_addr, input_w, acc_w, max_block_len, max_block_len_acc, concurrent_loops)) val ex = Module(new LoopMatmulExecute(block_size, coreMaxAddrBits, iterator_bitwidth, max_addr, max_acc_addr, concurrent_loops)) - val stC = Module(new LoopMatmulStC(block_size, coreMaxAddrBits, iterator_bitwidth, max_acc_addr, input_w, acc_w, concurrent_loops)) + val stC = Module(new LoopMatmulStC(block_size, coreMaxAddrBits, iterator_bitwidth, max_acc_addr, input_w, acc_w, max_block_len, concurrent_loops)) // Create command queue val cmd = Queue(io.in) @@ -686,6 +693,9 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: stC.io.ex_j := ex.io.j stC.io.ex_i := ex.io.i + val loops_configured = RegInit(0.U(16.W)) + dontTouch(loops_configured) + // Create config registers when(cmd.valid && is_loop_cmd && !loop_being_configured.configured) { @@ -728,6 +738,8 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: loop_being_configured.b_transpose := cmd.bits.rs2(1) loop_being_configured.configured := true.B + + loops_configured := loops_configured + 1.U } } } diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 040df5940..58037c646 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -56,11 +56,18 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val wraps_around = Bool() def overlaps(other: OpT): Bool = { - (other.start <= start && (start <= other.end || other.wraps_around)) || - (start <= other.start && (other.start <= end || wraps_around)) + ((other.start <= start && (start < other.end || other.wraps_around)) || + (start <= other.start && (other.start < end || wraps_around))) && + !(start.is_garbage() || other.start.is_garbage()) // TODO the "is_garbage" check might not really be necessary } } + val instructions_allocated = RegInit(0.U(32.W)) + when (io.alloc.fire()) { + instructions_allocated := instructions_allocated + 1.U + } + dontTouch(instructions_allocated) + class Entry extends Bundle { val q = q_t.cloneType @@ -78,6 +85,9 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val deps = Vec(rob_entries, Bool()) def ready(dummy: Int = 0): Bool = !deps.reduce(_ || _) + + // Debugging signals + val allocated_at = UInt(instructions_allocated.getWidth.W) } val entries = Reg(Vec(rob_entries, UDValid(new Entry))) @@ -96,7 +106,8 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val last_allocated = Reg(UInt(log2Up(rob_entries).W)) val a_stride = Reg(UInt(16.W)) // TODO magic numbers // TODO we also need to check the transpose to see how many rows we're reading - val block_strides = Reg(Vec(load_states, UInt(block_stride_bits.W))) + val ld_block_strides = Reg(Vec(load_states, UInt(block_stride_bits.W))) + val st_block_stride = block_rows.U val new_entry = Wire(new Entry) new_entry := DontCare @@ -113,7 +124,19 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val wars_op1_probe = WireInit(0.U(rob_entries.W)) val wars_op2_probe = WireInit(0.U(rob_entries.W)) - dontTouch(new_entry) + val raws_op1_probe = WireInit(0.U(rob_entries.W)) + val raws_op2_probe = WireInit(0.U(rob_entries.W)) + + dontTouch(raws_probe) + dontTouch(waws_probe) + dontTouch(wars_probe) + dontTouch(wars_op1_probe) + dontTouch(wars_op2_probe) + dontTouch(raws_op1_probe) + dontTouch(raws_op2_probe) + dontTouch(older_in_same_q_probe) + dontTouch(is_st_and_must_wait_for_prior_ex_config_probe) + dontTouch(is_ex_config_and_must_wait_for_prior_st_probe) when (io.alloc.fire()) { val spAddrBits = 32 @@ -146,9 +169,16 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf new_entry.op2.bits.end := new_entry.op2.bits.start + compute_rows new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(compute_rows)._2 }.otherwise { + val block_stride = st_block_stride + + val mvout_cols = cmd.rs2(32 + mvout_cols_bits - 1, 32) val mvout_rows = cmd.rs2(48 + mvout_rows_bits - 1, 48) - new_entry.op2.bits.end := new_entry.op2.bits.start + mvout_rows - new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(mvout_rows)._2 + + val mvout_mats = mvout_cols / block_cols.U + (mvout_cols % block_cols.U =/= 0.U) + val total_mvout_rows = ((mvout_mats - 1.U) * block_stride) + mvout_rows + + new_entry.op2.bits.end := new_entry.op2.bits.start + total_mvout_rows + new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(total_mvout_rows)._2 } new_entry.dst.valid := funct === PRELOAD_CMD || funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD @@ -160,19 +190,22 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf }.otherwise { val id = MuxCase(0.U, Seq((new_entry.cmd.inst.funct === LOAD2_CMD) -> 1.U, (new_entry.cmd.inst.funct === LOAD3_CMD) -> 2.U)) - val block_stride = block_strides(id) + val block_stride = ld_block_strides(id) val mvin_cols = cmd.rs2(spAddrBits + mvin_cols_bits - 1, spAddrBits) + val mvin_rows = cmd.rs2(spAddrBits + mvin_cols_bits + mvin_rows_bits - 1, spAddrBits + mvin_cols_bits) + val mvin_mats = mvin_cols / block_cols.U + (mvin_cols % block_cols.U =/= 0.U) - val mvin_rows = mvin_mats * block_stride + val total_mvin_rows = ((mvin_mats - 1.U) * block_stride) + mvin_rows - new_entry.dst.bits.end := new_entry.dst.bits.start + mvin_rows - new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(mvin_rows)._2 + new_entry.dst.bits.end := new_entry.dst.bits.start + total_mvin_rows + new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(total_mvin_rows)._2 } val is_load = funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD || (funct === CONFIG_CMD && config_cmd_type === CONFIG_LOAD) val is_store = funct === STORE_CMD || (funct === CONFIG_CMD && config_cmd_type === CONFIG_STORE) val is_ex = funct === PRELOAD_CMD || funct_is_compute || (funct === CONFIG_CMD && (config_cmd_type === CONFIG_EX || config_cmd_type === CONFIG_IM2COL)) + val is_im2col = funct === CONFIG_CMD && config_cmd_type === CONFIG_IM2COL // im2col commands are a subset of ex commands, so they still go in the ex queue new_entry.q := Mux1H(Seq( is_load -> ldq, @@ -190,6 +223,18 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf (new_entry.op2.valid && new_entry.op2.bits.overlaps(e.bits.dst.bits))) } + val raws_op1 = entries.map { e => + // We search for all entries which write to an address which we read from + e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( + (new_entry.op1.valid && new_entry.op1.bits.overlaps(e.bits.dst.bits))) + } + + val raws_op2 = entries.map { e => + // We search for all entries which write to an address which we read from + e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( + (new_entry.op2.valid && new_entry.op2.bits.overlaps(e.bits.dst.bits))) + } + // TODO we should checck whether op1 and op2 are valid here val wars = entries.map { e => // We search for all entries which read from an address that we write to @@ -232,23 +277,18 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf new_entry.deps := (Cat(raws) | Cat(wars) | Cat(waws) | Cat(older_in_same_q) | Cat(is_st_and_must_wait_for_prior_ex_config) | Cat(is_ex_config_and_must_wait_for_prior_st)).asBools().reverse - raws_probe := Cat(raws) - waws_probe := Cat(waws) - wars_probe := Cat(wars) - wars_op1_probe := Cat(wars_op1) - wars_op2_probe := Cat(wars_op2) - older_in_same_q_probe := Cat(older_in_same_q) - is_st_and_must_wait_for_prior_ex_config_probe := Cat(is_st_and_must_wait_for_prior_ex_config) - is_ex_config_and_must_wait_for_prior_st_probe := Cat(is_ex_config_and_must_wait_for_prior_st) - - dontTouch(raws_probe) - dontTouch(waws_probe) - dontTouch(wars_probe) - dontTouch(wars_op1_probe) - dontTouch(wars_op2_probe) - dontTouch(older_in_same_q_probe) - dontTouch(is_st_and_must_wait_for_prior_ex_config_probe) - dontTouch(is_ex_config_and_must_wait_for_prior_st_probe) + raws_probe := Cat(raws.reverse) + waws_probe := Cat(waws.reverse) + wars_probe := Cat(wars.reverse) + wars_op1_probe := Cat(wars_op1.reverse) + wars_op2_probe := Cat(wars_op2.reverse) + raws_op1_probe := Cat(raws_op1.reverse) + raws_op2_probe := Cat(raws_op2.reverse) + older_in_same_q_probe := Cat(older_in_same_q.reverse) + is_st_and_must_wait_for_prior_ex_config_probe := Cat(is_st_and_must_wait_for_prior_ex_config.reverse) + is_ex_config_and_must_wait_for_prior_st_probe := Cat(is_ex_config_and_must_wait_for_prior_st.reverse) + + new_entry.allocated_at := instructions_allocated new_entry.complete_on_issue := new_entry.is_config && new_entry.q =/= exq @@ -257,12 +297,12 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf last_allocated := new_entry_id - when (new_entry.is_config && new_entry.q === exq) { + when (new_entry.is_config && new_entry.q === exq && !is_im2col) { a_stride := new_entry.cmd.rs1(31, 16) // TODO magic numbers // TODO this needs to be kept in sync with ExecuteController.scala }.elsewhen(new_entry.is_config && new_entry.q === ldq) { val id = new_entry.cmd.rs1(4,3) // TODO magic numbers val block_stride = new_entry.cmd.rs1(31, 16) // TODO magic numbers - block_strides(id) := block_stride + ld_block_strides(id) := block_stride } } @@ -314,12 +354,15 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf io.st_utilization := utilization_st_q io.ex_utilization := utilization_ex_q - val packed_deps = VecInit(entries.map(e => Cat(e.bits.deps.reverse))) - dontTouch(packed_deps) - val valids = VecInit(entries.map(_.valid)) val functs = VecInit(entries.map(_.bits.cmd.inst.funct)) val issueds = VecInit(entries.map(_.bits.issued)) + val packed_deps = VecInit(entries.map(e => Cat(e.bits.deps.reverse))) + + dontTouch(valids) + dontTouch(functs) + dontTouch(issueds) + dontTouch(packed_deps) val pop_count_packed_deps = VecInit(entries.map(e => Mux(e.valid, PopCount(e.bits.deps), 0.U))) val min_pop_count = pop_count_packed_deps.reduce((acc, d) => minOf(acc, d)) @@ -336,11 +379,9 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf } assert(cycles_since_issue < 10000.U, "pipeline stall") - val instructions_allocated = RegInit(0.U(32.W)) - when (io.alloc.fire()) { - instructions_allocated := instructions_allocated + 1.U + for (e <- entries) { + dontTouch(e.bits.allocated_at) } - dontTouch(instructions_allocated) val cntr = Counter(10000000) when (cntr.inc()) { diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 2dbaf6c77..61fec0aba 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -31,7 +31,10 @@ class ScratchpadMemWriteRequest(local_addr_t: LocalAddr) (implicit p: Parameters) extends CoreBundle { val vaddr = UInt(coreMaxAddrBits.W) val laddr = local_addr_t.cloneType + val len = UInt(16.W) // TODO don't use a magic number for the width here + val block = UInt(8.W) // TODO don't use a magic number for the width here + val cmd_id = UInt(8.W) // TODO don't use a magic number here val status = new MStatus @@ -227,6 +230,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, writeData_is_all_zeros -> 0.U, writeData_is_full_width -> fullAccWriteData )) + writer.module.io.req.bits.block := write_issue_q.io.deq.bits.block writer.module.io.req.bits.status := write_issue_q.io.deq.bits.status writer.module.io.req.bits.pool_en := write_issue_q.io.deq.bits.pool_en writer.module.io.req.bits.store_en := write_issue_q.io.deq.bits.store_en diff --git a/src/main/scala/gemmini/StoreController.scala b/src/main/scala/gemmini/StoreController.scala index 561061674..98584bca8 100644 --- a/src/main/scala/gemmini/StoreController.scala +++ b/src/main/scala/gemmini/StoreController.scala @@ -35,8 +35,13 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm val stride = Reg(UInt(coreMaxAddrBits.W)) val block_rows = meshRows * tileRows + val block_stride = block_rows.U + val block_cols = meshColumns * tileColumns + val max_blocks = (dma_maxbytes / (block_cols * inputType.getWidth / 8)) max 1 + //val row_counter = RegInit(0.U(log2Ceil(block_rows).W)) - val row_counter = RegInit(0.U(12.W)) + val row_counter = RegInit(0.U(12.W)) // TODO magic number + val block_counter = RegInit(0.U(8.W)) // TODO magic number // Pooling variables val pool_stride = Reg(UInt(2.W)) // When this is 0, pooling is disabled // TODO magic number @@ -71,6 +76,7 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm val localaddr = cmd.bits.cmd.rs2.asTypeOf(local_addr_t) val cols = cmd.bits.cmd.rs2(32 + mvout_cols_bits - 1, 32) // TODO magic numbers val rows = cmd.bits.cmd.rs2(48 + mvout_rows_bits - 1, 48) // TODO magic numbers + val blocks = (cols / block_cols.U) + (cols % block_cols.U =/= 0.U) val config_stride = cmd.bits.cmd.rs2 val config_pool_stride = cmd.bits.cmd.rs1(5, 4) // TODO magic numbers val config_pool_size = cmd.bits.cmd.rs1(7, 6) // TODO magic numbers @@ -84,7 +90,8 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm val mstatus = cmd.bits.cmd.status - val localaddr_plus_row_counter = localaddr + row_counter + val current_vaddr = vaddr + row_counter * stride + val current_localaddr = localaddr + (block_counter * block_stride + row_counter) val pool_row_addr = localaddr + (orow * pool_ocols +& ocol) when (orow_is_negative || ocol_is_negative || orow >= pool_orows || ocol >= pool_ocols) { @@ -106,7 +113,7 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm val rob_id = UInt(log2Up(rob_entries).W) } - val cmd_tracker_max_rows = (block_rows max + val cmd_tracker_max_rows = ((block_rows * max_blocks) max (((1 << pool_orows.getWidth)-1) * ((1 << pool_ocols.getWidth)-1) + 2*((1 << pool_lpad.getWidth)-1) + 2*((1 << pool_upad.getWidth)-1))) min ((config.sp_banks * config.sp_bank_entries) max (config.acc_banks * config.acc_bank_entries)) @@ -116,21 +123,22 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm // DMA IO wiring io.dma.req.valid := (control_state === waiting_for_command && cmd.valid && DoStore && cmd_tracker.io.alloc.ready) || control_state === waiting_for_dma_req_ready || - (control_state === sending_rows && row_counter =/= 0.U) || // TODO Do we really have to check whether the counters should be 0 here? + (control_state === sending_rows && (block_counter =/= 0.U || row_counter =/= 0.U)) || (control_state === pooling && (wcol_counter =/= 0.U || wrow_counter =/= 0.U || pocol_counter =/= 0.U || porow_counter =/= 0.U)) - io.dma.req.bits.vaddr := Mux(pooling_is_enabled || mvout_1d_enabled, pool_vaddr, vaddr + row_counter * stride) - io.dma.req.bits.laddr := Mux(pooling_is_enabled, pool_row_addr, localaddr_plus_row_counter) //Todo: laddr for 1D? + io.dma.req.bits.vaddr := Mux(pooling_is_enabled || mvout_1d_enabled, pool_vaddr, current_vaddr) + io.dma.req.bits.laddr := Mux(pooling_is_enabled, pool_row_addr, current_localaddr) //Todo: laddr for 1D? - io.dma.req.bits.len := cols + io.dma.req.bits.len := Mux(block_counter === blocks - 1.U, ((cols - 1.U) % block_cols.U) + 1.U, block_cols.U) + io.dma.req.bits.block := block_counter io.dma.req.bits.status := mstatus io.dma.req.bits.pool_en := pooling_is_enabled && (wrow_counter =/= 0.U || wcol_counter =/= 0.U) - io.dma.req.bits.store_en := !pooling_is_enabled || - (wrow_counter === pool_size - 1.U && wcol_counter === pool_size - 1.U) + io.dma.req.bits.store_en := Mux(pooling_is_enabled, wrow_counter === pool_size - 1.U && wcol_counter === pool_size - 1.U, + block_counter === blocks - 1.U) // Command tracker IO cmd_tracker.io.alloc.valid := control_state === waiting_for_command && cmd.valid && DoStore - cmd_tracker.io.alloc.bits.bytes_to_read := Mux(!pooling_is_enabled, Mux(mvout_1d_enabled, mvout_1d_rows, rows), pool_total_rows) // TODO do we have to add upad and lpad to this? + cmd_tracker.io.alloc.bits.bytes_to_read := Mux(!pooling_is_enabled, Mux(mvout_1d_enabled, mvout_1d_rows, rows*blocks), pool_total_rows) // TODO do we have to add upad and lpad to this? cmd_tracker.io.alloc.bits.tag.rob_id := cmd.bits.rob_id.bits cmd_tracker.io.request_returned.valid := io.dma.resp.fire() // TODO use a bundle connect @@ -155,13 +163,18 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm pocol_counter := wrappingAdd(pocol_counter, 1.U, pool_ocols) porow_counter := wrappingAdd(porow_counter, 1.U, pool_orows, pocol_counter === pool_ocols - 1.U) } - row_counter := Mux(mvout_1d_enabled, wrappingAdd(row_counter, 1.U, mvout_1d_rows), wrappingAdd(row_counter, 1.U, rows)) + + block_counter := wrappingAdd(block_counter, 1.U, blocks) + row_counter := Mux(mvout_1d_enabled, wrappingAdd(row_counter, 1.U, mvout_1d_rows), wrappingAdd(row_counter, 1.U, rows, block_counter === blocks - 1.U)) }.otherwise { wcol_counter := wrappingAdd(wcol_counter, 1.U, pool_size) wrow_counter := wrappingAdd(wrow_counter, 1.U, pool_size, wcol_counter === pool_size - 1.U) pocol_counter := wrappingAdd(pocol_counter, 1.U, pool_pocols, wrow_counter === pool_size - 1.U && wcol_counter === pool_size - 1.U) porow_counter := wrappingAdd(porow_counter, 1.U, pool_porows, pocol_counter === pool_pocols - 1.U && wrow_counter === pool_size - 1.U && wcol_counter === pool_size - 1.U) } + + assert(!(io.dma.req.bits.laddr.read_full_acc_row && blocks > 1.U), "Block-mvouts are not permitted when moving out full accumulator data") + assert(!((pooling_is_enabled || mvout_1d_enabled) && blocks > 1.U), "Block-mvouts are not permitted when pooling") } // Control logic @@ -201,11 +214,13 @@ class StoreController[T <: Data : Arithmetic, U <: Data, V <: Data](config: Gemm } is (sending_rows) { - // TODO Is it really possible for row_counter to be 0 here? - val last_row = row_counter === 0.U || (Mux(mvout_1d_enabled, row_counter === mvout_1d_rows - 1.U, row_counter === rows - 1.U) && io.dma.req.fire()) + val last_block = block_counter === blocks - 1.U && io.dma.req.fire() + val last_row = Mux(mvout_1d_enabled, row_counter === mvout_1d_rows - 1.U, row_counter === rows - 1.U) && io.dma.req.fire() //normal mvout: row, 1D mvout: orows*ocols - when (last_row) { + val only_one_dma_req = block_counter === 0.U && row_counter === 0.U // This is a special case when only one DMA request is made + + when ((last_block && last_row) || only_one_dma_req) { control_state := waiting_for_command cmd.ready := true.B } From 1c5fb098e52fa089c4cf1997ea9b93779f41af8b Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Thu, 11 Mar 2021 11:29:18 -0800 Subject: [PATCH 040/123] Address PR feedback --- src/main/scala/gemmini/AccumulatorMem.scala | 2 -- src/main/scala/gemmini/Configs.scala | 2 +- src/main/scala/gemmini/GemminiConfigs.scala | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 1eaddd15f..9adfdd47a 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -87,8 +87,6 @@ class AccumulatorMem[T <: Data, U <: Data]( val w_sum = VecInit((RegNext(acc_rdata) zip wdata_buf).map { case (rv, wv) => VecInit((rv zip wv).map(t => t._1 + t._2)) }) - val counter = RegInit(0.U(32.W)) - counter := counter + 1.U if (!acc_singleported) { val mem = TwoPortSyncMem(n, t, t.getWidth / 8) // TODO We assume byte-alignment here. Use aligned_to instead diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index 7e37a7ea0..c57911b84 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -48,7 +48,7 @@ object GemminiConfigs { ex_queue_length = 8, rob_entries = 16, - hasIm2col = true, //declare im2col block + hasIm2col = false, //declare im2col block sp_banks = 4, sp_singleported = true, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index 646819046..b8df083b1 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -72,6 +72,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( case CapacityInKilobytes(kb) => kb * 1024 * 8 / (acc_banks * meshColumns * tileColumns * accType.getWidth) case CapacityInMatrices(ms) => ms * meshRows * tileRows / acc_banks } + require (!acc_singleported || (num_acc_sub_banks <= 4 && isPow2(num_acc_sub_banks))) val local_addr_t = new LocalAddr(sp_banks, sp_bank_entries, acc_banks, acc_bank_entries) From 373bf99c6aa95fbb4a41498b722c7bf0d2350994 Mon Sep 17 00:00:00 2001 From: SeahK Date: Thu, 11 Mar 2021 16:35:53 -0800 Subject: [PATCH 041/123] merging with dev, fixed loopmatmul bug --- src/main/scala/gemmini/Controller.scala | 12 +- src/main/scala/gemmini/DMA.scala | 128 +++++++++++++- src/main/scala/gemmini/GemminiISA.scala | 3 + src/main/scala/gemmini/LoadController.scala | 10 +- src/main/scala/gemmini/LoopLoader.scala | 175 ++++++++++++++++++++ src/main/scala/gemmini/LoopMatmul.scala | 19 ++- src/main/scala/gemmini/Scratchpad.scala | 29 ++++ src/main/scala/gemmini/Util.scala | 9 + 8 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 src/main/scala/gemmini/LoopLoader.scala diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 75d59c755..4d1a00fa7 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -109,8 +109,18 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] val raw_cmd = Queue(io.cmd) + // Loop Loader (load A or B) + // ToDo: collaborate with loopconv fsm (currently, only Loop matmul fsm) + val pause_monitor = spad.module.io.pause_out + val (loop_ld_cmd, loop_ld_unroller_busy, loop_ld_latency, loop_ld_alert, loop_ld_pause_turn) = LoopLoader(raw_cmd, pause_monitor, meshRows*tileRows, coreMaxAddrBits, sp_banks * sp_bank_entries, + inputType.getWidth, dma_maxbytes) + loop_ld_cmd.ready := false.B + spad.module.io.latency_in := loop_ld_latency + spad.module.io.alert_cycles_in := loop_ld_alert + spad.module.io.pause_turn_in := loop_ld_pause_turn + // TODO replace 4,12,2 with parameters based on ROB size - val (conv_cmd, loop_conv_unroller_busy) = LoopConv(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, + val (conv_cmd, loop_conv_unroller_busy) = LoopConv(loop_ld_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 7af1751da..43e5b3ad7 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -27,6 +27,11 @@ class StreamReadRequest[U <: Data](spad_rows: Int, acc_rows: Int, mvin_scale_t_b val block_stride = UInt(16.W) // TODO magic number val cmd_id = UInt(8.W) // TODO magic number + // for conflict monitoring + val monitor_conflict = Bool() + val monitor_conflict_start = Bool() + val monitor_conflict_end = Bool() + override def cloneType: StreamReadRequest.this.type = new StreamReadRequest(spad_rows, acc_rows, mvin_scale_t_bits).asInstanceOf[this.type] } @@ -61,7 +66,25 @@ class StreamReader[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T val tlb = new FrontendTLBIO val busy = Output(Bool()) val flush = Input(Bool()) + + //for monitoring conflicts, latency + val latency_in = Input(UInt(16.W)) + val alert_cycles_in = Input(UInt(6.W)) + val latency_out = Output(UInt(16.W)) + val alert_cycles_out = Output(UInt(6.W)) + val pause_turn_in = Input(UInt(3.W)) + val pause_turn_out = Output(UInt(3.W)) + + //for pausing monitoring + val pause_out = Output(Bool()) }) + io.latency_out := io.latency_in + io.alert_cycles_out := io.alert_cycles_in + io.pause_turn_out := io.pause_turn_in + core.module.io.latency := io.latency_out + core.module.io.alert_cycles := io.alert_cycles_out + io.pause_out := core.module.io.pause + core.module.io.pause_turn := io.pause_turn_out val nCmds = (nXacts / meshRows) + 1 @@ -135,6 +158,12 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val beatData = Decoupled(new StreamReadBeat(nXacts, beatBits, maxBytes)) val tlb = new FrontendTLBIO val flush = Input(Bool()) + + //for monitoring conflicts, latency + val latency = Input(UInt(16.W)) + val alert_cycles = Input(UInt(6.W)) + val pause_turn = Input(UInt(3.W)) + val pause = Output(Bool()) }) val s_idle :: s_req_new_block :: Nil = Enum(2) @@ -157,6 +186,11 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val shift = UInt(log2Up(maxBytes).W) //val paddr = UInt(paddrBits.W) val vaddr = UInt(vaddrBits.W) + + //for bank conflict monitoring + val monitor_conflict = Bool() + val monitor_conflict_start = Bool() + val monitor_conflict_end = Bool() } // TODO Can we filter out the larger read_sizes here if the systolic array is small, in the same way that we do so @@ -176,6 +210,11 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf packet.shift := vaddr_offset packet.vaddr := vaddr_aligned_to_size + //for bank conflict monitoring + packet.monitor_conflict := req.monitor_conflict + packet.monitor_conflict_start := req.monitor_conflict_start + packet.monitor_conflict_end := req.monitor_conflict_end + packet } val read_packet = read_packets.reduce { (acc, p) => @@ -185,6 +224,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val read_lg_size = read_packet.lg_size val read_bytes_read = read_packet.bytes_read val read_shift = read_packet.shift + //for bank conflict monitoring + val read_monitor = read_packet.monitor_conflict + val read_monitor_start = read_packet.monitor_conflict_start + val read_monitor_end = read_packet.monitor_conflict_end // Firing off TileLink read requests and allocating space inside the reservation buffer for them val get = edge.Get( @@ -197,6 +240,11 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_a = DataMirror.internal.chiselTypeClone[TLBundleA](tl.a.bits) val vaddr = Output(UInt(vaddrBits.W)) val status = Output(new MStatus) + + //for bank conflict monitoring + val monitor_conflict = Output(Bool()) + val monitor_conflict_start = Output(Bool()) + val monitor_conflict_end = Output(Bool()) } val untranslated_a = Wire(Decoupled(new TLBundleAWithInfo)) @@ -204,6 +252,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf untranslated_a.bits.tl_a := get untranslated_a.bits.vaddr := read_vaddr untranslated_a.bits.status := req.status + //for bank conflict monitoring + untranslated_a.bits.monitor_conflict := read_monitor + untranslated_a.bits.monitor_conflict_start := read_monitor_start + untranslated_a.bits.monitor_conflict_end := read_monitor_end // 0 goes to retries, 1 goes to state machine val retry_a = Wire(Decoupled(new TLBundleAWithInfo)) @@ -225,13 +277,83 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf translate_q.io.enq <> tlb_q.io.deq translate_q.io.deq.ready := true.B - retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready) + //retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready) + //retry_a.bits := translate_q.io.deq.bits + //assert(retry_a.ready) + + ///////////////////////////////////////////////////////////////////////////////////////// + val conflict_detected = RegInit(false.B) + retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready || conflict_detected) retry_a.bits := translate_q.io.deq.bits assert(retry_a.ready) - tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss - tl.a.bits := translate_q.io.deq.bits.tl_a + val tl_miss = tl.a.valid && !tl.a.ready + val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict + val tl_miss_counter = RegInit(0.U(6.W)) + val alert_cycles = RegInit(io.alert_cycles) + val pause_turn = RegInit(io.pause_turn) + val latency = RegInit(io.latency) + + tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) + when(tl_miss_counter >= alert_cycles){ //reached limit + conflict_detected := true.B + }.elsewhen(!tl_counter_trigger){ + tl_miss_counter := 0.U + } + // pause monitoring detecting logic + val (s_reset :: s_monitor_start :: s_conflict_detected :: Nil) = Enum(3) + val m_state = RegInit(s_reset) + val pause_detect = RegInit(false.B) + val pause_count = RegInit(0.U(2.W)) //Todo: parameterize it? + //val pause_monitor_start = RegInit(0.U(6.W)) + io.pause := pause_detect + when(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.monitor_conflict_end){ + when(m_state === s_reset) { + when(translate_q.io.deq.bits.monitor_conflict_start){ // to avoid false detection + m_state := s_monitor_start + //pause_monitor_start := pause_monitor_start + 1.U + alert_cycles := io.alert_cycles + latency := io.latency + pause_turn := io.pause_turn + } + }.elsewhen(m_state === s_monitor_start){ + when(tl_miss_counter >= alert_cycles){ + m_state := s_conflict_detected + } + //pause_monitor_start := 0.U + } + }.elsewhen(translate_q.io.deq.bits.monitor_conflict_end) { + when(m_state === s_conflict_detected) { + m_state := s_reset + pause_count := 0.U + pause_detect := false.B + }.elsewhen(m_state === s_monitor_start) { // no detection during time window + when(pause_count === pause_turn) { // pause monitoring + pause_detect := true.B // on 3rd time (ToDo: parameterize this?) + m_state := s_reset + pause_count := 0.U // reset pause counter + }.otherwise { + pause_count := pause_count + 1.U + m_state := s_reset + } + } + //pause_monitor_start := 0.U + } //ToDo: how to restart monitoring after pausing + + val tl_miss_timer = RegInit(0.U(16.W)) + tl_miss_timer := floorAdd(tl_miss_timer, 1.U, latency + 1.U, conflict_detected) + when(tl_miss_timer === latency){ //resolve miss counter temporary + tl_miss_counter := 0.U //reset miss counter + conflict_detected := false.B + } + tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss && !conflict_detected + tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr + ///////////////////////////////////////////////////////////////////////////////////////// + + //tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss + //tl.a.bits := translate_q.io.deq.bits.tl_a + //tl.a.bits.address := io.tlb.resp.paddr io.reserve.valid := state === s_req_new_block && untranslated_a.ready // TODO decouple "reserve.valid" from "tl.a.ready" io.reserve.entry.shift := read_shift diff --git a/src/main/scala/gemmini/GemminiISA.scala b/src/main/scala/gemmini/GemminiISA.scala index b49087c7f..53b7939d6 100644 --- a/src/main/scala/gemmini/GemminiISA.scala +++ b/src/main/scala/gemmini/GemminiISA.scala @@ -31,6 +31,9 @@ object GemminiISA { val LOOP_CONV_WS_CONFIG_5 = 20.U // *weights | *output val LOOP_CONV_WS_CONFIG_6 = 21.U // *bias, *input + val LOOP_LD_CONFIG_BOUNDS = 23.U + val LOOP_LD_CONFIG_ADDRS = 24.U + // rs1[2:0] values val CONFIG_EX = 0.U val CONFIG_LOAD = 1.U diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index d9221f5b2..ab668fdfd 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -37,11 +37,16 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val vaddr = cmd.bits.cmd.rs1 val localaddr = cmd.bits.cmd.rs2.asTypeOf(local_addr_t) val cols = cmd.bits.cmd.rs2(32 + mvin_cols_bits - 1, 32) // TODO magic numbers - val rows = cmd.bits.cmd.rs2(48 + mvin_rows_bits - 1, 48) // TODO magic numbers + //val rows = cmd.bits.cmd.rs2(48 + mvin_rows_bits - 1, 48) // TODO magic numbers + val rows = cmd.bits.cmd.rs2(60, 48) // TODO magic numbers val config_stride = cmd.bits.cmd.rs2 val config_scale = cmd.bits.cmd.rs1(32 + mvin_scale_t_bits - 1, 32) // TODO magic numbers val config_shrink = cmd.bits.cmd.rs1(2) // TODO magic numbers val config_block_stride = cmd.bits.cmd.rs1(31, 16) // TODO magic numbers + //monitor conflict using either A or B + val monitor_conflict = (cmd.bits.cmd.inst.funct === LOAD2_CMD || cmd.bits.cmd.inst.funct === LOAD_CMD) && cmd.bits.cmd.rs2(63) + val monitor_conflict_start = monitor_conflict && cmd.bits.cmd.rs2(61) + val monitor_conflict_end = monitor_conflict && cmd.bits.cmd.rs2(62) val mstatus = cmd.bits.cmd.status @@ -94,6 +99,9 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig io.dma.req.bits.has_acc_bitwidth := localaddr_plus_row_counter.is_acc_addr && !shrink io.dma.req.bits.all_zeros := all_zeros io.dma.req.bits.status := mstatus + io.dma.req.bits.monitor_conflict := monitor_conflict + io.dma.req.bits.monitor_conflict_start := monitor_conflict_start + io.dma.req.bits.monitor_conflict_end := monitor_conflict_end // Command tracker IO cmd_tracker.io.alloc.valid := control_state === waiting_for_command && cmd.valid && DoLoad diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala new file mode 100644 index 000000000..d41b58933 --- /dev/null +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -0,0 +1,175 @@ +package gemmini + +import chisel3._ +import chisel3.util._ +import chisel3.experimental._ +import freechips.rocketchip.tile.RoCCCommand +import freechips.rocketchip.config.Parameters +import GemminiISA._ +import Util._ + +class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: Int, dma_max_bytes: Int) + (implicit p: Parameters) extends Module { + val iterator_bitwidth = 16 + val max_block_len = (dma_max_bytes / (block_size * input_w / 8)) max 1 + + val io = IO(new Bundle { + val in = Flipped(Decoupled(new RoCCCommand)) + val out = Decoupled(new RoCCCommand) + val busy = Output(Bool()) + val latency = Output(UInt(iterator_bitwidth.W)) + val alert_cycle = Output(UInt(6.W)) + val pause_turn = Output(UInt(3.W)) + val pause_monitor = Input(Bool()) + }) + //queue for cmd + val cmd = Queue(io.in) + //val is_ldloop = cmd.bits.inst.funct === LOOP_LD + val is_ldconfig = cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS + + val pause_req = RegInit(false.B) + val loop_tag = RegInit(false.B) + val lock_tag = RegInit(false.B) + when(cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS){ + lock_tag := true.B + } // no need to force flip once seen LOOP_LD + when(cmd.bits.inst.funct === LOOP_WS){ + when(lock_tag){ + lock_tag := false.B + }.otherwise{ + loop_tag := ~loop_tag //force to flip to sync with loop matmul afterwards + } + } + // config states + val latency = RegInit(0.U(iterator_bitwidth.W)) //how many cycles to push + val alert_cycle = RegInit(0.U(6.W)) //raise flag after how much cycles? + val pause_turn = RegInit(1.U(3.W)) // how many turns to wait to pause monitoring TL ports + val dram_base_addr = RegInit(0.U(coreMaxAddrBits.W)) + val row_stride = RegInit(0.U(coreMaxAddrBits.W)) + + val row_iterator = RegInit(0.U(iterator_bitwidth.W))//Mux(req.transpose, j, k) //k + val col_iterator = RegInit(0.U(iterator_bitwidth.W))//Mux(req.transpose, k, j) //j + val max_row_iterator = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, max_j, max_k) + val max_col_iterator = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, max_k, max_j) + + val row_pad = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, pad_j, pad_k) + val col_pad = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, pad_k, pad_j) + + val max_blocks = max_block_len.asUInt() + val AB = RegInit(false.B) //false if B, true if A + //ToDo: rotate starting address like LoopMatmul.scala + val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) + val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) + val sp_addr_start = Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U) // Todo: need mux with 0 (skip A) + val dram_addr = dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U + val sp_addr = sp_addr_start + (row_iterator * max_col_iterator + col_iterator) * block_size.U + val blocks = Mux(col_iterator + max_blocks <= max_col_iterator, max_blocks, max_col_iterator-col_iterator) + val cols = (blocks * block_size.U) - Mux(col_iterator + blocks >= max_col_iterator, col_pad, 0.U) + val rows = block_size.U - Mux(max_row_iterator === max_row_iterator-1.U, row_pad, 0.U) + + object State extends ChiselEnum { + val idle, ld = Value + } + import State._ + val state = RegInit(idle) + val configured = RegInit(false.B) + + val conflict_monitor = !((alert_cycle === 0.U) || (latency === 0.U)) + val conflict_monitor_start = conflict_monitor && (row_iterator === 0.U && col_iterator === 0.U) + val conflict_monitor_end = conflict_monitor && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) + + //ToDo: either load A or B (for now just do with B) + val load_cmd = Wire(new RoCCCommand()) + load_cmd := DontCare + load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) + load_cmd.rs1 := dram_addr + load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + io.busy := cmd.valid || configured + io.alert_cycle := alert_cycle + io.latency := latency + io.pause_turn := pause_turn + + // fix loop_ws command + val loop_ws_state = RegInit(idle) + val is_loop_ws_addr = cmd.bits.inst.funct === LOOP_WS_CONFIG_ADDRS_AB + val fixed_loop_cmd = Wire(new RoCCCommand()) + fixed_loop_cmd := DontCare + fixed_loop_cmd.inst.funct := LOOP_WS_CONFIG_ADDRS_AB + fixed_loop_cmd.rs1 := Mux(AB, 0.U, cmd.bits.rs1) + fixed_loop_cmd.rs2 := Mux(AB, cmd.bits.rs2, 0.U) + + val unlock_monitor = RegInit(0.U(4.W)) + val unlock_cycle = RegInit(3.U(4.W)) + unlock_monitor := floorAdd(unlock_monitor, 1.U, unlock_cycle + pause_turn - 1.U, pause_req && is_loop_ws_addr & lock_tag && cmd.fire()) + when(!pause_req){ + unlock_monitor := 0.U + } + //when(!configured){ + when(cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS && cmd.valid){ + pause_req := io.pause_monitor + } + + val unlock = unlock_monitor >= unlock_cycle - 1.U // ToDo: change this number + + io.out.bits := Mux(configured, load_cmd, Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && conflict_monitor, fixed_loop_cmd, cmd.bits)) + io.out.bits.status := cmd.bits.status + io.out.valid := Mux(configured, state =/= idle, cmd.valid && !is_ldconfig) + cmd.ready := Mux(is_ldconfig, !configured, !configured && io.out.ready) + +// when(cmd.valid && is_ldconfig && state === idle && (!pause_req || unlock)){ + when(cmd.valid && is_ldconfig && state === idle){ + switch(cmd.bits.inst.funct){ + is(LOOP_LD_CONFIG_BOUNDS){ + pause_turn := cmd.bits.rs2(iterator_bitwidth * 3 + 12, iterator_bitwidth * 3 + 10) + alert_cycle := cmd.bits.rs2(iterator_bitwidth * 3 + 5, iterator_bitwidth * 3) + latency := cmd.bits.rs2(iterator_bitwidth * 3 - 1, iterator_bitwidth * 2) //ToDo: give this to DMA + unlock_cycle := cmd.bits.rs2(iterator_bitwidth * 3 + 9, iterator_bitwidth * 3 + 6) + max_col_iterator := cmd.bits.rs2(iterator_bitwidth * 2 - 1, iterator_bitwidth) + max_row_iterator := cmd.bits.rs2(iterator_bitwidth-1, 0) + + AB := cmd.bits.rs1(63) + col_pad := cmd.bits.rs1(iterator_bitwidth * 2 - 1, iterator_bitwidth) + row_pad := cmd.bits.rs1(iterator_bitwidth-1, 0) + } + is(LOOP_LD_CONFIG_ADDRS){ + when(!pause_req || unlock) { + dram_base_addr := cmd.bits.rs1 + row_stride := cmd.bits.rs2 + when(conflict_monitor) { // if latency == 0, don't unroll + configured := true.B + state := ld + }.otherwise { + loop_tag := ~loop_tag + } + } + } + } + } + when(io.out.fire() && state === ld){ + val row_blocks = 1.U + val col_blocks = max_blocks + + val next_col = floorAdd(col_iterator, col_blocks, max_col_iterator) + val next_row = floorAdd(row_iterator, row_blocks, max_row_iterator, next_col === 0.U) + + row_iterator := next_row + col_iterator := next_col + + when (next_row === 0.U && next_col === 0.U) { //finished loading + state := idle + configured := false.B + loop_tag := ~loop_tag + } + } + +} + +object LoopLoader{ + def apply(in: DecoupledIO[RoCCCommand], pause_monitor: Bool, block_size: Int, coreMaxAddrBits: Int, max_addr: Int, input_w: Int, dma_max_bytes: Int) + (implicit p: Parameters): Tuple5[DecoupledIO[RoCCCommand], Bool, UInt, UInt, UInt] = { + val lld = Module(new LoopLoader(block_size, coreMaxAddrBits, max_addr, input_w, dma_max_bytes)) + lld.io.in <> in + lld.io.pause_monitor <> pause_monitor + (lld.io.out, lld.io.busy, lld.io.latency, lld.io.alert_cycle, lld.io.pause_turn) + } +} \ No newline at end of file diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 74db8914d..24de572f2 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -77,12 +77,14 @@ class LoopMatmulLdA(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In io.k := k io.idle := state === idle - io.cmd.valid := state =/= idle && !io.rob_overloaded + io.cmd.valid := state =/= idle && !io.rob_overloaded && (req.dram_addr =/= 0.U) io.cmd.bits := mvin_cmd io.loop_id := req.loop_id - when (io.cmd.fire()) { + when (req.dram_addr === 0.U) { + state := idle + }.elsewhen (io.cmd.fire()) { // The order here is k, j, i val i_blocks = Mux(req.transpose, max_blocks, 1.U) val k_blocks = Mux(req.transpose, 1.U, max_blocks) @@ -159,7 +161,7 @@ class LoopMatmulLdB(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val max_col_dim = Mux(req.transpose, req.max_k, req.max_j) val max_blocks = Mux(max_col_dim <= max_block_len.U, max_col_dim, max_block_len.U) - val sp_addr_start = req.addr_end - req.max_k * req.max_j * block_size.U + val sp_addr_start = req.addr_end - req.max_k * req.max_j * block_size.U + block_size.U val dram_addr = req.dram_addr + (row_iterator * req.dram_stride + col_iterator) * block_size.U * (input_w/8).U val sp_addr = sp_addr_start + (row_iterator * max_col_iterator + col_iterator) * block_size.U @@ -178,13 +180,14 @@ class LoopMatmulLdB(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In io.j := j io.idle := state === idle - io.cmd.valid := state =/= idle && !io.rob_overloaded + io.cmd.valid := state =/= idle && !io.rob_overloaded && (req.dram_addr =/= 0.U) io.cmd.bits := mvin_cmd io.loop_id := req.loop_id - when (io.cmd.fire()) { - // The order here is k, j, i + when (req.dram_addr === 0.U) { + state := idle + }.elsewhen (io.cmd.fire()) { // The order here is k, j, i val j_blocks = Mux(req.transpose, 1.U, max_blocks) val k_blocks = Mux(req.transpose, max_blocks, 1.U) @@ -348,7 +351,7 @@ class LoopMatmulExecute(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth val d_addr_start = (BigInt(1) << 31).U | req.c_addr_start val c_addr_start = (BigInt(3) << 30).U | req.c_addr_start - val b_addr_start = req.b_addr_end - req.max_k * req.max_j * block_size.U + val b_addr_start = req.b_addr_end - req.max_k * req.max_j * block_size.U + block_size.U val k = Reg(UInt(iterator_bitwidth.W)) val j = Reg(UInt(iterator_bitwidth.W)) @@ -622,7 +625,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: // Create states val concurrent_loops = 2 val loops = Reg(Vec(concurrent_loops, new LoopMatmulState(iterator_bitwidth, coreMaxAddrBits, max_addr, max_acc_addr))) - val head_loop_id = Reg(UInt(log2Up(concurrent_loops).W)) + val head_loop_id = RegInit(0.U(log2Up(concurrent_loops).W)) val tail_loop_id = (~head_loop_id).asUInt() // This is the loop that we always try to configure if available val head_loop = loops(head_loop_id) val tail_loop = loops(tail_loop_id) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 61fec0aba..3e06000fb 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -24,6 +24,11 @@ class ScratchpadMemReadRequest[U <: Data](local_addr_t: LocalAddr, scale_t_bits: val cmd_id = UInt(8.W) // TODO don't use a magic number here val status = new MStatus + //for bank conflict monitoring + val monitor_conflict = Bool() + val monitor_conflict_start = Bool() + val monitor_conflict_end = Bool() + override def cloneType: this.type = new ScratchpadMemReadRequest(local_addr_t, scale_t_bits).asInstanceOf[this.type] } @@ -199,6 +204,17 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, // Misc. ports val busy = Output(Bool()) val flush = Input(Bool()) + + // for detecting conflicts + val latency_in = Input(UInt(16.W)) + val alert_cycles_in = Input(UInt(6.W)) + val latency_out = Output(UInt(16.W)) + val alert_cycles_out = Output(UInt(6.W)) + val pause_turn_in = Input(UInt(3.W)) + val pause_turn_out = Output(UInt(3.W)) + + //for pausing monitoring + val pause_out = Output(Bool()) }) val write_dispatch_q = Queue(io.dma.write.req) @@ -269,6 +285,10 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, reader.module.io.req.bits.block_stride := read_issue_q.io.deq.bits.block_stride reader.module.io.req.bits.status := read_issue_q.io.deq.bits.status reader.module.io.req.bits.cmd_id := read_issue_q.io.deq.bits.cmd_id + //for bank conflict monitoring + reader.module.io.req.bits.monitor_conflict := read_issue_q.io.deq.bits.monitor_conflict + reader.module.io.req.bits.monitor_conflict_end := read_issue_q.io.deq.bits.monitor_conflict_end + reader.module.io.req.bits.monitor_conflict_start := read_issue_q.io.deq.bits.monitor_conflict_start val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier(config.mvin_scale_args, config.inputType, config.meshColumns * config.tileColumns, chiselTypeOf(reader.module.io.resp.bits), is_acc = false) val (mvin_scale_acc_in, mvin_scale_acc_out) = if (mvin_scale_shared) (mvin_scale_in, mvin_scale_out) else @@ -325,6 +345,15 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, writer.module.io.flush := io.flush reader.module.io.flush := io.flush + //for monitoring conflicts + io.latency_out := io.latency_in + io.alert_cycles_out := io.alert_cycles_in + io.pause_turn_out := io.pause_turn_in + io.pause_out := reader.module.io.pause_out + reader.module.io.latency_in := io.latency_out + reader.module.io.alert_cycles_in := io.alert_cycles_out + reader.module.io.pause_turn_in := io.pause_turn_out + io.busy := writer.module.io.busy || reader.module.io.busy || write_issue_q.io.deq.valid { diff --git a/src/main/scala/gemmini/Util.scala b/src/main/scala/gemmini/Util.scala index dd837c7d6..70eef0c4d 100644 --- a/src/main/scala/gemmini/Util.scala +++ b/src/main/scala/gemmini/Util.scala @@ -35,6 +35,15 @@ object Util { Mux(u +& v > max, max, u + v) } + def satAdd(u: UInt, v: UInt, max_plus_one: UInt, en: Bool = true.B): UInt = { + val max = max_plus_one - 1.U + + MuxCase(u + v, Seq( + (!en) -> u, + ((u +& v) > max) -> max + )) + } + def floorAdd(u: UInt, n: UInt, max_plus_one: UInt, en: Bool = true.B): UInt = { val max = max_plus_one - 1.U From a62a404dd620cf69800328e0817ba4475c686fc9 Mon Sep 17 00:00:00 2001 From: SeahK Date: Thu, 11 Mar 2021 17:51:19 -0800 Subject: [PATCH 042/123] change to loopconv --- src/main/scala/gemmini/LoopConv.scala | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 83f34fcf9..a14484aec 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -239,7 +239,7 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw val config_cmd = Wire(new RoCCCommand) config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD - config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.input_spad_stride << 16.U) | (0.U << 3) | 1.U + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (req.derived_params.input_spad_stride << 16.U).asUInt() | (0.U << 3).asUInt() | 1.U config_cmd.rs2 := in_channels * (input_w/8).U val mvin_cmd = Wire(new RoCCCommand) @@ -253,11 +253,13 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw io.idle := state === idle io.loop_id := req.loop_id - io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded + io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded && req.dram_addr =/= 0.U io.cmd.bits := Mux(state === config, config_cmd, mvin_cmd) // Sending outputs - when(io.cmd.fire()) { + when (req.dram_addr === 0.U) { + state := idle + }.elsewhen (io.cmd.fire()) { when (state === config) { state := ld }.otherwise { @@ -328,7 +330,7 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit // Derived parameters val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) val B_rows = out_channels_per_bank * kcols * krows * kchs - val addr_start = req.addr_end - B_rows + val addr_start = req.addr_end - B_rows + block_size.U // for possible loopconv bug (like the loopmatmul one) // Iterators val och = Reg(UInt(large_iterator_bitwidth.W)) @@ -348,25 +350,27 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit val config_cmd = Wire(new RoCCCommand) config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD - config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.weight_spad_stride << 16.U) | (1.U << 3) | 1.U + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (req.derived_params.weight_spad_stride << 16.U).asUInt() | (1.U << 3).asUInt() | 1.U config_cmd.rs2 := out_channels * (input_w/8).U val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD2_CMD mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := (K << 48.U) | (J << 32.U) | spad_addr + mvin_cmd.rs2 := (K << 48.U).asUInt() | (J << 32.U).asUInt() | spad_addr // Inputs and outputs io.req.ready := state === idle io.idle := state === idle io.loop_id := req.loop_id - io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded + io.cmd.valid := state =/= idle && !io.wait_for_prev_loop && !io.rob_overloaded && req.dram_addr =/= 0.U io.cmd.bits := Mux(state === config, config_cmd, mvin_cmd) // Sending outputs - when(io.cmd.fire()) { + when (req.dram_addr === 0.U) { + state := idle + }.elsewhen (io.cmd.fire()) { when (state === config) { state := ld }.otherwise { @@ -440,7 +444,7 @@ class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_itera val B_rows = out_channels_per_bank * kcols * krows * kchs val a_addr_start = req.a_addr_start - val b_addr_start = req.b_addr_end - B_rows + val b_addr_start = req.b_addr_end - B_rows + block_size.U //for possible loopconv bug (like loopmatmul) val d_addr_start = (BigInt(1) << 31).U | req.c_addr_start val c_addr_start = (BigInt(3) << 30).U | req.c_addr_start From efaf15c77175f62523d04e1c14fb9c785d5249d5 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 17:39:23 -0800 Subject: [PATCH 043/123] save --- src/main/scala/gemmini/GemminiISA.scala | 2 + src/main/scala/gemmini/LoopLoader.scala | 90 +++++++++++++++++++------ 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/main/scala/gemmini/GemminiISA.scala b/src/main/scala/gemmini/GemminiISA.scala index 53b7939d6..16e15235b 100644 --- a/src/main/scala/gemmini/GemminiISA.scala +++ b/src/main/scala/gemmini/GemminiISA.scala @@ -33,6 +33,8 @@ object GemminiISA { val LOOP_LD_CONFIG_BOUNDS = 23.U val LOOP_LD_CONFIG_ADDRS = 24.U + val LOOP_CONV_LD_CONFIG_BOUNDS = 25.U + val LOOP_CONV_LD_CONFIG_ADDRS = 26.U // rs1[2:0] values val CONFIG_EX = 0.U diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index d41b58933..6e5a0c13d 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -25,15 +25,17 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I //queue for cmd val cmd = Queue(io.in) //val is_ldloop = cmd.bits.inst.funct === LOOP_LD - val is_ldconfig = cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS + val is_matmul_ldconfig = cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS + val is_conv_ldconfig = cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_BOUNDS val pause_req = RegInit(false.B) val loop_tag = RegInit(false.B) val lock_tag = RegInit(false.B) + val is_conv = RegInit(false.B) when(cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS){ lock_tag := true.B } // no need to force flip once seen LOOP_LD - when(cmd.bits.inst.funct === LOOP_WS){ + when(cmd.bits.inst.funct === LOOP_WS || cmd.bits.inst.funct === LOOP_CONV_WS){ when(lock_tag){ lock_tag := false.B }.otherwise{ @@ -55,6 +57,15 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val row_pad = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, pad_j, pad_k) val col_pad = Reg(UInt(iterator_bitwidth.W)) //Mux(req.transpose, pad_k, pad_j) + //conv parameters + val out_channels = RegInit(0.U(16.W)) + val in_channels = RegInit(0.U(16.W)) + val kernel_dim = RegInit(0.U(4.W)) + val krows = RegInit(0.U(4.W)) + val kcols = RegInit(0.U(4.W)) + val kchs = RegInit(0.U(16.W)) + val ochs = RegInit(0.U(16.W)) + val max_blocks = max_block_len.asUInt() val AB = RegInit(false.B) //false if B, true if A //ToDo: rotate starting address like LoopMatmul.scala @@ -91,11 +102,11 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I // fix loop_ws command val loop_ws_state = RegInit(idle) - val is_loop_ws_addr = cmd.bits.inst.funct === LOOP_WS_CONFIG_ADDRS_AB + val is_loop_ws_addr = (cmd.bits.inst.funct === LOOP_WS_CONFIG_ADDRS_AB || cmd.bits.inst.funct === LOOP_CONV_WS_CONFIG_5) // for now, only weight for conv val fixed_loop_cmd = Wire(new RoCCCommand()) fixed_loop_cmd := DontCare - fixed_loop_cmd.inst.funct := LOOP_WS_CONFIG_ADDRS_AB - fixed_loop_cmd.rs1 := Mux(AB, 0.U, cmd.bits.rs1) + fixed_loop_cmd.inst.funct := cmd.bits.inst.funct//LOOP_WS_CONFIG_ADDRS_AB + fixed_loop_cmd.rs1 := Mux(cmd.bits.inst.funct === LOOP_CONV_WS_CONFIG_5, 0.U, Mux(AB, 0.U, cmd.bits.rs1)) //if conv, weight fixed_loop_cmd.rs2 := Mux(AB, cmd.bits.rs2, 0.U) val unlock_monitor = RegInit(0.U(4.W)) @@ -105,7 +116,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I unlock_monitor := 0.U } //when(!configured){ - when(cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS && cmd.valid){ + when((cmd.bits.inst.funct === LOOP_LD_CONFIG_BOUNDS || cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_BOUNDS) && cmd.valid){ pause_req := io.pause_monitor } @@ -113,11 +124,11 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I io.out.bits := Mux(configured, load_cmd, Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && conflict_monitor, fixed_loop_cmd, cmd.bits)) io.out.bits.status := cmd.bits.status - io.out.valid := Mux(configured, state =/= idle, cmd.valid && !is_ldconfig) - cmd.ready := Mux(is_ldconfig, !configured, !configured && io.out.ready) + io.out.valid := Mux(configured, state =/= idle, cmd.valid && !is_matmul_ldconfig && !is_conv_ldconfig) + cmd.ready := Mux(is_matmul_ldconfig || is_conv_ldconfig, !configured, !configured && io.out.ready) // when(cmd.valid && is_ldconfig && state === idle && (!pause_req || unlock)){ - when(cmd.valid && is_ldconfig && state === idle){ + when(cmd.valid && is_matmul_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_LD_CONFIG_BOUNDS){ pause_turn := cmd.bits.rs2(iterator_bitwidth * 3 + 12, iterator_bitwidth * 3 + 10) @@ -130,6 +141,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I AB := cmd.bits.rs1(63) col_pad := cmd.bits.rs1(iterator_bitwidth * 2 - 1, iterator_bitwidth) row_pad := cmd.bits.rs1(iterator_bitwidth-1, 0) + is_conv := false.B } is(LOOP_LD_CONFIG_ADDRS){ when(!pause_req || unlock) { @@ -144,21 +156,59 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I } } } + }.elsewhen(cmd.valid && is_conv_ldconfig && state === idle){ + switch(cmd.bits.inst.funct){ + is(LOOP_CONV_LD_CONFIG_BOUNDS){ + pause_turn := cmd.bits.rs2(60, 58) + alert_cycle := cmd.bits.rs2(53, 48) + latency := cmd.bits.rs2(47, 32) //ToDo: give this to DMA + unlock_cycle := cmd.bits.rs2(57, 54) + + kernel_dim := cmd.bits.rs1(31, 16)//can code more if needed + + krows := cmd.bits.rs2(63, 48) + kcols := cmd.bits.rs2(47, 32) + kchs := cmd.bits.rs2(31, 16) + ochs := cmd.bits.rs2(15, 0) + is_conv := true.B + } + is(LOOP_CONV_LD_CONFIG_ADDRS){ + when(!pause_req || unlock) { + dram_base_addr := cmd.bits.rs1 + out_channels := cmd.bits.rs2(31, 16) + in_channels := cmd.bits.rs2(15, 0) + //can code more + when(conflict_monitor) { + configured := true.B + state := ld + }.otherwise{ + loop_tag := ~loop_tag + } + } + } + } } - when(io.out.fire() && state === ld){ - val row_blocks = 1.U - val col_blocks = max_blocks - val next_col = floorAdd(col_iterator, col_blocks, max_col_iterator) - val next_row = floorAdd(row_iterator, row_blocks, max_row_iterator, next_col === 0.U) - row_iterator := next_row - col_iterator := next_col + when(io.out.fire() && state === ld) { + when(!is_conv) { + //matmul loop + val row_blocks = 1.U + val col_blocks = max_blocks - when (next_row === 0.U && next_col === 0.U) { //finished loading - state := idle - configured := false.B - loop_tag := ~loop_tag + val next_col = floorAdd(col_iterator, col_blocks, max_col_iterator) + val next_row = floorAdd(row_iterator, row_blocks, max_row_iterator, next_col === 0.U) + + row_iterator := next_row + col_iterator := next_col + + when(next_row === 0.U && next_col === 0.U) { //finished loading + state := idle + configured := false.B + loop_tag := ~loop_tag + } + }.otherwise{ + //conv loop } } From c7613a8cd84a5281997639c513f2fe6f970ddc9d Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 18:38:27 -0800 Subject: [PATCH 044/123] first trial with conv_fsm --- src/main/scala/gemmini/LoopLoader.scala | 68 ++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 6e5a0c13d..5049799b9 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -65,21 +65,39 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val kcols = RegInit(0.U(4.W)) val kchs = RegInit(0.U(16.W)) val ochs = RegInit(0.U(16.W)) + // conv Iterators + val och = RegInit(0.U(16.W)) + val krow = RegInit(0.U(4.W)) + val kcol = RegInit(0.U(4.W)) + val kch = RegInit(0.U(16.W)) val max_blocks = max_block_len.asUInt() val AB = RegInit(false.B) //false if B, true if A //ToDo: rotate starting address like LoopMatmul.scala val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) - val sp_addr_start = Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U) // Todo: need mux with 0 (skip A) - val dram_addr = dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U - val sp_addr = sp_addr_start + (row_iterator * max_col_iterator + col_iterator) * block_size.U + //for conv + val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) + val out_channels_per_bank = RegInit(0.U(8.W)) + out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) + val B_rows = out_channels_per_bank * kcols * krows * kchs + //val addr_start = B_sp_addr_end - B_rows + block_size.U + + val sp_addr_start = Mux(is_conv, B_sp_addr_end - B_rows + block_size.U, + Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U)) // Todo: need mux with 0 (skip A) + val dram_addr = Mux(is_conv, dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U, + dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channels +& och) * (input_w/8).U) + val sp_addr = sp_addr_start + Mux(is_conv, (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch, + (row_iterator * max_col_iterator + col_iterator) * block_size.U) val blocks = Mux(col_iterator + max_blocks <= max_col_iterator, max_blocks, max_col_iterator-col_iterator) val cols = (blocks * block_size.U) - Mux(col_iterator + blocks >= max_col_iterator, col_pad, 0.U) val rows = block_size.U - Mux(max_row_iterator === max_row_iterator-1.U, row_pad, 0.U) + // for conv rows and cols + val J = Mux(ochs - och > max_ochs_per_mvin, max_ochs_per_mvin, ochs - och) + val K = Mux(kchs - kch > block_size.U, block_size.U, kchs - kch) object State extends ChiselEnum { - val idle, ld = Value + val idle, config, ld = Value //added config for conv } import State._ val state = RegInit(idle) @@ -95,11 +113,26 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + + //for conv + val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow + val weight_spad_stride = krows * kcols * kchs + val config_cmd = Wire(new RoCCCommand) + config_cmd := DontCare + config_cmd.inst.funct := CONFIG_CMD + config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (weight_spad_stride << 16.U).asUInt() | (1.U << 3).asUInt() | 1.U + config_cmd.rs2 := out_channels * (input_w/8).U + //for conv + val mvin_cmd = Wire(new RoCCCommand) + mvin_cmd := DontCare + mvin_cmd.inst.funct := LOAD2_CMD // for now, only weight + mvin_cmd.rs1 := dram_addr + mvin_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr + io.busy := cmd.valid || configured io.alert_cycle := alert_cycle io.latency := latency io.pause_turn := pause_turn - // fix loop_ws command val loop_ws_state = RegInit(idle) val is_loop_ws_addr = (cmd.bits.inst.funct === LOOP_WS_CONFIG_ADDRS_AB || cmd.bits.inst.funct === LOOP_CONV_WS_CONFIG_5) // for now, only weight for conv @@ -122,7 +155,8 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock = unlock_monitor >= unlock_cycle - 1.U // ToDo: change this number - io.out.bits := Mux(configured, load_cmd, Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && conflict_monitor, fixed_loop_cmd, cmd.bits)) + io.out.bits := Mux(configured, Mux(is_conv, Mux(state === config, config_cmd, mvin_cmd), load_cmd), + Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && conflict_monitor, fixed_loop_cmd, cmd.bits)) io.out.bits.status := cmd.bits.status io.out.valid := Mux(configured, state =/= idle, cmd.valid && !is_matmul_ldconfig && !is_conv_ldconfig) cmd.ready := Mux(is_matmul_ldconfig || is_conv_ldconfig, !configured, !configured && io.out.ready) @@ -171,6 +205,12 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I kchs := cmd.bits.rs2(31, 16) ochs := cmd.bits.rs2(15, 0) is_conv := true.B + + // initialize for safety + krow := 0.U + kcol := 0.U + kch := 0.U + och := 0.U } is(LOOP_CONV_LD_CONFIG_ADDRS){ when(!pause_req || unlock) { @@ -180,7 +220,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I //can code more when(conflict_monitor) { configured := true.B - state := ld + state := config // for conv, idle -> config -> ld }.otherwise{ loop_tag := ~loop_tag } @@ -209,7 +249,21 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I } }.otherwise{ //conv loop + val next_kch = floorAdd(kch, block_size.U, kchs) + val next_kcol = floorAdd(kcol, 1.U, kcols, next_kch === 0.U) + val next_krow = floorAdd(krow, 1.U, krows, next_kcol === 0.U && next_kch === 0.U) + val next_och = floorAdd(och, max_ochs_per_mvin, ochs, next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + + kch := next_kch + kcol := next_kcol + krow := next_krow + och := next_och + + state := Mux(next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U, + idle, ld) } + }.elsewhen(io.out.fire() && state === config){ //for conv config + state := ld } } From 49786e9b8835a3c853a9bf67ac8bb6ae0df80539 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 19:05:15 -0800 Subject: [PATCH 045/123] first trial with conv_fsm --- src/main/scala/gemmini/LoopLoader.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 5049799b9..e34094e8e 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -32,7 +32,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val loop_tag = RegInit(false.B) val lock_tag = RegInit(false.B) val is_conv = RegInit(false.B) - when(cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS){ + when(cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_ADDRS){ lock_tag := true.B } // no need to force flip once seen LOOP_LD when(cmd.bits.inst.funct === LOOP_WS || cmd.bits.inst.funct === LOOP_CONV_WS){ @@ -140,7 +140,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I fixed_loop_cmd := DontCare fixed_loop_cmd.inst.funct := cmd.bits.inst.funct//LOOP_WS_CONFIG_ADDRS_AB fixed_loop_cmd.rs1 := Mux(cmd.bits.inst.funct === LOOP_CONV_WS_CONFIG_5, 0.U, Mux(AB, 0.U, cmd.bits.rs1)) //if conv, weight - fixed_loop_cmd.rs2 := Mux(AB, cmd.bits.rs2, 0.U) + fixed_loop_cmd.rs2 := Mux(is_conv, cmd.bits.rs2, Mux(AB, cmd.bits.rs2, 0.U)) //for now, not do input for conv val unlock_monitor = RegInit(0.U(4.W)) val unlock_cycle = RegInit(3.U(4.W)) From e67ecf5128b575ab64b1e5aec0c7be9f6aace948 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 19:15:43 -0800 Subject: [PATCH 046/123] fixed tag initialization --- src/main/scala/gemmini/LoopLoader.scala | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index e34094e8e..544256c48 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -29,9 +29,13 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val is_conv_ldconfig = cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_BOUNDS val pause_req = RegInit(false.B) - val loop_tag = RegInit(false.B) val lock_tag = RegInit(false.B) val is_conv = RegInit(false.B) + // for switching between conv and matmul + val loop_tag_conv = RegInit(false.B) + val loop_tag_matmul = RegInit(false.B) + val loop_tag = Mux(is_conv, loop_tag_conv, loop_tag_matmul) + when(cmd.bits.inst.funct === LOOP_LD_CONFIG_ADDRS || cmd.bits.inst.funct === LOOP_CONV_LD_CONFIG_ADDRS){ lock_tag := true.B } // no need to force flip once seen LOOP_LD @@ -39,7 +43,11 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(lock_tag){ lock_tag := false.B }.otherwise{ - loop_tag := ~loop_tag //force to flip to sync with loop matmul afterwards + when(is_conv){ + loop_tag_conv := ~loop_tag_conv + }.otherwise{ + loop_tag_matmul := ~loop_tag_matmul + } //force to flip to sync with loop matmul afterwards } } // config states @@ -185,7 +193,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I configured := true.B state := ld }.otherwise { - loop_tag := ~loop_tag + loop_tag_matmul := ~loop_tag_matmul } } } @@ -222,7 +230,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I configured := true.B state := config // for conv, idle -> config -> ld }.otherwise{ - loop_tag := ~loop_tag + loop_tag_conv := ~loop_tag_conv } } } @@ -245,7 +253,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(next_row === 0.U && next_col === 0.U) { //finished loading state := idle configured := false.B - loop_tag := ~loop_tag + loop_tag_matmul := ~loop_tag_matmul } }.otherwise{ //conv loop @@ -259,8 +267,11 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I krow := next_krow och := next_och - state := Mux(next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U, - idle, ld) + when(next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U){ //finished loading + state := idle + configured := false.B + loop_tag_conv := ~loop_tag_conv + } } }.elsewhen(io.out.fire() && state === config){ //for conv config state := ld From 50147e952f1b81fdbcb27d3cf74f8fa7b690c9ed Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 21:23:34 -0800 Subject: [PATCH 047/123] printf --- src/main/scala/gemmini/DMA.scala | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 43e5b3ad7..e712b7a61 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -349,6 +349,16 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss && !conflict_detected tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr + val cycles = freechips.rocketchip.util.WideCounter(32) + when(tl.a.fire()){ + printf("GEMMINI_MEM %x %x %x %x\n", cycles.value, p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address, tl.a.bits.size) + //printf(midas.targetutils.SynthesizePrintf("GEMMINI_MEM: %x %x %x\n", p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address, tl.a.bits.size)) + } + when(tl_miss){ + printf("GEMMINI_BLOCK %x %x\n", cycles.value, p(freechips.rocketchip.tile.TileKey).hartId.U) + //printf(midas.targetutils.SynthesizePrintf("GEMMINI_BLOCK: %x %x \n", p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address)) + + } ///////////////////////////////////////////////////////////////////////////////////////// //tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss From 8879d2cf4f1b674feef2c11b7604b27bd1b53425 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 21:52:58 -0800 Subject: [PATCH 048/123] ISA change --- src/main/scala/gemmini/GemminiISA.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/GemminiISA.scala b/src/main/scala/gemmini/GemminiISA.scala index 16e15235b..6f181457e 100644 --- a/src/main/scala/gemmini/GemminiISA.scala +++ b/src/main/scala/gemmini/GemminiISA.scala @@ -31,10 +31,10 @@ object GemminiISA { val LOOP_CONV_WS_CONFIG_5 = 20.U // *weights | *output val LOOP_CONV_WS_CONFIG_6 = 21.U // *bias, *input - val LOOP_LD_CONFIG_BOUNDS = 23.U - val LOOP_LD_CONFIG_ADDRS = 24.U - val LOOP_CONV_LD_CONFIG_BOUNDS = 25.U - val LOOP_CONV_LD_CONFIG_ADDRS = 26.U + val LOOP_LD_CONFIG_BOUNDS = 22.U + val LOOP_LD_CONFIG_ADDRS = 23.U + val LOOP_CONV_LD_CONFIG_BOUNDS = 24.U + val LOOP_CONV_LD_CONFIG_ADDRS = 25.U // rs1[2:0] values val CONFIG_EX = 0.U From 8e429ad0d577ba28504ebd95ffad26d298674176 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sat, 13 Mar 2021 22:10:04 -0800 Subject: [PATCH 049/123] config bugs --- src/main/scala/gemmini/LoopLoader.scala | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 544256c48..ab74f81b5 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -202,16 +202,15 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I switch(cmd.bits.inst.funct){ is(LOOP_CONV_LD_CONFIG_BOUNDS){ pause_turn := cmd.bits.rs2(60, 58) + unlock_cycle := cmd.bits.rs2(57, 54) alert_cycle := cmd.bits.rs2(53, 48) latency := cmd.bits.rs2(47, 32) //ToDo: give this to DMA - unlock_cycle := cmd.bits.rs2(57, 54) - - kernel_dim := cmd.bits.rs1(31, 16)//can code more if needed + kernel_dim := cmd.bits.rs2(15, 0)//can code more if needed - krows := cmd.bits.rs2(63, 48) - kcols := cmd.bits.rs2(47, 32) - kchs := cmd.bits.rs2(31, 16) - ochs := cmd.bits.rs2(15, 0) + krows := cmd.bits.rs1(63, 48) + kcols := cmd.bits.rs1(47, 32) + kchs := cmd.bits.rs1(31, 16) + ochs := cmd.bits.rs1(15, 0) is_conv := true.B // initialize for safety From a7d0b318944936f6c1fd562f5c432d99b5c1cb0d Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 14 Mar 2021 01:20:14 -0800 Subject: [PATCH 050/123] debugging --- src/main/scala/gemmini/LoopLoader.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index ab74f81b5..d276bde53 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -86,8 +86,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) - val out_channels_per_bank = RegInit(0.U(8.W)) - out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) + val out_channels_per_bank = ochs / block_size.U +& (ochs % block_size.U =/= 0.U) val B_rows = out_channels_per_bank * kcols * krows * kchs //val addr_start = B_sp_addr_end - B_rows + block_size.U From 908a9bc916dd442c10fefc44cb289941f8c7fe06 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 14 Mar 2021 01:36:11 -0800 Subject: [PATCH 051/123] decreased wire width --- src/main/scala/gemmini/LoopLoader.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index d276bde53..884a9136d 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -86,7 +86,8 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) - val out_channels_per_bank = ochs / block_size.U +& (ochs % block_size.U =/= 0.U) + val out_channels_per_bank = WireInit(0.U(8.W)) + out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) val B_rows = out_channels_per_bank * kcols * krows * kchs //val addr_start = B_sp_addr_end - B_rows + block_size.U From 0c24167aa76dd510507912d2cc33efb46856ac4d Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 14 Mar 2021 10:58:49 -0700 Subject: [PATCH 052/123] fixing bugs --- src/main/scala/gemmini/LoopLoader.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 884a9136d..256d5e1ec 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -93,7 +93,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val sp_addr_start = Mux(is_conv, B_sp_addr_end - B_rows + block_size.U, Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U)) // Todo: need mux with 0 (skip A) - val dram_addr = Mux(is_conv, dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U, + val dram_addr = Mux(!is_conv, dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U, dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channels +& och) * (input_w/8).U) val sp_addr = sp_addr_start + Mux(is_conv, (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch, (row_iterator * max_col_iterator + col_iterator) * block_size.U) @@ -112,7 +112,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val configured = RegInit(false.B) val conflict_monitor = !((alert_cycle === 0.U) || (latency === 0.U)) - val conflict_monitor_start = conflict_monitor && (row_iterator === 0.U && col_iterator === 0.U) + val conflict_monitor_start = conflict_monitor && (row_iterator === 0.U && col_iterator === 0.U) //ToDo: with conv val conflict_monitor_end = conflict_monitor && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) //ToDo: either load A or B (for now just do with B) From 548d16c2dce97cf300fa925c6c077a0b8814998b Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 14 Mar 2021 11:17:17 -0700 Subject: [PATCH 053/123] start/end for conv --- src/main/scala/gemmini/LoopLoader.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 256d5e1ec..fa196c220 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -112,8 +112,9 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val configured = RegInit(false.B) val conflict_monitor = !((alert_cycle === 0.U) || (latency === 0.U)) - val conflict_monitor_start = conflict_monitor && (row_iterator === 0.U && col_iterator === 0.U) //ToDo: with conv - val conflict_monitor_end = conflict_monitor && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) + val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) //ToDo: with conv + val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), + (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) //ToDo: either load A or B (for now just do with B) val load_cmd = Wire(new RoCCCommand()) From a3b497e59b3fef5bc33e981690816851b37b1149 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 14 Mar 2021 11:36:58 -0700 Subject: [PATCH 054/123] fixing bugs --- src/main/scala/gemmini/LoopLoader.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index fa196c220..7497117f5 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -112,9 +112,9 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val configured = RegInit(false.B) val conflict_monitor = !((alert_cycle === 0.U) || (latency === 0.U)) - val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) //ToDo: with conv + val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), - (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) + (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) && (state === ld) //ToDo: either load A or B (for now just do with B) val load_cmd = Wire(new RoCCCommand()) From 820fcf841f2ed54bf4e0e79f658763a86781dc61 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 15 Mar 2021 11:12:20 -0700 Subject: [PATCH 055/123] adding support for sw padding --- src/main/scala/gemmini/LoopLoader.scala | 6 +++++- src/main/scala/gemmini/ROB.scala | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 7497117f5..08ff2077d 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -73,6 +73,8 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val kcols = RegInit(0.U(4.W)) val kchs = RegInit(0.U(16.W)) val ochs = RegInit(0.U(16.W)) + val padding = RegInit(false.B) // SW padding for bank conflict + // conv Iterators val och = RegInit(0.U(16.W)) val krow = RegInit(0.U(4.W)) @@ -85,6 +87,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv + val out_channel_stride = Mux(padding, out_channels + max_blocks * block_size.U, out_channels) val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) val out_channels_per_bank = WireInit(0.U(8.W)) out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) @@ -94,7 +97,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val sp_addr_start = Mux(is_conv, B_sp_addr_end - B_rows + block_size.U, Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U)) // Todo: need mux with 0 (skip A) val dram_addr = Mux(!is_conv, dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U, - dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channels +& och) * (input_w/8).U) + dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channel_stride +& och) * (input_w/8).U) val sp_addr = sp_addr_start + Mux(is_conv, (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch, (row_iterator * max_col_iterator + col_iterator) * block_size.U) val blocks = Mux(col_iterator + max_blocks <= max_col_iterator, max_blocks, max_col_iterator-col_iterator) @@ -223,6 +226,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I is(LOOP_CONV_LD_CONFIG_ADDRS){ when(!pause_req || unlock) { dram_base_addr := cmd.bits.rs1 + padding := cmd.bits.rs2(32) out_channels := cmd.bits.rs2(31, 16) in_channels := cmd.bits.rs2(15, 0) //can code more diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 58037c646..7ceecbf0e 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -370,14 +370,14 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf dontTouch(pop_count_packed_deps) dontTouch(min_pop_count) - val cycles_since_issue = RegInit(0.U(16.W)) + val cycles_since_issue = RegInit(0.U(20.W)) when (io.issue.ld.fire() || io.issue.st.fire() || io.issue.ex.fire() || !io.busy) { cycles_since_issue := 0.U }.elsewhen(io.busy) { cycles_since_issue := cycles_since_issue + 1.U } - assert(cycles_since_issue < 10000.U, "pipeline stall") + assert(cycles_since_issue < 100000.U, "pipeline stall") for (e <- entries) { dontTouch(e.bits.allocated_at) From 91f7283467ba1fa56aa46b5fa7b3788310f615fd Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 15 Mar 2021 11:41:26 -0700 Subject: [PATCH 056/123] sw padding support for conv fsm --- src/main/scala/gemmini/LoopConv.scala | 28 ++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index a14484aec..bf5e4d3dc 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -178,6 +178,7 @@ class LoopConvLdInputReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val addr_start = UInt(log2Up(max_acc_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) + val dram_padding = Bool() } class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, @@ -219,10 +220,11 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw val irow_padded = irow +& upad.zext() val icol_padded = icol +& lpad.zext() val is_zeros = irow < 0.S || irow >= irows_unpadded.zext() || icol < 0.S || icol >= icols_unpadded.zext() + val ich_stride = Mux(req.dram_padding, in_channels + block_size.U * max_block_len.U, in_channels) // Addresses val dram_addr = Mux(is_zeros, 0.U, - req.dram_addr +& (((b * in_dim * in_dim +& irow*in_dim +& icol) * in_channels +& ich) * (input_w/8).U).asUInt()) + req.dram_addr +& (((b * in_dim * in_dim +& irow*in_dim +& icol) * ich_stride +& ich) * (input_w/8).U).asUInt()) val spad_addr = req.addr_start.zext() +& (ich / block_size.S) * batches * irows * icols +& b * irows * icols +& irow_padded * icols +& icol_padded // Sizes @@ -299,6 +301,7 @@ class LoopConvLdWeightReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val addr_end = UInt(log2Up(max_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) + val dram_padding = Bool() } class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, @@ -338,8 +341,10 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) val kch = Reg(UInt(large_iterator_bitwidth.W)) + val och_stride = Mux(req.dram_padding, out_channels + block_size.U * max_block_len.U, out_channels) + // Addresses - val dram_addr = req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channels +& och) * (input_w/8).U + val dram_addr = req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * och_stride +& och) * (input_w/8).U val spad_addr = addr_start + (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch // Sizes @@ -548,9 +553,10 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val dram_addr = UInt(coreMaxAddrBits.W) val no_pool = Bool() val loop_id = UInt(log2Up(concurrent_loops).W) + val dram_padding = Bool() } -class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { +class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow val io = IO(new Bundle { @@ -587,8 +593,9 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val ocol = Reg(UInt(small_iterator_bitwidth.W)) val och = Reg(UInt(large_iterator_bitwidth.W)) + val och_stride = Mux(req.dram_padding, out_channels + block_size.U * max_block_len.U, out_channels) // Addresses - val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * out_channels + och) * (input_w/8).U + val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol // Sizes @@ -651,6 +658,8 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_bias = Bool() val no_pool = Bool() + val dram_ich_padding = Bool() + val dram_och_padding = Bool() val configured = Bool() @@ -754,7 +763,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I val ld_input = Module(new LoopConvLdInput(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) val ld_weights = Module(new LoopConvLdWeight(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) val ex = Module(new LoopConvExecute(block_size, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, max_acc_addr, concurrent_loops)) - val st = Module(new LoopConvSt(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, input_w, concurrent_loops)) + val st = Module(new LoopConvSt(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, input_w, max_block_len, concurrent_loops)) // Create command queue val cmd = Queue(io.in) @@ -870,6 +879,9 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.no_pool := cmd.bits.rs2(0) + loop_being_configured.dram_ich_padding := cmd.bits.rs2(1) + loop_being_configured.dram_och_padding := cmd.bits.rs2(2) + loop_being_configured.configured := true.B } } @@ -889,6 +901,8 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_bias.io.req.bits.dram_addr := loop_requesting_ld_bias.bias_dram_addr ld_bias.io.req.bits.no_bias := loop_requesting_ld_bias.no_bias ld_bias.io.req.bits.loop_id := loop_requesting_ld_bias_id + ld_bias.io.req.bits.dram_padding := loop_requesting_ld_bias.dram_och_padding + ld_bias.io.req.valid := !loop_requesting_ld_bias.ld_bias_started && loop_requesting_ld_bias.configured @@ -910,6 +924,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_input.io.req.bits.addr_start := loop_requesting_ld_input.a_addr_start ld_input.io.req.bits.dram_addr := loop_requesting_ld_input.input_dram_addr ld_input.io.req.bits.loop_id := loop_requesting_ld_input_id + ld_input.io.req.bits.dram_padding := loop_requesting_ld_input.dram_ich_padding ld_input.io.req.valid := !loop_requesting_ld_input.ld_input_started && loop_requesting_ld_input.configured @@ -926,6 +941,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_weights.io.req.bits.addr_end := loop_requesting_ld_weights.b_addr_end ld_weights.io.req.bits.dram_addr := loop_requesting_ld_weights.weights_dram_addr ld_weights.io.req.bits.loop_id := loop_requesting_ld_weights_id + ld_weights.io.req.bits.dram_padding := loop_requesting_ld_weights.dram_och_padding ld_weights.io.req.valid := !loop_requesting_ld_weights.ld_weights_started && loop_requesting_ld_weights.configured @@ -965,6 +981,8 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr st.io.req.bits.no_pool := loop_requesting_st.no_pool st.io.req.bits.loop_id := loop_requesting_st_id + st.io.req.bits.dram_padding := loop_requesting_st.dram_och_padding + st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured From 98537656a20ad069a768c07c55c1b6235b8212d1 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 15 Mar 2021 11:48:49 -0700 Subject: [PATCH 057/123] fixing bugs --- src/main/scala/gemmini/LoopConv.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index bf5e4d3dc..142f9d49a 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -901,7 +901,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_bias.io.req.bits.dram_addr := loop_requesting_ld_bias.bias_dram_addr ld_bias.io.req.bits.no_bias := loop_requesting_ld_bias.no_bias ld_bias.io.req.bits.loop_id := loop_requesting_ld_bias_id - ld_bias.io.req.bits.dram_padding := loop_requesting_ld_bias.dram_och_padding ld_bias.io.req.valid := !loop_requesting_ld_bias.ld_bias_started && loop_requesting_ld_bias.configured From 9ec4e50bf5c4e024e0e7e154350c5111a7a5ec13 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Mon, 15 Mar 2021 15:00:23 -0700 Subject: [PATCH 058/123] Fix garbage writes locking up scratchpad pipeline --- src/main/scala/gemmini/Scratchpad.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 7def60ad6..f4812cc62 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -554,6 +554,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, bio.read.resp.valid && write_issue_q.io.enq.ready && write_scale_q.io.deq.bits.laddr.is_acc_addr && + !write_scale_q.io.deq.bits.laddr.is_garbage() && write_scale_q.io.deq.bits.laddr.acc_bank() === i.U) { write_scale_q.io.deq.ready := true.B acc_scale_unit.io.in.valid := true.B From 144c8808cda454e546c688a591879dd7f1950b69 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 15 Mar 2021 19:22:06 -0700 Subject: [PATCH 059/123] stride bug --- src/main/scala/gemmini/LoopConv.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 142f9d49a..f6c9b807f 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -242,7 +242,7 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (req.derived_params.input_spad_stride << 16.U).asUInt() | (0.U << 3).asUInt() | 1.U - config_cmd.rs2 := in_channels * (input_w/8).U + config_cmd.rs2 := ich_stride * (input_w/8).U val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare @@ -356,7 +356,7 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (req.derived_params.weight_spad_stride << 16.U).asUInt() | (1.U << 3).asUInt() | 1.U - config_cmd.rs2 := out_channels * (input_w/8).U + config_cmd.rs2 := och_stride * (input_w/8).U val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare From 929ff929012c3163cd126535a4ecec4ea1e3151c Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 15 Mar 2021 19:23:36 -0700 Subject: [PATCH 060/123] stride bug --- src/main/scala/gemmini/LoopLoader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 08ff2077d..ffc61b2f3 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -133,7 +133,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U).asUInt() | (weight_spad_stride << 16.U).asUInt() | (1.U << 3).asUInt() | 1.U - config_cmd.rs2 := out_channels * (input_w/8).U + config_cmd.rs2 := out_channel_stride * (input_w/8).U //for conv val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare From d9237b5a6cbdaa0e33ba8f50797d945313be3ce7 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 17 Mar 2021 10:26:14 +0400 Subject: [PATCH 061/123] Bump gemmini-rocc-tests (#81) --- software/gemmini-rocc-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 463e3eebd..d68fe69ce 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 463e3eebdfd96bb2874a42556ce0337688ef817a +Subproject commit d68fe69ce930dd18bf62ad28ab3015ef5087177d From d2420af218a40f56c7f27296cca39451244aa9c4 Mon Sep 17 00:00:00 2001 From: SeahK Date: Wed, 17 Mar 2021 12:01:02 -0700 Subject: [PATCH 062/123] added och division --- src/main/scala/gemmini/LoopConv.scala | 12 ++++++++++-- src/main/scala/gemmini/LoopLoader.scala | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index f6c9b807f..ce28b1140 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -302,6 +302,7 @@ class LoopConvLdWeightReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) val dram_padding = Bool() + val dram_stride_divide = UInt(4.W) } class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, @@ -341,7 +342,8 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) val kch = Reg(UInt(large_iterator_bitwidth.W)) - val och_stride = Mux(req.dram_padding, out_channels + block_size.U * max_block_len.U, out_channels) + val total_out_channels = out_channels * req.dram_stride_divide + val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) // Addresses val dram_addr = req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * och_stride +& och) * (input_w/8).U @@ -554,6 +556,7 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val no_pool = Bool() val loop_id = UInt(log2Up(concurrent_loops).W) val dram_padding = Bool() + val dram_stride_divide = UInt(4.W) } class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { @@ -593,7 +596,8 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val ocol = Reg(UInt(small_iterator_bitwidth.W)) val och = Reg(UInt(large_iterator_bitwidth.W)) - val och_stride = Mux(req.dram_padding, out_channels + block_size.U * max_block_len.U, out_channels) + val total_out_channels = out_channels * req.dram_stride_divide + val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) // Addresses val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol @@ -660,6 +664,7 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_pool = Bool() val dram_ich_padding = Bool() val dram_och_padding = Bool() + val dram_och_divide = UInt(4.W) val configured = Bool() @@ -881,6 +886,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.dram_ich_padding := cmd.bits.rs2(1) loop_being_configured.dram_och_padding := cmd.bits.rs2(2) + loop_being_configured.dram_och_divide := cmd.bits.rs2(6,3) loop_being_configured.configured := true.B } @@ -941,6 +947,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_weights.io.req.bits.dram_addr := loop_requesting_ld_weights.weights_dram_addr ld_weights.io.req.bits.loop_id := loop_requesting_ld_weights_id ld_weights.io.req.bits.dram_padding := loop_requesting_ld_weights.dram_och_padding + ld_weights.io.req.bits.dram_stride_divide := loop_requesting_ld_weights.dram_och_divide ld_weights.io.req.valid := !loop_requesting_ld_weights.ld_weights_started && loop_requesting_ld_weights.configured @@ -981,6 +988,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.no_pool := loop_requesting_st.no_pool st.io.req.bits.loop_id := loop_requesting_st_id st.io.req.bits.dram_padding := loop_requesting_st.dram_och_padding + st.io.req.bits.dram_stride_divide := loop_requesting_st.dram_och_divide st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index ffc61b2f3..167a97c82 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -87,7 +87,9 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv - val out_channel_stride = Mux(padding, out_channels + max_blocks * block_size.U, out_channels) + val och_divide = RegInit(1.U(4.W)) + val total_out_channel = out_channels * och_divide + val out_channel_stride = Mux(padding, total_out_channel + max_blocks * block_size.U, total_out_channel) val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) val out_channels_per_bank = WireInit(0.U(8.W)) out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) @@ -227,6 +229,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(!pause_req || unlock) { dram_base_addr := cmd.bits.rs1 padding := cmd.bits.rs2(32) + och_divide := cmd.bits.rs2(33) out_channels := cmd.bits.rs2(31, 16) in_channels := cmd.bits.rs2(15, 0) //can code more From 9ce629d8c642bd53c3ceb01a949a160c064f0f81 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Mar 2021 16:32:34 -0700 Subject: [PATCH 063/123] ROB stall tracker should track io.completed as well --- src/main/scala/gemmini/ROB.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 58037c646..c002da487 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -372,7 +372,7 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val cycles_since_issue = RegInit(0.U(16.W)) - when (io.issue.ld.fire() || io.issue.st.fire() || io.issue.ex.fire() || !io.busy) { + when (io.issue.ld.fire() || io.issue.st.fire() || io.issue.ex.fire() || !io.busy || io.completed.fire()) { cycles_since_issue := 0.U }.elsewhen(io.busy) { cycles_since_issue := cycles_since_issue + 1.U From 2622d0742ae3646e6e7d284c067881d9c8adffc3 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Mar 2021 16:33:01 -0700 Subject: [PATCH 064/123] Use all singleported accumulator write queue entries --- src/main/scala/gemmini/AccumulatorMem.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 9adfdd47a..1939c6243 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -158,7 +158,7 @@ class AccumulatorMem[T <: Data, U <: Data]( } when (w_buf_valid && isThisBank(waddr_buf)) { - assert(!RegNext(((w_q_tail.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_)))) + assert(!((w_q_tail.asBools zip w_q.map(_.valid)).map({ case (h,v) => h && v }).reduce(_||_))) w_q_tail := w_q_tail << 1 | w_q_tail(nEntries-1) for (i <- 0 until nEntries) { when (w_q_tail(i)) { From 54449a7699b417053e4b5e1a68c1c0d164bc5a73 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Mar 2021 16:33:46 -0700 Subject: [PATCH 065/123] Revert DefaultConfig to original behavior --- src/main/scala/gemmini/Configs.scala | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index c57911b84..c60064976 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -53,8 +53,8 @@ object GemminiConfigs { sp_banks = 4, sp_singleported = true, acc_banks = 2, - acc_singleported = true, - num_acc_sub_banks = 2, + acc_singleported = false, + num_acc_sub_banks = -1, sp_capacity = CapacityInKilobytes(256), shifter_banks = 1, // TODO add separate parameters for left and up shifter banks dataflow = Dataflow.BOTH, @@ -106,7 +106,7 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 5, Float(8, 24), 4, + 4, Float(8, 24), 4, identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (elem_t)y);})" )), @@ -147,7 +147,7 @@ object GemminiConfigs { Mux(overflow, sat, rec_fn_to_in.io.out.asTypeOf(t)) }, - 5, Float(8, 24), 4, + 1, Float(8, 24), -1, // TODO pipelining should be 5 identity = "1.0", c_str = "({float y = ROUND_NEAR_EVEN((x) * (scale)); y > INT8_MAX ? INT8_MAX : (y < INT8_MIN ? INT8_MIN : (acc_t)y);})" ), @@ -163,8 +163,14 @@ object GemminiConfigs { ex_write_to_acc = true ) - val chipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(64), acc_capacity=CapacityInKilobytes(32), dataflow=Dataflow.WS) - val largeChipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), dataflow=Dataflow.WS, + val chipConfig = defaultConfig.copy(sp_capacity=CapacityInKilobytes(64), acc_capacity=CapacityInKilobytes(32), dataflow=Dataflow.WS, + acc_scale_args=defaultConfig.acc_scale_args.copy(latency=4), + acc_singleported=true, + num_acc_sub_banks=2, + ex_read_from_acc=false, + ex_write_to_spad=false + ) + val largeChipConfig = chipConfig.copy(sp_capacity=CapacityInKilobytes(128), acc_capacity=CapacityInKilobytes(64), meshRows=32, meshColumns=32 ) From bccb2f61b3218d3f5c4440ed02c8c3a389945b43 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Wed, 17 Mar 2021 18:27:51 -0700 Subject: [PATCH 066/123] Fix full-data for un-multiplexed acc-scale --- src/main/scala/gemmini/AccumulatorScale.scala | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorScale.scala b/src/main/scala/gemmini/AccumulatorScale.scala index 20e7fc41d..304126fe0 100644 --- a/src/main/scala/gemmini/AccumulatorScale.scala +++ b/src/main/scala/gemmini/AccumulatorScale.scala @@ -5,6 +5,13 @@ import chisel3.util._ import Util._ +class AccumulatorReadRespWithFullData[T <: Data: Arithmetic, U <: Data](fullDataType: Vec[Vec[T]], scale_t: U, shift_width: Int) extends Bundle { + val resp = new AccumulatorReadResp(fullDataType, scale_t, shift_width) + val full_data = fullDataType.cloneType + override def cloneType: this.type = new AccumulatorReadRespWithFullData(fullDataType.cloneType, scale_t, shift_width).asInstanceOf[this.type] +} + + class AccumulatorScaleResp[T <: Data: Arithmetic](fullDataType: Vec[Vec[T]], rDataType: Vec[Vec[T]]) extends Bundle { val full_data = fullDataType.cloneType val data = rDataType.cloneType @@ -75,28 +82,34 @@ class AccumulatorScale[T <: Data: Arithmetic, U <: Data]( val acc_scale_latency = scale_args.latency if (num_scale_units == -1) { - val pipe_out = Pipeline(io.in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: AccumulatorReadResp[T,U]) => x) :+ { - x: AccumulatorReadResp[T,U] => - val activated_rdata = VecInit(x.data.map(v => VecInit(v.map { e => + val in = Wire(Decoupled(new AccumulatorReadRespWithFullData(fullDataType, scale_t, shift_width)(ev))) + in.valid := io.in.valid + io.in.ready := in.ready + in.bits.resp := io.in.bits + in.bits.full_data := io.in.bits.data + + val pipe_out = Pipeline(in, acc_scale_latency, Seq.fill(acc_scale_latency)((x: AccumulatorReadRespWithFullData[T,U]) => x) :+ { + x: AccumulatorReadRespWithFullData[T,U] => + val activated_rdata = VecInit(x.resp.data.map(v => VecInit(v.map { e => // val e_scaled = e >> x.shiftls - val e_scaled = scale_args.scale_func(e, x.scale) + val e_scaled = scale_args.scale_func(e, x.resp.scale) val e_clipped = e_scaled.clippedToWidthOf(rDataType.head.head) val e_act = MuxCase(e_clipped, Seq( - (x.act === Activation.RELU) -> e_clipped.relu, - (x.act === Activation.RELU6) -> e_clipped.relu6(x.relu6_shift))) + (x.resp.act === Activation.RELU) -> e_clipped.relu, + (x.resp.act === Activation.RELU6) -> e_clipped.relu6(x.resp.relu6_shift))) e_act }))) val result = WireInit(x) - result.data := activated_rdata + result.resp.data := activated_rdata result }) out.valid := pipe_out.valid pipe_out.ready := out.ready - out.bits.full_data := pipe_out.bits.data - out.bits.data := pipe_out.bits.data - out.bits.fromDMA := pipe_out.bits.fromDMA - out.bits.acc_bank_id := pipe_out.bits.acc_bank_id + out.bits.full_data := pipe_out.bits.full_data + out.bits.data := pipe_out.bits.resp.data + out.bits.fromDMA := pipe_out.bits.resp.fromDMA + out.bits.acc_bank_id := pipe_out.bits.resp.acc_bank_id } else { val width = io.in.bits.data.size * io.in.bits.data(0).size val nEntries = 3 From 8e262b432a76d2e030ca95b711c8c79707d2f333 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Fri, 19 Mar 2021 02:47:39 +0400 Subject: [PATCH 067/123] Double-buffer conv layers that have pooling (#82) * Fix DMA perf bug * Update ROB so that fence is not required for pooling mvout * Add pooling support for conv-fsm * Fix pooling conv-fsm bug * Update SPIKE.hash * Bump gemmini-rocc-tests --- SPIKE.hash | 2 +- software/gemmini-rocc-tests | 2 +- src/main/scala/gemmini/DMA.scala | 9 ++-- src/main/scala/gemmini/LocalAddr.scala | 2 +- src/main/scala/gemmini/LoopConv.scala | 73 ++++++++++++++++++++------ src/main/scala/gemmini/ROB.scala | 19 ++++++- 6 files changed, 84 insertions(+), 23 deletions(-) diff --git a/SPIKE.hash b/SPIKE.hash index ccafc0dbd..fe452b870 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -9b0082a416a4f1967fda434c7129953fad77b2af +a9eabb91de49495f9a231c119d81e6af491549fd diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index d68fe69ce..3d356f347 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit d68fe69ce930dd18bf62ad28ab3015ef5087177d +Subproject commit 3d356f347cc7b6aaa33cd71cce79af6a515e48f3 diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 7af1751da..16a05dfbd 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -361,7 +361,7 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val vaddr = UInt(vaddrBits.W) val is_full = Bool() - val bytes_written = UInt(log2Up(dataBytes+1).W) + val bytes_written = UInt(log2Up(maxBytes+1).W) val bytes_written_per_beat = Vec(maxBeatsPerReq, UInt(log2Up(beatBytes+1).W)) def total_beats(dummy: Int = 0) = Mux(size < beatBytes.U, 1.U, size / beatBytes.U) @@ -370,8 +370,8 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: val smallest_write_size = aligned_to max beatBytes val write_sizes = (smallest_write_size to maxBytes by aligned_to). filter(s => isPow2(s)). - filter(s => s % beatBytes == 0). - filter(s => s <= dataBytes*2 || s == smallest_write_size) + filter(s => s % beatBytes == 0) /*. + filter(s => s <= dataBytes*2 || s == smallest_write_size)*/ val write_packets = write_sizes.map { s => val lg_s = log2Ceil(s) val vaddr_aligned_to_size = if (s == 1) vaddr else Cat(vaddr(vaddrBits-1, lg_s), 0.U(lg_s.W)) @@ -416,6 +416,9 @@ class StreamWriter[T <: Data: Arithmetic](nXacts: Int, beatBits: Int, maxBytes: } val write_packet = RegEnableThru(best_write_packet, state === s_writing_new_block) + for (wp <- write_packets) + dontTouch(wp) + val write_size = write_packet.size val lg_write_size = write_packet.lg_size val write_beats = write_packet.total_beats() diff --git a/src/main/scala/gemmini/LocalAddr.scala b/src/main/scala/gemmini/LocalAddr.scala index 6520b7f9c..b003fd7b4 100644 --- a/src/main/scala/gemmini/LocalAddr.scala +++ b/src/main/scala/gemmini/LocalAddr.scala @@ -14,7 +14,7 @@ class LocalAddr(sp_banks: Int, sp_bank_entries: Int, acc_banks: Int, acc_bank_en private val spBankRowBits = log2Up(sp_bank_entries) private val accBankBits = log2Up(acc_banks) - private val accBankRowBits = log2Up(acc_bank_entries) + val accBankRowBits = log2Up(acc_bank_entries) val is_acc_addr = Bool() val accumulate = Bool() diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 83f34fcf9..51c5d84ce 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -562,7 +562,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: }) object State extends ChiselEnum { - val idle, st = Value + val idle, st, pre_pool_config, pool, post_pool_config = Value } import State._ val state = RegInit(idle) @@ -575,7 +575,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val acc_addr_start = (BigInt(1) << 31).U | req.addr_start // Derived parameters - val skip = !(req.no_pool && (req.dram_addr =/= 0.U)) + val skip = req.dram_addr === 0.U // Iterators val b = Reg(UInt(large_iterator_bitwidth.W)) @@ -587,10 +587,15 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * out_channels + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol + val pool_dram_addr = req.dram_addr + ((b * pool_out_dim * pool_out_dim) * out_channels + och) * (input_w/8).U + val pool_spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols + // Sizes val I = Mux(ocols - ocol > block_size.U, block_size.U, ocols - ocol) val J = Mux(ochs - och > block_size.U, block_size.U, ochs - och) + val channels = J + // Commands val mvout_cmd = Wire(new RoCCCommand) mvout_cmd := DontCare @@ -598,36 +603,72 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: mvout_cmd.rs1 := dram_addr mvout_cmd.rs2 := (I << 48.U) | (J << 32.U) | spad_addr + val pre_pool_config_cmd = Wire(new RoCCCommand) + pre_pool_config_cmd := DontCare + pre_pool_config_cmd.inst.funct := CONFIG_CMD + pre_pool_config_cmd.rs1 := (ocols << 56) | (orows << 48) | (pocols << 40) | (porows << 32) | (pool_out_dim << 24) | + (plpad << 10) | (pupad << 8) | (pool_size << 6) | (pool_stride << 4) | // TODO magic numbers + CONFIG_STORE + pre_pool_config_cmd.rs2 := out_channels * (input_w / 8).U + + val post_pool_config_cmd = Wire(new RoCCCommand) + post_pool_config_cmd := DontCare + post_pool_config_cmd.inst.funct := CONFIG_CMD + post_pool_config_cmd.rs1 := CONFIG_STORE + post_pool_config_cmd.rs2 := out_channels * (input_w / 8).U + + val pool_cmd = Wire(new RoCCCommand) + pool_cmd := DontCare + pool_cmd.inst.funct := STORE_CMD + pool_cmd.rs1 := pool_dram_addr + pool_cmd.rs2 := (channels << 32.U) | pool_spad_addr + // Inputs and outputs io.req.ready := state === idle io.idle := state === idle io.loop_id := req.loop_id io.cmd.valid := state =/= idle && !io.rob_overloaded && !skip && io.ex_completed - io.cmd.bits := mvout_cmd + io.cmd.bits := MuxLookup(state.asUInt, mvout_cmd, Seq(pre_pool_config.asUInt -> pre_pool_config_cmd, + pool.asUInt -> pool_cmd, post_pool_config.asUInt -> post_pool_config_cmd)) // Sending outputs when (skip) { state := idle }.elsewhen(io.cmd.fire()) { - val next_och = floorAdd(och, block_size.U, ochs) - val next_ocol = floorAdd(ocol, block_size.U, ocols, next_och === 0.U) - val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U && next_och === 0.U) - val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && next_ocol === 0.U && next_och === 0.U) - - och := next_och - ocol := next_ocol - orow := next_orow - b := next_b - - state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U, - idle, st) + when (req.no_pool) { + val next_och = floorAdd(och, block_size.U, ochs) + val next_ocol = floorAdd(ocol, block_size.U, ocols, next_och === 0.U) + val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U && next_och === 0.U) + val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && next_ocol === 0.U && next_och === 0.U) + + och := next_och + ocol := next_ocol + orow := next_orow + b := next_b + + state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U, + idle, st) + }.elsewhen(state === pre_pool_config) { + state := pool + }.elsewhen(state === post_pool_config) { + state := idle + }.otherwise { + val next_och = floorAdd(och, block_size.U, ochs) + val next_b = floorAdd(b, 1.U, batches, next_och === 0.U) + + och := next_och + b := next_b + + state := Mux(next_b === 0.U && next_och === 0.U, + post_pool_config, pool) + } } // Accepting requests when (io.req.fire()) { req := io.req.bits - state := st + state := Mux(io.req.bits.no_pool, st, pre_pool_config) b := 0.U orow := 0.U diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index c002da487..5979aaaf7 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -105,9 +105,12 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf io.alloc.ready := !full val last_allocated = Reg(UInt(log2Up(rob_entries).W)) + + // Config values set by programmer val a_stride = Reg(UInt(16.W)) // TODO magic numbers // TODO we also need to check the transpose to see how many rows we're reading val ld_block_strides = Reg(Vec(load_states, UInt(block_stride_bits.W))) val st_block_stride = block_rows.U + val pooling_is_enabled = Reg(Bool()) val new_entry = Wire(new Entry) new_entry := DontCare @@ -168,6 +171,17 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val compute_rows = cmd.rs2(48 + log2Up(block_rows + 1) - 1, 48) new_entry.op2.bits.end := new_entry.op2.bits.start + compute_rows new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(compute_rows)._2 + }.elsewhen (pooling_is_enabled) { + // If pooling is enabled, then we assume that this command simply mvouts everything in this accumulator bank from + // start to the end of the bank + val acc_bank = new_entry.op2.bits.start.acc_bank() + + val next_bank_addr = WireInit(0.U.asTypeOf(local_addr_t)) + next_bank_addr.is_acc_addr := true.B + next_bank_addr.data := (acc_bank + 1.U) << local_addr_t.accBankRowBits + + new_entry.op2.bits.end := next_bank_addr + new_entry.op2.bits.wraps_around := next_bank_addr.acc_bank() === 0.U }.otherwise { val block_stride = st_block_stride @@ -178,7 +192,7 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val total_mvout_rows = ((mvout_mats - 1.U) * block_stride) + mvout_rows new_entry.op2.bits.end := new_entry.op2.bits.start + total_mvout_rows - new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(total_mvout_rows)._2 + new_entry.op2.bits.wraps_around := pooling_is_enabled || new_entry.op2.bits.start.add_with_overflow(total_mvout_rows)._2 } new_entry.dst.valid := funct === PRELOAD_CMD || funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD @@ -303,6 +317,9 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val id = new_entry.cmd.rs1(4,3) // TODO magic numbers val block_stride = new_entry.cmd.rs1(31, 16) // TODO magic numbers ld_block_strides(id) := block_stride + }.elsewhen(new_entry.is_config && new_entry.q === stq) { + val pool_stride = new_entry.cmd.rs1(5, 4) // TODO magic numbers + pooling_is_enabled := pool_stride =/= 0.U } } From cc299e7cebea91e23ff339e967af8069ebc95cca Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Sat, 20 Mar 2021 03:54:53 +0400 Subject: [PATCH 068/123] Only flush spatial array in OS mode (#85) Resnet50 utilization after this change: 41.7% --- software/gemmini-rocc-tests | 2 +- .../scala/gemmini/ExecuteController.scala | 144 +++++++++--------- src/main/scala/gemmini/ROB.scala | 4 +- 3 files changed, 73 insertions(+), 77 deletions(-) diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 3d356f347..f95d9d5ed 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 3d356f347cc7b6aaa33cd71cce79af6a515e48f3 +Subproject commit f95d9d5ed9609b8647352e11a2b7ef9529f34db7 diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index 300dab167..a1cb3230c 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -544,55 +544,51 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In io.completed := cmd.bits(0).rob_id cmd.pop := 1.U - } - // Preload - .elsewhen(DoPreloads(0) && cmd.valid(1) && (raw_hazards_are_impossible.B || !raw_hazard_pre)) { - perform_single_preload := true.B - performing_single_preload := true.B - - //start_inputting_a := current_dataflow === Dataflow.OS.id.U - //start_inputting_d := true.B + // Preload + .elsewhen(DoPreloads(0) && cmd.valid(1) && (raw_hazards_are_impossible.B || !raw_hazard_pre)) { + perform_single_preload := true.B + performing_single_preload := true.B - start_inputting_a := a_should_be_fed_into_transposer - start_inputting_b := b_should_be_fed_into_transposer - start_inputting_d := true.B + //start_inputting_a := current_dataflow === Dataflow.OS.id.U + //start_inputting_d := true.B - control_state := compute - } + start_inputting_a := a_should_be_fed_into_transposer + start_inputting_b := b_should_be_fed_into_transposer + start_inputting_d := true.B - // Overlap compute and preload - .elsewhen(DoComputes(0) && cmd.valid(1) && DoPreloads(1) && (raw_hazards_are_impossible.B || (cmd.valid(2) && !raw_hazard_mulpre))) { - perform_mul_pre := true.B - performing_mul_pre := true.B + control_state := compute + } - start_inputting_a := true.B - start_inputting_b := true.B - start_inputting_d := true.B + // Overlap compute and preload + .elsewhen(DoComputes(0) && cmd.valid(1) && DoPreloads(1) && (raw_hazards_are_impossible.B || (cmd.valid(2) && !raw_hazard_mulpre))) { + perform_mul_pre := true.B + performing_mul_pre := true.B - control_state := compute - } + start_inputting_a := true.B + start_inputting_b := true.B + start_inputting_d := true.B - // Single mul - .elsewhen(DoComputes(0)) { - perform_single_mul := true.B - performing_single_mul := true.B + control_state := compute + } - //start_inputting_a := current_dataflow === Dataflow.WS.id.U - //start_inputting_b := true.B + // Single mul + .elsewhen(DoComputes(0)) { + perform_single_mul := true.B + performing_single_mul := true.B - start_inputting_a := !a_should_be_fed_into_transposer - start_inputting_b := !b_should_be_fed_into_transposer - start_inputting_b := true.B + start_inputting_a := !a_should_be_fed_into_transposer + start_inputting_b := !b_should_be_fed_into_transposer + start_inputting_b := true.B - control_state := compute - } + control_state := compute + } - // Flush - .elsewhen(matmul_in_progress) { - control_state := flush - } - }.elsewhen(matmul_in_progress) { + // Flush + .elsewhen(matmul_in_progress && (current_dataflow === Dataflow.OS.id.U || DoConfig)) { + control_state := flush + } + }.elsewhen(matmul_in_progress && current_dataflow === Dataflow.OS.id.U) { // TODO code duplication control_state := flush } @@ -616,50 +612,50 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } } } - // Overlapping - .elsewhen(perform_mul_pre) { - start_inputting_a := true.B - start_inputting_b := true.B - start_inputting_d := true.B + // Overlapping + .elsewhen(perform_mul_pre) { + start_inputting_a := true.B + start_inputting_b := true.B + start_inputting_d := true.B - when(about_to_fire_all_rows) { - cmd.pop := 2.U - control_state := waiting_for_cmd + when(about_to_fire_all_rows) { + cmd.pop := 2.U + control_state := waiting_for_cmd - pending_completed_rob_ids(0) := cmd.bits(0).rob_id - pending_completed_rob_ids(1).valid := cmd.bits(1).rob_id.valid && c_address_rs2.is_garbage() - pending_completed_rob_ids(1).bits := cmd.bits(1).rob_id.bits + pending_completed_rob_ids(0) := cmd.bits(0).rob_id + pending_completed_rob_ids(1).valid := cmd.bits(1).rob_id.valid && c_address_rs2.is_garbage() + pending_completed_rob_ids(1).bits := cmd.bits(1).rob_id.bits - when(current_dataflow === Dataflow.OS.id.U) { - in_prop_flush := !rs2s(1).asTypeOf(local_addr_t).is_garbage() - } + when(current_dataflow === Dataflow.OS.id.U) { + in_prop_flush := !rs2s(1).asTypeOf(local_addr_t).is_garbage() } } - // Only compute - .elsewhen(perform_single_mul) { - start_inputting_a := !a_should_be_fed_into_transposer - start_inputting_b := !b_should_be_fed_into_transposer + } + // Only compute + .elsewhen(perform_single_mul) { + start_inputting_a := !a_should_be_fed_into_transposer + start_inputting_b := !b_should_be_fed_into_transposer - when(about_to_fire_all_rows) { - cmd.pop := 1.U - control_state := waiting_for_cmd + when(about_to_fire_all_rows) { + cmd.pop := 1.U + control_state := waiting_for_cmd - pending_completed_rob_ids(0) := cmd.bits(0).rob_id - } - } - } - is(flush) { - when(mesh.io.flush.fire()) { - control_state := flushing - } - } - is(flushing) { - when(mesh.io.flush.ready) { - // TODO we waste a cycle here if it was better to continue with the flush - control_state := waiting_for_cmd - } - } + pending_completed_rob_ids(0) := cmd.bits(0).rob_id } + } + } + is(flush) { + when(mesh.io.flush.fire()) { + control_state := flushing + } + } + is(flushing) { + when(mesh.io.flush.ready) { + // TODO we waste a cycle here if it was better to continue with the flush + control_state := waiting_for_cmd + } + } + } // Computing logic diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 5979aaaf7..4cb15b09f 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -206,8 +206,8 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf (new_entry.cmd.inst.funct === LOAD3_CMD) -> 2.U)) val block_stride = ld_block_strides(id) - val mvin_cols = cmd.rs2(spAddrBits + mvin_cols_bits - 1, spAddrBits) - val mvin_rows = cmd.rs2(spAddrBits + mvin_cols_bits + mvin_rows_bits - 1, spAddrBits + mvin_cols_bits) + val mvin_cols = cmd.rs2(32 + mvin_cols_bits - 1, 32) + val mvin_rows = cmd.rs2(48 + mvin_rows_bits - 1, 48) val mvin_mats = mvin_cols / block_cols.U + (mvin_cols % block_cols.U =/= 0.U) val total_mvin_rows = ((mvin_mats - 1.U) * block_stride) + mvin_rows From 2f1b2422bd18414491671793bb00824a1af639d2 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Sat, 20 Mar 2021 06:25:10 +0400 Subject: [PATCH 069/123] Bump gemmini-rocc-tests (#88) --- software/gemmini-rocc-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index f95d9d5ed..7b9e82baf 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit f95d9d5ed9609b8647352e11a2b7ef9529f34db7 +Subproject commit 7b9e82baf0c14df36f97854310c8ba354838d9ab From 2620d30f4ce5326afb80f1cda6f88889de0063b8 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Sat, 20 Mar 2021 11:04:55 +0400 Subject: [PATCH 070/123] Make the internal conv-fsm truly weight-stationary (#86) Resnet50 utilization: 43% --- SPIKE.hash | 2 +- src/main/scala/gemmini/LoopConv.scala | 46 +++++++++++++++------------ 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/SPIKE.hash b/SPIKE.hash index fe452b870..7a511bbcf 100644 --- a/SPIKE.hash +++ b/SPIKE.hash @@ -1 +1 @@ -a9eabb91de49495f9a231c119d81e6af491549fd +dbd3b0874dde4eead6b8d0c4195ee8b41dd113fc diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 51c5d84ce..0bcfda269 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -445,13 +445,13 @@ class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_itera val c_addr_start = (BigInt(3) << 30).U | req.c_addr_start // Iterators - val b = Reg(UInt(large_iterator_bitwidth.W)) - val orow = Reg(UInt(small_iterator_bitwidth.W)) - val ocol = Reg(UInt(small_iterator_bitwidth.W)) val och = Reg(UInt(large_iterator_bitwidth.W)) val krow = Reg(UInt(tiny_iterator_bitwidth.W)) val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) val kch = Reg(UInt(large_iterator_bitwidth.W)) + val b = Reg(UInt(large_iterator_bitwidth.W)) + val orow = Reg(UInt(small_iterator_bitwidth.W)) + val ocol = Reg(UInt(small_iterator_bitwidth.W)) val irow = orow * stride +& krow val icol = ocol * stride +& kcol @@ -462,10 +462,14 @@ class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_itera // Addresses val a_addr = a_addr_start +& (kch / block_size.U) * batches * irows * icols +& b * irows * icols +& irow * icols +& icol - val b_addr = b_addr_start +& (och / block_size.U) * krows * kcols * kchs +& krow * kcols * kchs +& kcol * kchs +& kch val c_addr = Mux(ex_overwrite && krow === 0.U && kcol === 0.U && kch === 0.U, d_addr_start, c_addr_start) +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol + val new_weights = b === 0.U && orow === 0.U && ocol === 0.U + val b_addr = Mux(new_weights, + b_addr_start +& (och / block_size.U) * krows * kcols * kchs +& krow * kcols * kchs +& kcol * kchs +& kch, + GARBAGE_ADDR) + // Commands val pre_cmd = Wire(new RoCCCommand) pre_cmd := DontCare @@ -475,7 +479,7 @@ class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_itera val comp_cmd = Wire(new RoCCCommand()) comp_cmd := DontCare - comp_cmd.inst.funct := COMPUTE_AND_FLIP_CMD + comp_cmd.inst.funct := Mux(new_weights, COMPUTE_AND_FLIP_CMD, COMPUTE_AND_STAY_CMD) comp_cmd.rs1 := (I << 48) | (K << 32) | a_addr comp_cmd.rs2 := (I << 48) | (J << 32) | GARBAGE_ADDR @@ -495,28 +499,28 @@ class LoopConvExecute(block_size: Int, large_iterator_bitwidth: Int, small_itera when (state === pre) { state := comp }.otherwise { - val next_kch = floorAdd(kch, block_size.U, kchs) - val next_kcol = floorAdd(kcol, 1.U, kcols, next_kch === 0.U) - val next_krow = floorAdd(krow, 1.U, krows, next_kcol === 0.U && next_kch === 0.U) - val next_och = floorAdd(och, block_size.U, ochs, - next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) - val next_ocol = floorAdd(ocol, block_size.U, ocols, - next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) - val next_orow = floorAdd(orow, 1.U, orows, - next_ocol === 0.U && next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) - val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && - next_ocol === 0.U && next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U) + val next_ocol = floorAdd(ocol, block_size.U, ocols) + val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U) + val next_b = floorAdd(b, 1.U, batches, next_orow === 0.U && next_ocol === 0.U) + val next_kch = floorAdd(kch, block_size.U, kchs, + next_b === 0.U && next_orow === 0.U && next_ocol === 0.U) + val next_kcol = floorAdd(kcol, 1.U, kcols, + next_kch === 0.U && next_b === 0.U && next_orow === 0.U && next_ocol === 0.U) + val next_krow = floorAdd(krow, 1.U, krows, + next_kcol === 0.U && next_kch === 0.U && next_b === 0.U && next_orow === 0.U && next_ocol === 0.U) + val next_och = floorAdd(och, block_size.U, ochs, next_krow === 0.U && + next_kcol === 0.U && next_kch === 0.U && next_b === 0.U && next_orow === 0.U && next_ocol === 0.U) + ocol := next_ocol + orow := next_orow + b := next_b kch := next_kch kcol := next_kcol krow := next_krow och := next_och - ocol := next_ocol - orow := next_orow - b := next_b - state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && - next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U, + state := Mux(next_och === 0.U && next_krow === 0.U && next_kcol === 0.U && next_kch === 0.U && next_b === 0.U && + next_orow === 0.U && next_ocol === 0.U, idle, pre) } } From 88279c4aa7a2c458832d7fcf4f5a45f0e560a391 Mon Sep 17 00:00:00 2001 From: Jerry Zhao Date: Sat, 20 Mar 2021 01:09:41 -0700 Subject: [PATCH 071/123] Split rob entries based off requested resources for efficiency (#73) * Split rob entries based off requested resources for efficiency * Condense ROB entries into 2 address slots per entry * Set ROB size back to original * Give LoopConv awareness of ROB size --- src/main/scala/gemmini/Configs.scala | 4 +- src/main/scala/gemmini/ConfigsFP.scala | 3 +- src/main/scala/gemmini/Controller.scala | 10 +- src/main/scala/gemmini/DSEConfigs.scala | 4 +- src/main/scala/gemmini/GemminiConfigs.scala | 4 +- src/main/scala/gemmini/ROB.scala | 270 +++++++++++--------- 6 files changed, 171 insertions(+), 124 deletions(-) diff --git a/src/main/scala/gemmini/Configs.scala b/src/main/scala/gemmini/Configs.scala index c60064976..3c9ac6827 100644 --- a/src/main/scala/gemmini/Configs.scala +++ b/src/main/scala/gemmini/Configs.scala @@ -47,7 +47,9 @@ object GemminiConfigs { st_queue_length = 2, ex_queue_length = 8, - rob_entries = 16, + rob_full_entries = 16, + rob_partial_entries = 8, + hasIm2col = false, //declare im2col block sp_banks = 4, diff --git a/src/main/scala/gemmini/ConfigsFP.scala b/src/main/scala/gemmini/ConfigsFP.scala index 946915cac..a7c065f34 100644 --- a/src/main/scala/gemmini/ConfigsFP.scala +++ b/src/main/scala/gemmini/ConfigsFP.scala @@ -24,7 +24,8 @@ object GemminiFPConfigs { st_queue_length = 2, ex_queue_length = 8, - rob_entries = 16, + rob_full_entries = 16, + rob_partial_entries = 8, hasIm2col = false, diff --git a/src/main/scala/gemmini/Controller.scala b/src/main/scala/gemmini/Controller.scala index 2b7c9a823..cc28697d7 100644 --- a/src/main/scala/gemmini/Controller.scala +++ b/src/main/scala/gemmini/Controller.scala @@ -109,18 +109,20 @@ class GemminiModule[T <: Data: Arithmetic, U <: Data, V <: Data] val raw_cmd = Queue(io.cmd) + val max_lds = rob_partial_entries + val max_exs = rob_full_entries + val max_sts = rob_partial_entries / 2 + // TODO replace 4,12,2 with parameters based on ROB size val (conv_cmd, loop_conv_unroller_busy) = LoopConv(raw_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - meshRows*tileRows, coreMaxAddrBits, rob_entries, 4, 12, 2, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, + meshRows*tileRows, coreMaxAddrBits, rob_entries, max_lds, max_exs, max_sts, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) // val (compressed_cmd, compressor_busy) = InstCompressor(unrolled_cmd) // compressed_cmd.ready := false.B // val (unrolled_cmd, loop_matmul_unroller_busy) = LoopMatmul(unrolled_cmd_after_conv, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, - val max_lds = rob_entries * 1 / 4 - val max_exs = rob_entries * 3 / 4 - val max_sts = rob_entries * 1 / 8 + val (loop_cmd, loop_matmul_unroller_busy) = LoopMatmul(conv_cmd, rob.io.ld_utilization, rob.io.st_utilization, rob.io.ex_utilization, meshRows*tileRows, coreMaxAddrBits, rob_entries, max_lds, max_exs, max_sts, sp_banks * sp_bank_entries, acc_banks * acc_bank_entries, inputType.getWidth, accType.getWidth, dma_maxbytes) diff --git a/src/main/scala/gemmini/DSEConfigs.scala b/src/main/scala/gemmini/DSEConfigs.scala index da3b1795b..540fdac6f 100644 --- a/src/main/scala/gemmini/DSEConfigs.scala +++ b/src/main/scala/gemmini/DSEConfigs.scala @@ -21,7 +21,9 @@ object DSEBaseConfig { ld_queue_length = 4, st_queue_length = 2, ex_queue_length = 8, - rob_entries = 8, + rob_full_entries = 8, + rob_partial_entries = 1, + sp_banks = 4, // TODO support one-bank designs acc_banks = 1, acc_singleported = false, diff --git a/src/main/scala/gemmini/GemminiConfigs.scala b/src/main/scala/gemmini/GemminiConfigs.scala index b8df083b1..6ee0c1682 100644 --- a/src/main/scala/gemmini/GemminiConfigs.scala +++ b/src/main/scala/gemmini/GemminiConfigs.scala @@ -23,7 +23,8 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( ld_queue_length: Int, st_queue_length: Int, ex_queue_length: Int, - rob_entries: Int, + rob_full_entries: Int, + rob_partial_entries: Int, sp_banks: Int, // TODO support one-bank designs sp_singleported: Boolean, sp_capacity: GemminiMemCapacity, @@ -116,6 +117,7 @@ case class GemminiArrayConfig[T <: Data : Arithmetic, U <: Data, V <: Data]( //========================================================================== // cisc-gemmini miscellaneous constants (some redundant with above) //========================================================================== + val rob_entries = rob_full_entries + rob_partial_entries val ROB_ENTRIES = rob_entries val LOG2_ROB_ENTRIES = log2Up(rob_entries) diff --git a/src/main/scala/gemmini/ROB.scala b/src/main/scala/gemmini/ROB.scala index 4cb15b09f..784cf879f 100644 --- a/src/main/scala/gemmini/ROB.scala +++ b/src/main/scala/gemmini/ROB.scala @@ -73,9 +73,14 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val is_config = Bool() - val op1 = UDValid(new OpT) - val op2 = UDValid(new OpT) - val dst = UDValid(new OpT) + val opa = UDValid(new OpT) + val opa_is_dst = Bool() + val opb = UDValid(new OpT) + + // val op1 = UDValid(new OpT) + // val op1 = UDValid(new OpT) + // val op2 = UDValid(new OpT) + // val dst = UDValid(new OpT) val issued = Bool() @@ -89,8 +94,10 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf // Debugging signals val allocated_at = UInt(instructions_allocated.getWidth.W) } + val full_entries = Reg(Vec(rob_full_entries, UDValid(new Entry))) + val partial_entries = Reg(Vec(rob_partial_entries, UDValid(new Entry))) - val entries = Reg(Vec(rob_entries, UDValid(new Entry))) + val entries = full_entries ++ partial_entries val empty = !entries.map(_.valid).reduce(_ || _) val full = entries.map(_.valid).reduce(_ && _) @@ -101,10 +108,6 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val solitary_preload = utilization === 1.U && entries.map(e => e.valid && e.bits.cmd.inst.funct === PRELOAD_CMD).reduce(_ || _) io.busy := !empty && !(solitary_preload && io.solitary_preload) - // Read in commands to the buffer - io.alloc.ready := !full - - val last_allocated = Reg(UInt(log2Up(rob_entries).W)) // Config values set by programmer val a_stride = Reg(UInt(16.W)) // TODO magic numbers // TODO we also need to check the transpose to see how many rows we're reading @@ -114,7 +117,11 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val new_entry = Wire(new Entry) new_entry := DontCare - val new_entry_id = MuxCase((rob_entries-1).U, entries.zipWithIndex.map { case (e, i) => !e.valid -> i.U }) + val new_full_allocs = Wire(Vec(rob_full_entries, Bool())) + new_full_allocs.foreach(_ := false.B) + val new_partial_allocs = Wire(Vec(rob_partial_entries, Bool())) + new_partial_allocs.foreach(_ := false.B) + val new_entry_oh = new_full_allocs ++ new_partial_allocs val alloc_fire = io.alloc.fire() val raws_probe = WireInit(0.U(rob_entries.W)) @@ -126,7 +133,6 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val wars_op1_probe = WireInit(0.U(rob_entries.W)) val wars_op2_probe = WireInit(0.U(rob_entries.W)) - val raws_op1_probe = WireInit(0.U(rob_entries.W)) val raws_op2_probe = WireInit(0.U(rob_entries.W)) @@ -141,7 +147,9 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf dontTouch(is_st_and_must_wait_for_prior_ex_config_probe) dontTouch(is_ex_config_and_must_wait_for_prior_st_probe) - when (io.alloc.fire()) { + dontTouch(new_entry) + io.alloc.ready := false.B + when (io.alloc.valid) { val spAddrBits = 32 val cmd = io.alloc.bits val funct = cmd.inst.funct @@ -153,35 +161,55 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf new_entry.is_config := funct === CONFIG_CMD - new_entry.op1.valid := funct === PRELOAD_CMD || funct_is_compute - new_entry.op1.bits.start := cmd.rs1.asTypeOf(local_addr_t) + val op1 = Wire(UDValid(new OpT)) + op1.valid := false.B + op1.bits := DontCare + val op2 = Wire(UDValid(new OpT)) + op2.valid := false.B + op2.bits := DontCare + val dst = Wire(UDValid(new OpT)) + dst.valid := false.B + dst.bits := DontCare + assert(!(op1.valid && op2.valid && dst.valid)) + + new_entry.opa_is_dst := dst.valid + when (dst.valid) { + new_entry.opa := dst + new_entry.opb := Mux(op1.valid, op1, op2) + } .otherwise { + new_entry.opa := Mux(op1.valid, op1, op2) + new_entry.opb := op2 + } + + op1.valid := funct === PRELOAD_CMD || funct_is_compute + op1.bits.start := cmd.rs1.asTypeOf(local_addr_t) when (funct === PRELOAD_CMD) { val preload_rows = cmd.rs1(48 + log2Up(block_rows + 1) - 1, 48) - new_entry.op1.bits.end := new_entry.op1.bits.start + preload_rows - new_entry.op1.bits.wraps_around := new_entry.op1.bits.start.add_with_overflow(preload_rows)._2 + op1.bits.end := op1.bits.start + preload_rows + op1.bits.wraps_around := op1.bits.start.add_with_overflow(preload_rows)._2 }.otherwise { val compute_rows = cmd.rs1(48 + log2Up(block_rows + 1) - 1, 48) * a_stride - new_entry.op1.bits.end := new_entry.op1.bits.start + compute_rows - new_entry.op1.bits.wraps_around := new_entry.op1.bits.start.add_with_overflow(compute_rows)._2 + op1.bits.end := op1.bits.start + compute_rows + op1.bits.wraps_around := op1.bits.start.add_with_overflow(compute_rows)._2 } - new_entry.op2.valid := funct_is_compute || funct === STORE_CMD - new_entry.op2.bits.start := cmd.rs2.asTypeOf(local_addr_t) + op2.valid := funct_is_compute || funct === STORE_CMD + op2.bits.start := cmd.rs2.asTypeOf(local_addr_t) when (funct_is_compute) { val compute_rows = cmd.rs2(48 + log2Up(block_rows + 1) - 1, 48) - new_entry.op2.bits.end := new_entry.op2.bits.start + compute_rows - new_entry.op2.bits.wraps_around := new_entry.op2.bits.start.add_with_overflow(compute_rows)._2 + op2.bits.end := op2.bits.start + compute_rows + op2.bits.wraps_around := op2.bits.start.add_with_overflow(compute_rows)._2 }.elsewhen (pooling_is_enabled) { // If pooling is enabled, then we assume that this command simply mvouts everything in this accumulator bank from // start to the end of the bank - val acc_bank = new_entry.op2.bits.start.acc_bank() + val acc_bank = op2.bits.start.acc_bank() val next_bank_addr = WireInit(0.U.asTypeOf(local_addr_t)) next_bank_addr.is_acc_addr := true.B next_bank_addr.data := (acc_bank + 1.U) << local_addr_t.accBankRowBits - new_entry.op2.bits.end := next_bank_addr - new_entry.op2.bits.wraps_around := next_bank_addr.acc_bank() === 0.U + op2.bits.end := next_bank_addr + op2.bits.wraps_around := next_bank_addr.acc_bank() === 0.U }.otherwise { val block_stride = st_block_stride @@ -191,16 +219,16 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val mvout_mats = mvout_cols / block_cols.U + (mvout_cols % block_cols.U =/= 0.U) val total_mvout_rows = ((mvout_mats - 1.U) * block_stride) + mvout_rows - new_entry.op2.bits.end := new_entry.op2.bits.start + total_mvout_rows - new_entry.op2.bits.wraps_around := pooling_is_enabled || new_entry.op2.bits.start.add_with_overflow(total_mvout_rows)._2 + op2.bits.end := op2.bits.start + total_mvout_rows + op2.bits.wraps_around := pooling_is_enabled || op2.bits.start.add_with_overflow(total_mvout_rows)._2 } - new_entry.dst.valid := funct === PRELOAD_CMD || funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD - new_entry.dst.bits.start := cmd.rs2(31, 0).asTypeOf(local_addr_t) + dst.valid := funct === PRELOAD_CMD || funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD + dst.bits.start := cmd.rs2(31, 0).asTypeOf(local_addr_t) when (funct === PRELOAD_CMD) { val preload_rows = cmd.rs2(48 + log2Up(block_rows + 1) - 1, 48) - new_entry.dst.bits.end := new_entry.dst.bits.start + preload_rows - new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(preload_rows)._2 + dst.bits.end := dst.bits.start + preload_rows + dst.bits.wraps_around := dst.bits.start.add_with_overflow(preload_rows)._2 }.otherwise { val id = MuxCase(0.U, Seq((new_entry.cmd.inst.funct === LOAD2_CMD) -> 1.U, (new_entry.cmd.inst.funct === LOAD3_CMD) -> 2.U)) @@ -212,8 +240,8 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf val mvin_mats = mvin_cols / block_cols.U + (mvin_cols % block_cols.U =/= 0.U) val total_mvin_rows = ((mvin_mats - 1.U) * block_stride) + mvin_rows - new_entry.dst.bits.end := new_entry.dst.bits.start + total_mvin_rows - new_entry.dst.bits.wraps_around := new_entry.dst.bits.start.add_with_overflow(total_mvin_rows)._2 + dst.bits.end := dst.bits.start + total_mvin_rows + dst.bits.wraps_around := dst.bits.start.add_with_overflow(total_mvin_rows)._2 } val is_load = funct === LOAD_CMD || funct === LOAD2_CMD || funct === LOAD3_CMD || (funct === CONFIG_CMD && config_cmd_type === CONFIG_LOAD) @@ -228,65 +256,58 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf )) assert(is_load || is_store || is_ex) + // This can be RAW op1/op2 <- dst, or WAW dst <- dst + val opa_matches_opa = VecInit(entries.map { e => e.valid && e.bits.opa.valid && new_entry.opa.bits.overlaps(e.bits.opa.bits) }) + // This can be WAR dst <- op1/op2 + val opa_matches_opb = VecInit(entries.map { e => e.valid && e.bits.opb.valid && new_entry.opa.bits.overlaps(e.bits.opb.bits) }) + // This can be RAW op2 <- dst + val opb_matches_opa = VecInit(entries.map { e => e.valid && e.bits.opa.valid && new_entry.opb.bits.overlaps(e.bits.opa.bits) }) + + val op1_matches_opa = VecInit((entries zip (opa_matches_opa zip opb_matches_opa)).map { case (e, (a, b)) => + e.valid && op1.valid && Mux(dst.valid, b, a) + }) + val op2_matches_opa = VecInit((entries zip (opa_matches_opa zip opb_matches_opa)).map { case (e, (a, b)) => + e.valid && op2.valid && Mux(dst.valid || op1.valid, b, a) + }) + val dst_matches_opa = VecInit((entries zip opa_matches_opa).map { case (e, a) => + e.valid && dst.valid && a + }) + val dst_matches_opb = VecInit((entries zip opa_matches_opb).map { case (e, b) => + e.valid && dst.valid && b + }) - // TODO we should checck whether op1 and op2 are valid here - val raws = entries.map { e => - // We search for all entries which write to an address which we read from - e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( - (new_entry.op1.valid && new_entry.op1.bits.overlaps(e.bits.dst.bits)) || - (new_entry.op2.valid && new_entry.op2.bits.overlaps(e.bits.dst.bits))) - } - - val raws_op1 = entries.map { e => - // We search for all entries which write to an address which we read from - e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( - (new_entry.op1.valid && new_entry.op1.bits.overlaps(e.bits.dst.bits))) - } - - val raws_op2 = entries.map { e => - // We search for all entries which write to an address which we read from - e.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && ( - (new_entry.op2.valid && new_entry.op2.bits.overlaps(e.bits.dst.bits))) - } - - // TODO we should checck whether op1 and op2 are valid here - val wars = entries.map { e => - // We search for all entries which read from an address that we write to - e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( - (e.bits.op1.valid && e.bits.op1.bits.overlaps(new_entry.dst.bits)) || - (e.bits.op2.valid && e.bits.op2.bits.overlaps(new_entry.dst.bits))) - } - - val wars_op1 = entries.map { e => - // We search for all entries which read from an address that we write to - e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( - e.bits.op1.bits.overlaps(new_entry.dst.bits)) - } + val op1_raws_opa = VecInit((entries zip op1_matches_opa).map { case (e, m) => + m && op1.valid && e.bits.q =/= new_entry.q && e.bits.opa_is_dst + }) + val op2_raws_opa = VecInit((entries zip op2_matches_opa).map { case (e, m) => + m && op2.valid && e.bits.q =/= new_entry.q && e.bits.opa_is_dst + }) + val raws = VecInit((op1_raws_opa zip op2_raws_opa).map { case (a, b) => a || b }) - val wars_op2 = entries.map { e => - // We search for all entries which read from an address that we write to - e.valid && new_entry.dst.valid && e.bits.q =/= new_entry.q && ( - e.bits.op2.bits.overlaps(new_entry.dst.bits)) - } + val dst_wars_opa = VecInit((entries zip dst_matches_opa).map { case (e, m) => + m && dst.valid && e.bits.q =/= new_entry.q && !e.bits.opa_is_dst + }) + val dst_wars_opb = VecInit((entries zip dst_matches_opb).map { case (e, m) => + m && dst.valid && e.bits.q =/= new_entry.q + }) + val wars = VecInit((dst_wars_opa zip dst_wars_opb).map { case (a, b) => a || b }) - // TODO we should checck whether op1 and op2 are valid here - val waws = entries.map { e => - // We search for all entries which write to an address that we write to - e.valid && new_entry.dst.valid && e.bits.dst.valid && e.bits.q =/= new_entry.q && - (new_entry.dst.bits.overlaps(e.bits.dst.bits) || e.bits.dst.bits.overlaps(new_entry.dst.bits)) - } + val dst_waws_opa = VecInit((entries zip dst_matches_opa).map { case (e, m) => + m && dst.valid && (e.bits.q =/= new_entry.q || new_entry.q === ldq) && e.bits.opa_is_dst + }) + val waws = dst_waws_opa - val older_in_same_q = entries.map { e => + val older_in_same_q = VecInit(entries.map { e => e.valid && e.bits.q === new_entry.q && !e.bits.issued - } + }) - val is_st_and_must_wait_for_prior_ex_config = entries.map { e => + val is_st_and_must_wait_for_prior_ex_config = VecInit(entries.map { e => e.valid && new_entry.q === stq && !new_entry.is_config && e.bits.q === exq && e.bits.is_config - } + }) - val is_ex_config_and_must_wait_for_prior_st = entries.map { e => + val is_ex_config_and_must_wait_for_prior_st = VecInit(entries.map { e => e.valid && new_entry.q === exq && new_entry.is_config && e.bits.q === stq && !e.bits.is_config - } + }) new_entry.deps := (Cat(raws) | Cat(wars) | Cat(waws) | Cat(older_in_same_q) | Cat(is_st_and_must_wait_for_prior_ex_config) | Cat(is_ex_config_and_must_wait_for_prior_st)).asBools().reverse @@ -294,10 +315,6 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf raws_probe := Cat(raws.reverse) waws_probe := Cat(waws.reverse) wars_probe := Cat(wars.reverse) - wars_op1_probe := Cat(wars_op1.reverse) - wars_op2_probe := Cat(wars_op2.reverse) - raws_op1_probe := Cat(raws_op1.reverse) - raws_op2_probe := Cat(raws_op2.reverse) older_in_same_q_probe := Cat(older_in_same_q.reverse) is_st_and_must_wait_for_prior_ex_config_probe := Cat(is_st_and_must_wait_for_prior_ex_config.reverse) is_ex_config_and_must_wait_for_prior_st_probe := Cat(is_ex_config_and_must_wait_for_prior_st.reverse) @@ -306,48 +323,66 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf new_entry.complete_on_issue := new_entry.is_config && new_entry.q =/= exq - entries(new_entry_id).valid := true.B - entries(new_entry_id).bits := new_entry - - last_allocated := new_entry_id + val is_full = PopCount(Seq(dst.valid, op1.valid, op2.valid)) > 1.U + val full_alloc_id = MuxCase((rob_full_entries-1).U, full_entries.zipWithIndex.map { case (e, i) => !e.valid -> i.U }) + val partial_alloc_id = MuxCase((rob_partial_entries-1).U, partial_entries.zipWithIndex.map { case (e, i) => !e.valid -> i.U }) + + when (!is_full && !partial_entries(partial_alloc_id).valid) { + io.alloc.ready := true.B + partial_entries(partial_alloc_id).valid := true.B + partial_entries(partial_alloc_id).bits := new_entry + partial_entries(partial_alloc_id).bits.opb.valid := false.B + partial_entries(partial_alloc_id).bits.opb.bits := DontCare + new_partial_allocs(partial_alloc_id) := true.B + } .elsewhen (!full_entries(full_alloc_id).valid) { + io.alloc.ready := true.B + full_entries(full_alloc_id).valid := true.B + full_entries(full_alloc_id).bits := new_entry + new_full_allocs(full_alloc_id) := true.B + } - when (new_entry.is_config && new_entry.q === exq && !is_im2col) { - a_stride := new_entry.cmd.rs1(31, 16) // TODO magic numbers // TODO this needs to be kept in sync with ExecuteController.scala - }.elsewhen(new_entry.is_config && new_entry.q === ldq) { - val id = new_entry.cmd.rs1(4,3) // TODO magic numbers - val block_stride = new_entry.cmd.rs1(31, 16) // TODO magic numbers - ld_block_strides(id) := block_stride - }.elsewhen(new_entry.is_config && new_entry.q === stq) { - val pool_stride = new_entry.cmd.rs1(5, 4) // TODO magic numbers - pooling_is_enabled := pool_stride =/= 0.U + when (io.alloc.fire()) { + when (new_entry.is_config && new_entry.q === exq && !is_im2col) { + a_stride := new_entry.cmd.rs1(31, 16) // TODO magic numbers // TODO this needs to be kept in sync with ExecuteController.scala + }.elsewhen(new_entry.is_config && new_entry.q === ldq) { + val id = new_entry.cmd.rs1(4,3) // TODO magic numbers + val block_stride = new_entry.cmd.rs1(31, 16) // TODO magic numbers + ld_block_strides(id) := block_stride + }.elsewhen(new_entry.is_config && new_entry.q === stq) { + val pool_stride = new_entry.cmd.rs1(5, 4) // TODO magic numbers + pooling_is_enabled := pool_stride =/= 0.U + } } } // Issue commands which are ready to be issued Seq((ldq, io.issue.ld), (stq, io.issue.st), (exq, io.issue.ex)).foreach { case (q, io) => - val issue_id = MuxCase((rob_entries-1).U, entries.zipWithIndex.map { case (e, i) => - (e.valid && e.bits.ready() && !e.bits.issued && e.bits.q === q) -> i.U - }) + val issue_valids = entries.map(e => e.valid && e.bits.ready() && !e.bits.issued && e.bits.q === q) + val issue_sel = PriorityEncoderOH(issue_valids) + val issue_id = OHToUInt(issue_sel) + val issue_entry = Mux1H(issue_sel, entries) - io.valid := entries.map(e => e.valid && e.bits.ready() && !e.bits.issued && e.bits.q === q).reduce(_ || _) - io.cmd := entries(issue_id).bits.cmd - io.rob_id := issue_id + io.valid := issue_valids.reduce(_||_) + io.cmd := issue_entry.bits.cmd + io.rob_id := OHToUInt(issue_sel) when (io.fire()) { - entries(issue_id).bits.issued := true.B - // Clear out all the dependency bits for instructions which depend on the same queue entries.zipWithIndex.foreach { case (e, i) => - val is_same_q = Mux(alloc_fire && new_entry_id === i.U, - new_entry.q === entries(issue_id).bits.q, - e.bits.q === entries(issue_id).bits.q) + val is_same_q = Mux(alloc_fire && new_entry_oh(i), + new_entry.q === issue_entry.bits.q, + e.bits.q === issue_entry.bits.q) - when (is_same_q || entries(issue_id).bits.complete_on_issue) { + when (is_same_q || issue_entry.bits.complete_on_issue) { e.bits.deps(issue_id) := false.B } } - - entries(issue_id).valid := !entries(issue_id).bits.complete_on_issue + for ((e, i) <- entries.zipWithIndex) { + when (issue_sel(i)) { + e.bits.issued := true.B + e.valid := !e.bits.complete_on_issue + } + } } } @@ -355,8 +390,12 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf when (io.completed.fire()) { entries.foreach(_.bits.deps(io.completed.bits) := false.B) - entries(io.completed.bits).valid := false.B - assert(entries(io.completed.bits).valid) + for ((e, i) <- entries.zipWithIndex) { + when (i.U === io.completed.bits) { + e.valid := false.B + assert(e.valid) + } + } } // val utilization = PopCount(entries.map(e => e.valid)) @@ -410,7 +449,6 @@ class ROB[T <: Data : Arithmetic, U <: Data, V <: Data](config: GemminiArrayConf printf(p"Utilization st q: $utilization_st_q\n") printf(p"Utilization ex q: $utilization_ex_q\n") printf(p"Packed deps: $packed_deps\n") - printf(p"Last allocated: $last_allocated\n\n") } when (reset.asBool()) { From 3c845e3d0f9aa56fa360333f09282e60ee6f775a Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 21 Mar 2021 12:31:26 -0700 Subject: [PATCH 072/123] pooling fixed --- src/main/scala/gemmini/LoopConv.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index da2acead9..8e8fa4a58 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -606,7 +606,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol - val pool_dram_addr = req.dram_addr + ((b * pool_out_dim * pool_out_dim) * out_channels + och) * (input_w/8).U + val pool_dram_addr = req.dram_addr + ((b * pool_out_dim * pool_out_dim) * och_stride + och) * (input_w/8).U val pool_spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols // Sizes @@ -628,13 +628,13 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: pre_pool_config_cmd.rs1 := (ocols << 56) | (orows << 48) | (pocols << 40) | (porows << 32) | (pool_out_dim << 24) | (plpad << 10) | (pupad << 8) | (pool_size << 6) | (pool_stride << 4) | // TODO magic numbers CONFIG_STORE - pre_pool_config_cmd.rs2 := out_channels * (input_w / 8).U + pre_pool_config_cmd.rs2 := och_stride * (input_w / 8).U val post_pool_config_cmd = Wire(new RoCCCommand) post_pool_config_cmd := DontCare post_pool_config_cmd.inst.funct := CONFIG_CMD post_pool_config_cmd.rs1 := CONFIG_STORE - post_pool_config_cmd.rs2 := out_channels * (input_w / 8).U + post_pool_config_cmd.rs2 := och_stride * (input_w / 8).U val pool_cmd = Wire(new RoCCCommand) pool_cmd := DontCare From b49679db6659a96f2efbaff60cb2892380d7c551 Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Wed, 24 Mar 2021 10:56:20 +0400 Subject: [PATCH 073/123] Increase the throughput of matmuls in WS mode (#87) Resnet50 utilization: 46% --- src/main/scala/gemmini/AccumulatorMem.scala | 4 - .../scala/gemmini/ExecuteController.scala | 174 +++++++----- src/main/scala/gemmini/Mesh.scala | 45 +++- src/main/scala/gemmini/MeshWithDelays.scala | 253 +++++++++--------- src/main/scala/gemmini/PE.scala | 12 +- src/main/scala/gemmini/TagQueue.scala | 52 ++-- src/main/scala/gemmini/Tile.scala | 31 ++- src/main/scala/gemmini/Util.scala | 5 + 8 files changed, 337 insertions(+), 239 deletions(-) diff --git a/src/main/scala/gemmini/AccumulatorMem.scala b/src/main/scala/gemmini/AccumulatorMem.scala index 1939c6243..0fafb9521 100644 --- a/src/main/scala/gemmini/AccumulatorMem.scala +++ b/src/main/scala/gemmini/AccumulatorMem.scala @@ -207,7 +207,6 @@ class AccumulatorMem[T <: Data, U <: Data]( q.io.enq.bits.acc_bank_id := DontCare q.io.enq.valid := RegNext(io.read.req.fire()) - val p = q.io.deq io.read.resp.bits.data := p.bits.data @@ -219,7 +218,6 @@ class AccumulatorMem[T <: Data, U <: Data]( io.read.resp.valid := p.valid p.ready := io.read.resp.ready - val q_will_be_empty = (q.io.count +& q.io.enq.fire()) - q.io.deq.fire() === 0.U io.read.req.ready := q_will_be_empty && ( // Make sure we aren't accumulating, which would take over both ports @@ -230,8 +228,6 @@ class AccumulatorMem[T <: Data, U <: Data]( !block_read_req ) - - // io.write.current_waddr.valid := mem.io.wen // io.write.current_waddr.bits := mem.io.waddr io.write.ready := !io.write.bits.acc || (!(io.write.bits.addr === waddr_buf && w_buf_valid) && diff --git a/src/main/scala/gemmini/ExecuteController.scala b/src/main/scala/gemmini/ExecuteController.scala index a1cb3230c..d4e2089c1 100644 --- a/src/main/scala/gemmini/ExecuteController.scala +++ b/src/main/scala/gemmini/ExecuteController.scala @@ -187,19 +187,22 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh.io.a.valid := false.B mesh.io.b.valid := false.B mesh.io.d.valid := false.B - mesh.io.tag_in.valid := false.B - mesh.io.flush.valid := control_state === flush && !cntl_valid // We want to make sure that the mesh has absorbed all inputs before flushing + mesh.io.req.valid := control_state === flush mesh.io.a.bits := DontCare mesh.io.b.bits := DontCare mesh.io.d.bits := DontCare - mesh.io.tag_in.bits := DontCare - mesh.io.pe_control.propagate := Mux(control_state === flush, in_prop_flush, cntl.prop) - mesh.io.pe_control.dataflow := cntl.dataflow - mesh.io.pe_control.shift := cntl.shift - mesh.io.a_transpose := a_transpose - mesh.io.bd_transpose := bd_transpose - mesh.io.flush.bits := 0.U + mesh.io.req.bits.tag := DontCare + mesh.io.req.bits.tag.cols := cntl.c_cols + mesh.io.req.bits.tag.rows := cntl.c_rows + mesh.io.req.bits.total_rows := block_size.U + mesh.io.req.bits.pe_control.propagate := Mux(control_state === flush, in_prop_flush, cntl.prop) + mesh.io.req.bits.pe_control.dataflow := cntl.dataflow + mesh.io.req.bits.pe_control.shift := cntl.shift + mesh.io.req.bits.a_transpose := cntl.a_transpose + mesh.io.req.bits.bd_transpose := cntl.bd_transpose + mesh.io.req.bits.tag.rob_id := cntl.rob_id + mesh.io.req.bits.flush := Mux(control_state === flush && !cntl_valid, 1.U, 0.U) // We want to make sure that the mesh has absorbed all inputs before flushing // Hazards val raw_hazard_pre = mesh.io.tags_in_progress.map { t => @@ -230,7 +233,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val b_fire_counter = Reg(UInt(log2Up(block_size).W)) val d_fire_counter = Reg(UInt(log2Up(block_size).W)) - // These "*_fire_started" variables are only needed for 2x2 systolic arrays val a_fire_started = RegInit(false.B) val d_fire_started = RegInit(false.B) val b_fire_started = RegInit(false.B) @@ -260,6 +262,10 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val start_inputting_d = WireInit(false.B) val start_array_outputting = WireInit(false.B) + val a_garbage = a_address_rs1.is_garbage() || !start_inputting_a + val b_garbage = b_address_rs2.is_garbage() || !start_inputting_b + val d_garbage = d_address_rs1.is_garbage() || !start_inputting_d + // TODO merge these into one enum val perform_single_preload = RegInit(false.B) val perform_single_mul = RegInit(false.B) @@ -269,6 +275,25 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val performing_single_mul = WireInit(perform_single_mul && control_state === compute) val performing_mul_pre = WireInit(perform_mul_pre && control_state === compute) + val total_rows = WireInit(block_size.U) // The total number of rows of A, B, and D to feed into the mesh + + // TODO Also reduce the number of rows when "perform_single_preload === true.B" + when (current_dataflow === Dataflow.WS.id.U && d_garbage && + !a_should_be_fed_into_transposer && !b_should_be_fed_into_transposer && !d_should_be_fed_into_transposer) { + val rows_a = Mux(a_garbage, 1.U, a_rows) + val rows_b = Mux(b_garbage, 1.U, b_rows) + + /* We can only retire one ROB instruction per cycle (max), but if total_rows == 1, then we would be trying to retire + 2 ROB instructions per cycle (one for the preload, and one for the compute). Therefore, to prevent ROB + instructions from being lost, we set a minimum floor for total_rows of 2. + + Furthermore, two writes to the same accumulator address must occur at least 4 cycles apart to allow the write to + fully propagate through. Therefore, we raise the minimum floor for total_rows to 4. + TODO: add a WAW check to the ROB so that we can lower the floor back to 2 + */ + total_rows := maxOf(maxOf(rows_a, rows_b), 4.U) + } + //added for mul_pre sync val mul_pre_counter_sub = RegInit(0.U(3.W)) val mul_pre_counter_count = RegInit(0.U(3.W)) @@ -310,16 +335,9 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val others = operands.filter(_.priority != priority) val same_banks = others.map(o => same_bank(addr, o.addr, is_garbage, o.is_garbage, start_inputting, o.start_inputting, can_be_im2colled || o.can_be_im2colled)) - val same_counter = others.map(o => counter === o.counter) - - val one_ahead = { - if (block_size > 2) - others.map(o => counter === wrappingAdd(o.counter, 1.U, block_size)) - else { - others.map(o => (started && !o.started && counter === 1.U && o.counter === 0.U) || - (started && o.started && counter === 0.U && o.counter === 1.U)) - } - } + val same_counter = others.map(o => started === o.started && counter === o.counter) + + val one_ahead = others.map(o => started && counter === wrappingAdd(o.counter, 1.U, total_rows)) val higher_priorities = others.map(o => (o.priority < priority).B) @@ -341,22 +359,22 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In a_fire_counter := 0.U a_addr_offset := 0.U }.elsewhen (firing && a_fire && cntl_ready) { - a_fire_counter := wrappingAdd(a_fire_counter, 1.U, block_size) - a_addr_offset := Mux(a_fire_counter === (block_size-1).U, 0.U, a_addr_offset + a_addr_stride) + a_fire_counter := wrappingAdd(a_fire_counter, 1.U, total_rows) + a_addr_offset := Mux(a_fire_counter === (total_rows-1.U), 0.U, a_addr_offset + a_addr_stride) a_fire_started := true.B } when (!firing) { b_fire_counter := 0.U }.elsewhen (firing && b_fire && cntl_ready) { - b_fire_counter := wrappingAdd(b_fire_counter, 1.U, block_size) + b_fire_counter := wrappingAdd(b_fire_counter, 1.U, total_rows) b_fire_started := true.B } when (!firing) { d_fire_counter := 0.U }.elsewhen (firing && d_fire && cntl_ready) { - d_fire_counter := wrappingAdd(d_fire_counter, 1.U, block_size) + d_fire_counter := wrappingAdd(d_fire_counter, 1.U, total_rows) d_fire_started := true.B } @@ -377,18 +395,16 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In // The last line in this (long) Boolean is just to make sure that we don't think we're done as soon as we begin firing // TODO change when square requirement lifted - val about_to_fire_all_rows = ((a_fire_counter === (block_size-1).U && a_fire) || a_fire_counter === 0.U) && - ((b_fire_counter === (block_size-1).U && b_fire) || b_fire_counter === 0.U) && - ((d_fire_counter === (block_size-1).U && d_fire) || d_fire_counter === 0.U) && - (a_fire_counter =/= 0.U || b_fire_counter =/= 0.U || d_fire_counter =/= 0.U) && + val about_to_fire_all_rows = ((a_fire_counter === (total_rows-1.U) && a_fire) || a_fire_counter === 0.U) && + ((b_fire_counter === (total_rows-1.U) && b_fire) || b_fire_counter === 0.U) && + ((d_fire_counter === (total_rows-1.U) && d_fire) || d_fire_counter === 0.U) && + (a_fire_started || b_fire_started || d_fire_started) && cntl_ready - if (block_size == 2) { - when (about_to_fire_all_rows) { - a_fire_started := false.B - b_fire_started := false.B - d_fire_started := false.B - } + when (about_to_fire_all_rows) { + a_fire_started := false.B + b_fire_started := false.B + d_fire_started := false.B } val d_fire_counter_mulpre = WireInit(b_fire_counter) @@ -402,7 +418,6 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val read_b = b_valid && !b_read_from_acc && dataBbank === i.U && start_inputting_b && !accumulate_zeros && b_row_is_not_all_zeros //&& !im2col_wire val read_d = d_valid && !d_read_from_acc && dataDbank === i.U && start_inputting_d && !preload_zeros && d_row_is_not_all_zeros //&& !im2col_wire - Seq((read_a, a_ready), (read_b, b_ready), (read_d, d_ready)).foreach { case (rd, r) => when (rd && !io.srams.read(i).req.ready) { r := false.B @@ -639,25 +654,23 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In when(about_to_fire_all_rows) { cmd.pop := 1.U control_state := waiting_for_cmd - pending_completed_rob_ids(0) := cmd.bits(0).rob_id } } - } + } is(flush) { - when(mesh.io.flush.fire()) { + when(mesh.io.req.fire()) { control_state := flushing } } is(flushing) { - when(mesh.io.flush.ready) { + when(mesh.io.req.ready) { // TODO we waste a cycle here if it was better to continue with the flush control_state := waiting_for_cmd } } } - // Computing logic val computing = performing_mul_pre || performing_single_mul || performing_single_preload @@ -697,6 +710,11 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val c_rows = UInt(log2Up(block_size + 1).W) val c_cols = UInt(log2Up(block_size + 1).W) + val a_transpose = Bool() + val bd_transpose = Bool() + + val total_rows = UInt(log2Up(block_size + 1).W) + val rob_id = UDValid(UInt(log2Up(rob_entries).W)) val dataflow = UInt(1.W) @@ -704,6 +722,8 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In val shift = UInt(log2Up(accType.getWidth).W) val im2colling = Bool() + + val first = Bool() } mesh_cntl_signals_q.io.enq.valid := computing @@ -720,9 +740,9 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.enq.bits.b_bank_acc := dataBBankAcc mesh_cntl_signals_q.io.enq.bits.d_bank_acc := dataDBankAcc - mesh_cntl_signals_q.io.enq.bits.a_garbage := a_address_rs1.is_garbage() || !start_inputting_a - mesh_cntl_signals_q.io.enq.bits.b_garbage := b_address_rs2.is_garbage() || !start_inputting_b - mesh_cntl_signals_q.io.enq.bits.d_garbage := d_address_rs1.is_garbage() || !start_inputting_d + mesh_cntl_signals_q.io.enq.bits.a_garbage := a_garbage + mesh_cntl_signals_q.io.enq.bits.b_garbage := b_garbage + mesh_cntl_signals_q.io.enq.bits.d_garbage := d_garbage mesh_cntl_signals_q.io.enq.bits.a_read_from_acc := a_read_from_acc mesh_cntl_signals_q.io.enq.bits.b_read_from_acc := b_read_from_acc @@ -735,6 +755,8 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.enq.bits.b_unpadded_cols := Mux(b_row_is_not_all_zeros, b_cols, 0.U) mesh_cntl_signals_q.io.enq.bits.d_unpadded_cols := Mux(d_row_is_not_all_zeros, d_cols, 0.U) + mesh_cntl_signals_q.io.enq.bits.total_rows := total_rows + mesh_cntl_signals_q.io.enq.bits.a_fire := a_fire mesh_cntl_signals_q.io.enq.bits.b_fire := b_fire mesh_cntl_signals_q.io.enq.bits.d_fire := d_fire @@ -743,6 +765,9 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.enq.bits.c_rows := c_rows mesh_cntl_signals_q.io.enq.bits.c_cols := c_cols + mesh_cntl_signals_q.io.enq.bits.a_transpose := a_transpose + mesh_cntl_signals_q.io.enq.bits.bd_transpose := bd_transpose + mesh_cntl_signals_q.io.enq.bits.rob_id.valid := !performing_single_mul && !c_address_rs2.is_garbage() mesh_cntl_signals_q.io.enq.bits.rob_id.bits := cmd.bits(preload_cmd_place).rob_id.bits @@ -752,6 +777,8 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.enq.bits.im2colling := im2col_wire && im2col_en //im2col_wire + mesh_cntl_signals_q.io.enq.bits.first := !a_fire_started && !b_fire_started && !d_fire_started + val readData = VecInit(io.srams.read.map(_.resp.bits.data)) val accReadData = if (ex_read_from_acc) VecInit(io.acc.read_resp.map(_.bits.data.asUInt())) else readData val im2ColData = io.im2col.resp.bits.a_im2col.asUInt() @@ -762,7 +789,8 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In mesh_cntl_signals_q.io.deq.ready := (!cntl.a_fire || mesh.io.a.fire() || !mesh.io.a.ready) && (!cntl.b_fire || mesh.io.b.fire() || !mesh.io.b.ready) && - (!cntl.d_fire || mesh.io.d.fire() || !mesh.io.d.ready) + (!cntl.d_fire || mesh.io.d.fire() || !mesh.io.d.ready) && + (!cntl.first || mesh.io.req.ready) val dataA_valid = cntl.a_garbage || cntl.a_unpadded_cols === 0.U || Mux(cntl.im2colling, im2ColValid, Mux(cntl.a_read_from_acc, accReadValid(cntl.a_bank_acc), readValid(cntl.a_bank))) @@ -815,25 +843,27 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } } - for (acc_r <- io.acc.read_resp) { - acc_r.ready := true.B + if (!ex_read_from_acc) { + for (acc_r <- io.acc.read_resp) { + acc_r.ready := true.B + } } when (cntl_valid) { // Default inputs mesh.io.a.valid := cntl.a_fire && dataA_valid - mesh.io.b.valid := (cntl.b_fire && dataB_valid) - mesh.io.d.valid := (cntl.d_fire && dataD_valid) - mesh.io.tag_in.valid := true.B + mesh.io.b.valid := cntl.b_fire && dataB_valid + mesh.io.d.valid := cntl.d_fire && dataD_valid mesh.io.a.bits := dataA.asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.b.bits := dataB.asTypeOf(Vec(meshColumns, Vec(tileColumns, inputType))) mesh.io.d.bits := dataD.asTypeOf(Vec(meshColumns, Vec(tileColumns, inputType))) - mesh.io.tag_in.bits.rob_id := cntl.rob_id - mesh.io.tag_in.bits.addr := cntl.c_addr - mesh.io.tag_in.bits.cols := cntl.c_cols - mesh.io.tag_in.bits.rows := cntl.c_rows + mesh.io.req.valid := mesh_cntl_signals_q.io.deq.fire() && (cntl.a_fire || cntl.b_fire || cntl.d_fire) + + mesh.io.req.bits.tag.addr := cntl.c_addr + + mesh.io.req.bits.total_rows := cntl.total_rows } when (cntl_valid && cntl.perform_single_preload) { @@ -844,31 +874,34 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In when (cntl_valid && cntl.perform_single_mul) { mesh.io.a.bits := Mux(a_should_be_fed_into_transposer, 0.U, dataA.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) mesh.io.b.bits := Mux(b_should_be_fed_into_transposer, 0.U, dataB.asUInt).asTypeOf(Vec(meshRows, Vec(tileRows, inputType))) - mesh.io.tag_in.bits.addr.make_this_garbage() + mesh.io.req.bits.tag.addr.make_this_garbage() } // Scratchpad writes - val output_counter = new Counter(block_size) + // val output_counter = new Counter(block_size) + val output_counter = RegInit(0.U(log2Up(block_size).W)) - val w_address = Mux(current_dataflow === Dataflow.WS.id.U, mesh.io.tag_out.addr + output_counter.value, - mesh.io.tag_out.addr + (block_size.U - 1.U - output_counter.value)) + val w_total_output_rows = mesh.io.resp.bits.total_rows + + val w_address = Mux(current_dataflow === Dataflow.WS.id.U, mesh.io.resp.bits.tag.addr + output_counter, + mesh.io.resp.bits.tag.addr + (w_total_output_rows - 1.U - output_counter)) val write_to_acc = w_address.is_acc_addr val w_bank = Mux(write_to_acc, w_address.acc_bank(), w_address.sp_bank()) val w_row = Mux(write_to_acc, w_address.acc_row(), w_address.sp_row()) - val is_garbage_addr = mesh.io.tag_out.addr.is_garbage() + val is_garbage_addr = mesh.io.resp.bits.tag.addr.is_garbage() - val w_matrix_rows = mesh.io.tag_out.rows - val w_matrix_cols = mesh.io.tag_out.cols + val w_matrix_rows = mesh.io.resp.bits.tag.rows + val w_matrix_cols = mesh.io.resp.bits.tag.cols - val write_this_row = Mux(current_dataflow === Dataflow.WS.id.U, output_counter.value < w_matrix_rows, - block_size.U - 1.U - output_counter.value < w_matrix_rows) + val write_this_row = Mux(current_dataflow === Dataflow.WS.id.U, output_counter < w_matrix_rows, + w_total_output_rows - 1.U - output_counter < w_matrix_rows) val w_mask = (0 until block_size).map(_.U < w_matrix_cols) // This is an element-wise mask, rather than a byte-wise mask // Write to normal scratchpad for(i <- 0 until sp_banks) { - val activated_wdata = VecInit(mesh.io.out.bits.map(v => VecInit(v.map { e => + val activated_wdata = VecInit(mesh.io.resp.bits.data.map(v => VecInit(v.map { e => val e_clipped = e.clippedToWidthOf(inputType) val e_act = MuxCase(e_clipped, Seq( (activation === Activation.RELU) -> e_clipped.relu, @@ -895,7 +928,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In if (ex_write_to_acc) { io.acc.write(i).valid := start_array_outputting && w_bank === i.U && write_to_acc && !is_garbage_addr && write_this_row io.acc.write(i).bits.addr := w_row - io.acc.write(i).bits.data := VecInit(mesh.io.out.bits.map(v => VecInit(v.map(e => e.withWidthOf(accType))))) + io.acc.write(i).bits.data := VecInit(mesh.io.resp.bits.data.map(v => VecInit(v.map(e => e.withWidthOf(accType))))) io.acc.write(i).bits.acc := w_address.accumulate io.acc.write(i).bits.mask := w_mask.flatMap(b => Seq.fill(accType.getWidth / (aligned_to * 8))(b)) } else { @@ -914,13 +947,14 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In //val complete_lock = RegInit(false.B) //Seah: added for WS accumulator - when(mesh.io.out.fire() && mesh.io.tag_out.rob_id.valid) { - //when(current_dataflow === Dataflow.WS.id.U) { - when(output_counter.inc()) { + when(mesh.io.resp.fire() && mesh.io.resp.bits.tag.rob_id.valid) { + output_counter := wrappingAdd(output_counter, 1.U, w_total_output_rows) + val last = mesh.io.resp.bits.last + + when(last) { mesh_completed_rob_id_fire := true.B io.completed.valid := true.B - io.completed.bits := mesh.io.tag_out.rob_id.bits - + io.completed.bits := mesh.io.resp.bits.tag.rob_id.bits } start_array_outputting := !is_garbage_addr } @@ -935,7 +969,7 @@ class ExecuteController[T <: Data, U <: Data, V <: Data](xLen: Int, tagWidth: In } } val complete_bits_count = RegInit(0.U(15.W)) - when(io.completed.valid){ + when(io.completed.valid) { complete_bits_count := complete_bits_count + 1.U } dontTouch(complete_bits_count) diff --git a/src/main/scala/gemmini/Mesh.scala b/src/main/scala/gemmini/Mesh.scala index 22ece6f35..074ed4458 100644 --- a/src/main/scala/gemmini/Mesh.scala +++ b/src/main/scala/gemmini/Mesh.scala @@ -15,21 +15,26 @@ import chisel3.experimental._ * @param meshColumns */ class Mesh[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, - df: Dataflow.Value, pe_latency: Int, + df: Dataflow.Value, pe_latency: Int, max_simultaneous_matmuls: Int, val tileRows: Int, val tileColumns: Int, val meshRows: Int, val meshColumns: Int) extends Module { val io = IO(new Bundle { - val in_a = Input(Vec(meshRows, Vec(tileRows, inputType))) - val in_b = Input(Vec(meshColumns, Vec(tileColumns, inputType))) - val in_d = Input(Vec(meshColumns, Vec(tileColumns, inputType))) - val in_control = Input(Vec(meshColumns, Vec(tileColumns, new PEControl(accType)))) - val out_b = Output(Vec(meshColumns, Vec(tileColumns, outputType))) - val out_c = Output(Vec(meshColumns, Vec(tileColumns, outputType))) + val in_a = Input(Vec(meshRows, Vec(tileRows, inputType))) + val in_b = Input(Vec(meshColumns, Vec(tileColumns, inputType))) + val in_d = Input(Vec(meshColumns, Vec(tileColumns, inputType))) + val in_control = Input(Vec(meshColumns, Vec(tileColumns, new PEControl(accType)))) + val in_id = Input(Vec(meshColumns, Vec(tileColumns, UInt(log2Up(max_simultaneous_matmuls).W)))) // The unique id of this particular matmul + val in_last = Input(Vec(meshColumns, Vec(tileColumns, Bool()))) + val out_b = Output(Vec(meshColumns, Vec(tileColumns, outputType))) + val out_c = Output(Vec(meshColumns, Vec(tileColumns, outputType))) val in_valid = Input(Vec(meshColumns, Vec(tileColumns, Bool()))) val out_valid = Output(Vec(meshColumns, Vec(tileColumns, Bool()))) + val out_control = Output(Vec(meshColumns, Vec(tileColumns, new PEControl(accType)))) + val out_id = Output(Vec(meshColumns, Vec(tileColumns, UInt(log2Up(max_simultaneous_matmuls).W)))) + val out_last = Output(Vec(meshColumns, Vec(tileColumns, Bool()))) }) // mesh(r)(c) => Tile at row r, column c - val mesh: Seq[Seq[Tile[T]]] = Seq.fill(meshRows, meshColumns)(Module(new Tile(inputType, outputType, accType, df, pe_latency, tileRows, tileColumns))) + val mesh: Seq[Seq[Tile[T]]] = Seq.fill(meshRows, meshColumns)(Module(new Tile(inputType, outputType, accType, df, pe_latency, max_simultaneous_matmuls, tileRows, tileColumns))) val meshT = mesh.transpose // Chain tile_a_out -> tile_a_in (pipeline a across each row) // TODO clock-gate A signals with in_garbage @@ -78,13 +83,35 @@ class Mesh[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, tile.io.out_valid } } + + // Chain in_id (pipeline across each column) + for (c <- 0 until meshColumns) { + meshT(c).foldLeft(io.in_id(c)) { + case (in_id, tile) => + tile.io.in_id := RegNext(in_id) + tile.io.out_id + } + } + + // Chain in_last (pipeline across each column) + for (c <- 0 until meshColumns) { + meshT(c).foldLeft(io.in_last(c)) { + case (in_last, tile) => + tile.io.in_last := RegNext(in_last) + tile.io.out_last + } + } + // Capture out_vec and out_control_vec (connect IO to bottom row of mesh) // (The only reason we have so many zips is because Scala doesn't provide a zipped function for Tuple4) - for (((b, c), (v, tile)) <- ((io.out_b zip io.out_c), (io.out_valid zip mesh.last)).zipped) { + for (((((((b, c), v), ctrl), id), last), tile) <- io.out_b zip io.out_c zip io.out_valid zip io.out_control zip io.out_id zip io.out_last zip mesh.last) { // TODO we pipelined this to make physical design easier. Consider removing these if possible // TODO shouldn't we clock-gate these signals with "garbage" as well? b := RegNext(tile.io.out_b) c := RegNext(tile.io.out_c) v := RegNext(tile.io.out_valid) + ctrl := RegNext(tile.io.out_control) + id := RegNext(tile.io.out_id) + last := RegNext(tile.io.out_last) } } diff --git a/src/main/scala/gemmini/MeshWithDelays.scala b/src/main/scala/gemmini/MeshWithDelays.scala index d400c6778..ec094a21c 100644 --- a/src/main/scala/gemmini/MeshWithDelays.scala +++ b/src/main/scala/gemmini/MeshWithDelays.scala @@ -6,6 +6,26 @@ import chisel3.util._ import gemmini.Util._ +class MeshWithDelaysReq[T <: Data: Arithmetic, TagT <: TagQueueTag with Data](accType: T, tagType: TagT, block_size: Int) extends Bundle { + val pe_control = new PEControl(accType) + val a_transpose = Bool() + val bd_transpose = Bool() + val total_rows = UInt(log2Up(block_size+1).W) + val tag = tagType + val flush = UInt(2.W) // TODO magic number + + override def cloneType: MeshWithDelaysReq.this.type = new MeshWithDelaysReq(accType, tagType, block_size).asInstanceOf[this.type] +} + +class MeshWithDelaysResp[T <: Data: Arithmetic, TagT <: TagQueueTag with Data](outType: T, meshCols: Int, tileCols: Int, block_size: Int, tagType: TagT) extends Bundle { + val data = Vec(meshCols, Vec(tileCols, outType)) + val total_rows = UInt(log2Up(block_size+1).W) + val tag = tagType + val last = Bool() + + override def cloneType: MeshWithDelaysResp.this.type = new MeshWithDelaysResp(outType, meshCols, tileCols, block_size, tagType).asInstanceOf[this.type] +} + // TODO Add io.out.ready back in. Before it was removed, it didn't work when banking, and it seemed to assume that SRAM outputs stay steady when ren is low // TODO Handle matrices where N1 =/= N2 =/= N3 // TODO do we flush for one cycle more than necessary? @@ -15,7 +35,7 @@ class MeshWithDelays[T <: Data: Arithmetic, U <: TagQueueTag with Data] (inputType: T, val outputType: T, accType: T, tagType: U, df: Dataflow.Value, pe_latency: Int, tileRows: Int, tileColumns: Int, meshRows: Int, meshColumns: Int, - leftBanks: Int, upBanks: Int, outBanks: Int = 1) + leftBanks: Int, upBanks: Int, outBanks: Int = 1, n_simultaneous_matmuls: Int = -1) extends Module { val A_TYPE = Vec(meshRows, Vec(tileRows, inputType)) @@ -24,26 +44,28 @@ class MeshWithDelays[T <: Data: Arithmetic, U <: TagQueueTag with Data] val D_TYPE = Vec(meshColumns, Vec(tileColumns, inputType)) val S_TYPE = Vec(meshColumns, Vec(tileColumns, new PEControl(accType))) - val tagqlen = (if (meshColumns == 1) 4 else 5) * (pe_latency+1) // TODO change the tag-queue so we can make this 3 + assert(meshRows*tileRows == meshColumns*tileColumns) + val block_size = meshRows*tileRows + + val max_simultaneous_matmuls = if (n_simultaneous_matmuls == -1) { + 5 * (pe_latency + 1) + } else { + n_simultaneous_matmuls + } + assert(max_simultaneous_matmuls >= 5 * (pe_latency + 1)) + + val tagqlen = max_simultaneous_matmuls+1 val io = IO(new Bundle { val a = Flipped(Decoupled(A_TYPE)) val b = Flipped(Decoupled(B_TYPE)) val d = Flipped(Decoupled(D_TYPE)) - // TODO make pe_control a ready-valid interface as well - val pe_control = Input(new PEControl(accType)) + val req = Flipped(Decoupled(new MeshWithDelaysReq(accType, tagType.cloneType, block_size))) - val a_transpose = Input(Bool()) - val bd_transpose = Input(Bool()) + val resp = Valid(new MeshWithDelaysResp(outputType, meshColumns, tileColumns, block_size, tagType.cloneType)) - val tag_in = Flipped(Decoupled(tagType)) - val tag_out = Output(tagType) val tags_in_progress = Output(Vec(tagqlen, tagType)) - - val out = Valid(C_TYPE) // TODO make this ready-valid - - val flush = Flipped(Decoupled(UInt(2.W))) }) def shifted[T <: Data](x: Vec[Vec[T]], banks: Int, reverse: Boolean = false) = { @@ -70,33 +92,43 @@ class MeshWithDelays[T <: Data: Arithmetic, U <: TagQueueTag with Data] } } - assert(meshRows*tileRows == meshColumns*tileColumns) - val block_size = meshRows*tileRows + val req = Reg(UDValid(new MeshWithDelaysReq(accType, tagType, block_size))) - val active = RegInit(0.U(1.W)) // Which buffer is currently being read from? - val not_active = (~active).asUInt() + val matmul_id = RegInit(0.U(log2Up(max_simultaneous_matmuls).W)) - val flushing = RegInit(false.B) - val flushing_or_about_to = flushing || io.flush.fire() - - val fire_counter = RegInit(0.U((log2Ceil(block_size) max 1).W)) - val fire_started = RegInit(false.B) + val total_fires = req.bits.total_rows + val fire_counter = RegInit(0.U(log2Up(block_size).W)) val a_buf = RegEnable(io.a.bits, io.a.fire()) val b_buf = RegEnable(io.b.bits, io.b.fire()) val d_buf = RegEnable(io.d.bits, io.d.fire()) - val in_prop_reg = Reg(UInt(1.W)) // TODO inelegant - val in_prop = WireInit(in_prop_reg) - val a_written = RegInit(false.B) val b_written = RegInit(false.B) val d_written = RegInit(false.B) - val tag_written = RegInit(false.B) + val in_prop = Reg(UInt(1.W)) // TODO inelegant - val buffering_done = fire_counter === 0.U && fire_started && tag_written - val waiting_on_non_matrix_inputs = fire_counter === 0.U && !(tag_written || io.tag_in.fire()) // TODO change when more non-matrix inputs are buffered + val input_next_row_into_spatial_array = req.valid && ((a_written && b_written && d_written) || req.bits.flush > 0.U) + + val last_fire = fire_counter === total_fires - 1.U && input_next_row_into_spatial_array + + when (io.req.fire()) { + req.push(io.req.bits) + in_prop := io.req.bits.pe_control.propagate ^ in_prop + matmul_id := wrappingAdd(matmul_id, 1.U, max_simultaneous_matmuls) + }.elsewhen (last_fire) { + req.valid := req.bits.flush > 1.U + req.bits.flush := req.bits.flush - 1.U + } + + when (input_next_row_into_spatial_array) { + a_written := false.B + b_written := false.B + d_written := false.B + + fire_counter := wrappingAdd(fire_counter, 1.U, total_fires) + } when (io.a.fire()) { a_written := true.B @@ -110,66 +142,47 @@ class MeshWithDelays[T <: Data: Arithmetic, U <: TagQueueTag with Data] d_written := true.B } - val next_row_input = (io.a.fire() || a_written) && (io.b.fire() || b_written) && (io.d.fire() || d_written) - - when (next_row_input || flushing_or_about_to) { - a_written := false.B - b_written := false.B - d_written := false.B - - fire_counter := wrappingAdd(fire_counter, 1.U, block_size) - fire_started := true.B // We only need to write to this here, rather than in a "when (buffering_done)" statement - } + io.a.ready := !a_written || input_next_row_into_spatial_array || io.req.ready + io.b.ready := !b_written || input_next_row_into_spatial_array || io.req.ready + io.d.ready := !d_written || input_next_row_into_spatial_array || io.req.ready - io.a.ready := !a_written - io.b.ready := !b_written - io.d.ready := !d_written + assert(req.valid || !input_next_row_into_spatial_array) - val pause = (waiting_on_non_matrix_inputs || !next_row_input) && !flushing_or_about_to + val pause = !req.valid || !input_next_row_into_spatial_array // Transposer - val a_is_from_transposer = Mux(io.pe_control.dataflow === Dataflow.OS.id.U, !io.a_transpose, io.a_transpose) - val b_is_from_transposer = io.pe_control.dataflow === Dataflow.OS.id.U && io.bd_transpose - val d_is_from_transposer = io.pe_control.dataflow === Dataflow.WS.id.U && io.bd_transpose + val a_is_from_transposer = Mux(req.bits.pe_control.dataflow === Dataflow.OS.id.U, !req.bits.a_transpose, req.bits.a_transpose) + val b_is_from_transposer = req.bits.pe_control.dataflow === Dataflow.OS.id.U && req.bits.bd_transpose + val d_is_from_transposer = req.bits.pe_control.dataflow === Dataflow.WS.id.U && req.bits.bd_transpose val transposer = Module(new AlwaysOutTransposer(block_size, inputType)) transposer.io.inRow.valid := !pause && (a_is_from_transposer || b_is_from_transposer || d_is_from_transposer) - // transposer.io.inRow.bits := VecInit( - // Mux(a_is_from_transposer, Mux(io.a.fire(), io.a.bits, a_buf), Mux(io.b.fire(), io.b.bits, b_buf)).flatten) - transposer.io.inRow.bits := MuxCase(VecInit(Mux(io.a.fire(), io.a.bits, a_buf).flatten), Seq( - b_is_from_transposer -> VecInit(Mux(io.b.fire(), io.b.bits, b_buf).flatten), - d_is_from_transposer -> VecInit(Mux(io.d.fire(), io.d.bits, d_buf).flatten.reverse) + transposer.io.inRow.bits := MuxCase(VecInit(a_buf.flatten), Seq( + b_is_from_transposer -> VecInit(b_buf.flatten), + d_is_from_transposer -> VecInit(d_buf.flatten.reverse), )) transposer.io.outCol.ready := true.B val transposer_out = VecInit(transposer.io.outCol.bits.grouped(tileRows).map(t => VecInit(t)).toSeq) // Wire up mesh's IO to this module's IO - val mesh = Module(new Mesh(inputType, outputType, accType, df, pe_latency, tileRows, tileColumns, meshRows, meshColumns)) + val mesh = Module(new Mesh(inputType, outputType, accType, df, pe_latency, max_simultaneous_matmuls, tileRows, tileColumns, meshRows, meshColumns)) // TODO wire only to *_buf here, instead of io.*.bits - - /*val a_shifter_in = WireInit(Mux(io.pe_control.dataflow === Dataflow.OS.id.U, - a_transposed, Mux(io.a.fire(), io.a.bits, a_buf)))*/ - val a_shifter_in = WireInit(Mux(a_is_from_transposer, - transposer_out, Mux(io.a.fire(), io.a.bits, a_buf))) - // val b_shifter_in = WireInit(Mux(io.b.fire(), io.b.bits, b_buf)) - val b_shifter_in = WireInit(Mux(b_is_from_transposer, - transposer_out, Mux(io.b.fire(), io.b.bits, b_buf))) - // val d_shifter_in = Mux(io.d.fire(), io.d.bits, d_buf) + val a_shifter_in = WireInit(Mux(a_is_from_transposer, transposer_out, a_buf)) + val b_shifter_in = WireInit(Mux(b_is_from_transposer, transposer_out, b_buf)) val d_shifter_in = WireInit(Mux(d_is_from_transposer, - VecInit(transposer_out.flatten.reverse.grouped(tileRows).map(VecInit(_)).toSeq), - Mux(io.d.fire(), io.d.bits, d_buf))) + VecInit(transposer_out.flatten.reverse.grouped(tileRows).map(VecInit(_)).toSeq), d_buf)) mesh.io.in_a := shifted(a_shifter_in, leftBanks) mesh.io.in_b := shifted(b_shifter_in, upBanks) mesh.io.in_d := shifted(d_shifter_in, upBanks) mesh.io.in_control.zipWithIndex.foreach { case (ss, i) => - ss.foreach(_.dataflow := ShiftRegister(io.pe_control.dataflow, i * (pe_latency + 1))) + ss.foreach(_.dataflow := ShiftRegister(req.bits.pe_control.dataflow, i * (pe_latency + 1))) ss.foreach(_.propagate := ShiftRegister(in_prop, i * (pe_latency + 1))) } - val result_shift = RegNext(io.pe_control.shift) // TODO will this arrive at the right time if memory isn't pipelined? + val result_shift = RegNext(req.bits.pe_control.shift) // TODO will this arrive at the right time if memory isn't pipelined? mesh.io.in_control.zipWithIndex.foreach { case (ctrl, i) => ctrl.foreach(_.shift := ShiftRegister(result_shift, i * (pe_latency + 1))) } @@ -177,87 +190,73 @@ class MeshWithDelays[T <: Data: Arithmetic, U <: TagQueueTag with Data] val not_paused_vec = VecInit(Seq.fill(meshColumns)(VecInit(Seq.fill(tileColumns)(!pause)))) mesh.io.in_valid := shifted(not_paused_vec, upBanks) - // We want to output C when we're output-stationary, but B when we're weight-stationary - // TODO these would actually overlap when we switch from output-stationary to weight-stationary - // TODO should we use io.m, or the mode output of the mesh? - io.out.bits := shifted(Mux(io.pe_control.dataflow === Dataflow.OS.id.U, mesh.io.out_c, mesh.io.out_b), outBanks, true) - - io.out.valid := shifted(mesh.io.out_valid, outBanks, reverse = true)(0)(0) + val matmul_id_vec = VecInit(Seq.fill(meshColumns)(VecInit(Seq.fill(tileColumns)(matmul_id)))) + mesh.io.in_id := shifted(matmul_id_vec, upBanks) - // Tags - val tag_queue = Module(new TagQueue(tagqlen, tagType)) // TODO understand the actual required size better + val matmul_last_vec = VecInit(Seq.fill(meshColumns)(VecInit(Seq.fill(tileColumns)(last_fire)))) + mesh.io.in_last := shifted(matmul_last_vec, upBanks) - val tag_garbage = Wire(tagType.cloneType) - tag_garbage := DontCare - tag_garbage.make_this_garbage() + // We want to output C when we're output-stationary, but B when we're weight-stationary + // TODO these would actually overlap when we switch from output-stationary to weight-stationary + val out_pe_control = shifted(mesh.io.out_control, outBanks, reverse = true)(0)(0) + io.resp.bits.data := shifted(Mux(out_pe_control.dataflow === Dataflow.OS.id.U, mesh.io.out_c, mesh.io.out_b), outBanks, true) - tag_queue.io.in.bits := Mux(flushing, tag_garbage, io.tag_in.bits) + io.resp.valid := shifted(mesh.io.out_valid, outBanks, reverse = true)(0)(0) - val tag_id_reg = RegInit(0.U(1.W)) // Used to keep track of when we should increment // TODO inelegant - val tag_id = WireInit(tag_id_reg) - val tag_id_delayed = ShiftRegister(tag_id, (meshRows + S_TYPE.size - 1) * (pe_latency + 1) + 1, 0.U, true.B) + val out_last = shifted(mesh.io.out_last, outBanks, reverse = true)(0)(0) + io.resp.bits.last := out_last - tag_queue.io.out.next := tag_id_delayed =/= RegNext(tag_id_delayed, 0.U) + // Tags + class TagWithIdAndTotalRows extends Bundle with TagQueueTag { + val tag = tagType.cloneType + val id = UInt(log2Up(max_simultaneous_matmuls).W) + val total_rows = UInt(log2Up(block_size+1).W) + + override def make_this_garbage(dummy: Int=0): Unit = { + total_rows := block_size.U + tag.make_this_garbage() + } - when (io.tag_in.fire()) { - tag_written := true.B - tag_id := ~tag_id_reg - tag_id_reg := tag_id + override def cloneType: TagWithIdAndTotalRows.this.type = (new TagWithIdAndTotalRows).asInstanceOf[this.type] } - io.tag_in.ready := !tag_written - tag_queue.io.in.valid := io.tag_in.fire() - - io.tag_out := tag_queue.io.out.bits(Mux(io.pe_control.dataflow === Dataflow.OS.id.U, 0.U, 1.U)) - io.tags_in_progress := tag_queue.io.out.all - - // Flipping logic - when(buffering_done && (next_row_input || flushing_or_about_to)) { - active := not_active - io.tag_in.ready := true.B - tag_written := io.tag_in.fire() + val matmul_id_of_output = wrappingAdd(matmul_id, Mux(io.req.bits.pe_control.dataflow === Dataflow.OS.id.U, 3.U, 2.U), max_simultaneous_matmuls) + val matmul_id_of_current = wrappingAdd(matmul_id, 1.U, max_simultaneous_matmuls) - tag_id := ~tag_id_reg - tag_id_reg := tag_id + val tagq = Module(new TagQueue(new TagWithIdAndTotalRows, tagqlen)) + tagq.io.enq.valid := io.req.fire() && io.req.bits.flush === 0.U + tagq.io.enq.bits.tag := io.req.bits.tag + tagq.io.enq.bits.total_rows := DontCare + tagq.io.enq.bits.id := matmul_id_of_output - when (!flushing) { - in_prop := io.pe_control.propagate ^ in_prop_reg - in_prop_reg := in_prop - } - } + val tag_garbage = Wire(tagType.cloneType) + tag_garbage := DontCare + tag_garbage.make_this_garbage() - // Flushing logic - val flush_counter = Reg(UInt(2.W)) + val out_matmul_id = WireInit(shifted(mesh.io.out_id, outBanks, reverse = true)(0)(0)) + io.resp.bits.tag := Mux(tagq.io.deq.valid && out_matmul_id === tagq.io.deq.bits.id, tagq.io.deq.bits.tag, tag_garbage) - io.flush.ready := !flushing - // assert(!(io.flush.valid && !buffering_done)) // TODO get rid of this once we get the ability to ignore D + dontTouch(out_matmul_id) - when (io.flush.fire()) { - flushing := true.B - flush_counter := io.flush.bits + tagq.io.deq.ready := io.resp.valid && io.resp.bits.last && out_matmul_id === tagq.io.deq.bits.id - // Avoid overwriting accumulated values - a_buf := 0.U.asTypeOf(A_TYPE) // TODO make 0 an Arithmetic member function - b_buf := 0.U.asTypeOf(B_TYPE) - a_shifter_in := 0.U.asTypeOf(A_TYPE) - b_shifter_in := 0.U.asTypeOf(B_TYPE) - } + val total_rows_q = Module(new Queue(new TagWithIdAndTotalRows, tagqlen)) + total_rows_q.io.enq.valid := io.req.fire() && io.req.bits.flush === 0.U + total_rows_q.io.enq.bits.tag := DontCare + total_rows_q.io.enq.bits.total_rows := io.req.bits.total_rows + total_rows_q.io.enq.bits.id := matmul_id_of_current - when (flushing) { - Seq(io.a.ready, io.b.ready, io.d.ready, io.tag_in.ready).foreach(_ := false.B) + io.resp.bits.total_rows := Mux(total_rows_q.io.deq.valid && out_matmul_id === total_rows_q.io.deq.bits.id, + total_rows_q.io.deq.bits.total_rows, block_size.U) - tag_written := true.B + total_rows_q.io.deq.ready := io.resp.valid && io.resp.bits.last && out_matmul_id === total_rows_q.io.deq.bits.id - when (buffering_done) { - flush_counter := flush_counter - 1.U - tag_queue.io.in.valid := true.B - } + io.req.ready := (!req.valid || last_fire) && tagq.io.enq.ready && total_rows_q.io.enq.ready + io.tags_in_progress := tagq.io.all.map(_.tag) - val about_to_finish_flushing = flush_counter === 0.U && fire_counter === (block_size-1).U // TODO change when non-square requirement lifted - when (about_to_finish_flushing) { - fire_counter := 0.U - tag_queue.io.in.valid := true.B - flushing := false.B - } + when (reset.toBool()) { + req.valid := false.B } + + assert(!(io.req.fire() && !tagq.io.enq.ready && io.req.bits.flush === 0.U)) } diff --git a/src/main/scala/gemmini/PE.scala b/src/main/scala/gemmini/PE.scala index 7c17cc394..79944b729 100644 --- a/src/main/scala/gemmini/PE.scala +++ b/src/main/scala/gemmini/PE.scala @@ -17,7 +17,7 @@ class PEControl[T <: Data : Arithmetic](accType: T) extends Bundle { * A PE implementing a MAC operation. Configured as fully combinational when integrated into a Mesh. * @param width Data width of operands */ -class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, latency: Int) +class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, latency: Int, max_simultaneous_matmuls: Int) (implicit ev: Arithmetic[T]) extends Module { // Debugging variables import ev._ @@ -32,6 +32,12 @@ class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, val in_control = Input(new PEControl(accType)) val out_control = Output(new PEControl(accType)) + val in_id = Input(UInt(log2Up(max_simultaneous_matmuls).W)) + val out_id = Output(UInt(log2Up(max_simultaneous_matmuls).W)) + + val in_last = Input(Bool()) + val out_last = Output(Bool()) + val in_valid = Input(Bool()) val out_valid = Output(Bool()) @@ -48,12 +54,16 @@ class PE[T <: Data](inputType: T, outputType: T, accType: T, df: Dataflow.Value, val dataflow = ShiftRegister(io.in_control.dataflow, latency) val prop = ShiftRegister(io.in_control.propagate, latency) val shift = ShiftRegister(io.in_control.shift, latency) + val id = ShiftRegister(io.in_id, latency) + val last = ShiftRegister(io.in_last, latency) val valid = ShiftRegister(io.in_valid, latency) // TODO should we clockgate the rest of the ShiftRegisters based on the values in this ShiftRegisters io.out_a := a io.out_control.dataflow := dataflow io.out_control.propagate := prop io.out_control.shift := shift + io.out_id := id + io.out_last := last io.out_valid := valid val last_s = RegEnable(prop, valid) diff --git a/src/main/scala/gemmini/TagQueue.scala b/src/main/scala/gemmini/TagQueue.scala index e7460b2e2..3c516ff0b 100644 --- a/src/main/scala/gemmini/TagQueue.scala +++ b/src/main/scala/gemmini/TagQueue.scala @@ -8,45 +8,45 @@ trait TagQueueTag { def make_this_garbage(dummy: Int = 0): Unit } -class TagQueue[T <: TagQueueTag with Data](entries: Int, t: T) extends Module { +class TagQueue[T <: Data with TagQueueTag](t: T, entries: Int) extends Module { val io = IO(new Bundle { - val in = new Bundle { - val valid = Input(Bool()) - val bits = Input(t) - } - - val out = new Bundle { - val next = Input(Bool()) - val bits = Output(Vec(2, t)) - val all = Output(Vec(entries, t)) - } - - // This should really be a constructor parameter, but Chisel errors out when it is - // val garbage = Input(t) + val enq = Flipped(Decoupled(t.cloneType)) + val deq = Decoupled(t.cloneType) + val all = Output(Vec(entries, t.cloneType)) }) - // val regs = RegInit(VecInit(Seq.fill(entries)(io.garbage))) val regs = Reg(Vec(entries, t.cloneType)) - val raddr = RegInit(0.U((log2Ceil(entries) max 1).W)) - val waddr = RegInit(3.U((log2Ceil(entries) max 1).W)) + val raddr = RegInit(0.U(log2Up(entries).W)) + val waddr = RegInit(0.U(log2Up(entries).W)) + val len = RegInit(0.U(log2Up(entries+1).W)) - val raddr_inc = wrappingAdd(raddr, 1.U, entries) - val raddr_inc2 = wrappingAdd(raddr, 2.U, entries) + val empty = len === 0.U + val full = len === entries.U - io.out.bits(0) := Mux(io.out.next, regs(raddr_inc), regs(raddr)) - io.out.bits(1) := Mux(io.out.next, regs(raddr_inc2), regs(raddr_inc)) - io.out.all := regs + io.enq.ready := !full + io.deq.valid := !empty + io.deq.bits := regs(raddr) + io.all := regs - when (io.in.valid) { + when (io.enq.fire()) { + regs(waddr) := io.enq.bits waddr := wrappingAdd(waddr, 1.U, entries) - regs(waddr) := io.in.bits } - when (io.out.next) { - raddr := raddr_inc + when (io.deq.fire()) { + regs(raddr).make_this_garbage() + raddr := wrappingAdd(raddr, 1.U, entries) + } + + when (io.enq.fire() && !io.deq.fire()) { + len := len + 1.U + }.elsewhen(!io.enq.fire() && io.deq.fire()) { + len := len - 1.U } when (reset.toBool()) { regs.foreach(_.make_this_garbage()) } + + assert(len <= entries.U) } diff --git a/src/main/scala/gemmini/Tile.scala b/src/main/scala/gemmini/Tile.scala index 1a2bfe745..59807893e 100644 --- a/src/main/scala/gemmini/Tile.scala +++ b/src/main/scala/gemmini/Tile.scala @@ -12,16 +12,23 @@ import chisel3.util._ * @param rows Number of PEs on each row * @param columns Number of PEs on each column */ -class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: Dataflow.Value, pe_latency: Int, val rows: Int, val columns: Int) extends Module { +class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: Dataflow.Value, pe_latency: Int, max_simultaneous_matmuls: Int, val rows: Int, val columns: Int) extends Module { val io = IO(new Bundle { val in_a = Input(Vec(rows, inputType)) val in_b = Input(Vec(columns, outputType)) // This is the output of the tile next to it val in_d = Input(Vec(columns, outputType)) + val in_control = Input(Vec(columns, new PEControl(accType))) + val in_id = Input(Vec(columns, UInt(log2Up(max_simultaneous_matmuls).W))) + val in_last = Input(Vec(columns, Bool())) + val out_a = Output(Vec(rows, inputType)) val out_c = Output(Vec(columns, outputType)) val out_b = Output(Vec(columns, outputType)) + val out_control = Output(Vec(columns, new PEControl(accType))) + val out_id = Output(Vec(columns, UInt(log2Up(max_simultaneous_matmuls).W))) + val out_last = Output(Vec(columns, Bool())) val in_valid = Input(Vec(columns, Bool())) val out_valid = Output(Vec(columns, Bool())) @@ -29,7 +36,7 @@ class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: val bad_dataflow = Output(Bool()) }) - val tile = Seq.fill(rows, columns)(Module(new PE(inputType, outputType, accType, df, pe_latency))) + val tile = Seq.fill(rows, columns)(Module(new PE(inputType, outputType, accType, df, pe_latency, max_simultaneous_matmuls))) val tileT = tile.transpose // TODO: abstract hori/vert broadcast, all these connections look the same @@ -78,11 +85,31 @@ class Tile[T <: Data : Arithmetic](inputType: T, outputType: T, accType: T, df: } } + // Broadcast 'id' vertically across the Tile + for (c <- 0 until columns) { + tileT(c).foldLeft(io.in_id(c)) { + case (id, pe) => + pe.io.in_id := id + pe.io.out_id + } + } + + // Broadcast 'last' vertically across the Tile + for (c <- 0 until columns) { + tileT(c).foldLeft(io.in_last(c)) { + case (last, pe) => + pe.io.in_last := last + pe.io.out_last + } + } + // Drive the Tile's bottom IO for (c <- 0 until columns) { io.out_c(c) := tile(rows-1)(c).io.out_c io.out_b(c) := tile(rows-1)(c).io.out_b io.out_control(c) := tile(rows-1)(c).io.out_control + io.out_id(c) := tile(rows-1)(c).io.out_id + io.out_last(c) := tile(rows-1)(c).io.out_last io.out_valid(c) := tile(rows-1)(c).io.out_valid } io.bad_dataflow := tile.map(_.map(_.io.bad_dataflow).reduce(_||_)).reduce(_||_) diff --git a/src/main/scala/gemmini/Util.scala b/src/main/scala/gemmini/Util.scala index dd837c7d6..511cfee28 100644 --- a/src/main/scala/gemmini/Util.scala +++ b/src/main/scala/gemmini/Util.scala @@ -91,6 +91,11 @@ object Util { Mux(enable, next, buf) } + def RegEnableThru[T <: Data](next: T, init: T, enable: Bool): T = { + val buf = RegEnable(next, init, enable) + Mux(enable, next, buf) + } + def maxOf(u1: UInt, u2: UInt): UInt = { Mux(u1 > u2, u1, u2) } From d39b7b4bf587dde3becff83ecd9cf14444fa641e Mon Sep 17 00:00:00 2001 From: Hasan Genc Date: Thu, 25 Mar 2021 01:07:03 +0400 Subject: [PATCH 074/123] Perform global-averaging on Gemmini (#90) ResNet50 utilization: 51% --- software/gemmini-rocc-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index 7b9e82baf..c4903ac3d 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit 7b9e82baf0c14df36f97854310c8ba354838d9ab +Subproject commit c4903ac3deead09541182adaea0104bf868915f1 From 38076658260f13334e6a48d67c0b7da85d3027bb Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 14:02:34 -0700 Subject: [PATCH 075/123] start working on profiling --- src/main/scala/gemmini/LoadController.scala | 3 +++ src/main/scala/gemmini/LoopMatmul.scala | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index ab668fdfd..d4382b3eb 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -47,6 +47,9 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val monitor_conflict = (cmd.bits.cmd.inst.funct === LOAD2_CMD || cmd.bits.cmd.inst.funct === LOAD_CMD) && cmd.bits.cmd.rs2(63) val monitor_conflict_start = monitor_conflict && cmd.bits.cmd.rs2(61) val monitor_conflict_end = monitor_conflict && cmd.bits.cmd.rs2(62) + //profiling + val profile_conflict_start = (cmd.bits.cmd.inst.funct === LOAD3_CMD) && cmd.bits.cmd.rs2(61) + val profile_conflict_end = (cmd.bits.cmd.inst.funct === LOAD3_CMD) && cmd.bits.cmd.rs2(62) val mstatus = cmd.bits.cmd.status diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 24de572f2..d73353f0d 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -260,11 +260,14 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val cols = (blocks * block_size.U) - Mux(j + blocks >= req.max_j, req.pad_j, 0.U) val rows = block_size.U - Mux(i === req.max_i-1.U, req.pad_i, 0.U) + //for conflict monitor profiling (ToDo: turn this off, use with conv as well) + val start_profile = (i === 1.U) && (j === 0.U) + val end_profile = (i === req.max_i - 1.U) && (j + blocks >= req.max_j) val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD3_CMD mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + mvin_cmd.rs2 := (end_profile << 62).asUInt() | (start_profile << 61).asUInt() | (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr io.req.ready := state === idle io.idle := state === idle From b6e61ad47edaf0174b3a6b5e81a822fd058f01ee Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 15:23:59 -0700 Subject: [PATCH 076/123] adding profiler --- src/main/scala/gemmini/DMA.scala | 33 +++++++++++++++++++++ src/main/scala/gemmini/LoadController.scala | 2 ++ src/main/scala/gemmini/Scratchpad.scala | 5 ++++ 3 files changed, 40 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 7c440a0ce..39a06bf69 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -31,6 +31,8 @@ class StreamReadRequest[U <: Data](spad_rows: Int, acc_rows: Int, mvin_scale_t_b val monitor_conflict = Bool() val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict_start = Bool() + val profile_conflict_end = Bool() override def cloneType: StreamReadRequest.this.type = new StreamReadRequest(spad_rows, acc_rows, mvin_scale_t_bits).asInstanceOf[this.type] } @@ -191,6 +193,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val monitor_conflict = Bool() val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict_start = Bool() + val profile_conflict_end = Bool() } // TODO Can we filter out the larger read_sizes here if the systolic array is small, in the same way that we do so @@ -214,6 +218,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf packet.monitor_conflict := req.monitor_conflict packet.monitor_conflict_start := req.monitor_conflict_start packet.monitor_conflict_end := req.monitor_conflict_end + packet.profile_conflict_end := req.profile_conflict_end + packet.profile_conflict_start := req.profile_conflict_start packet } @@ -228,6 +234,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val read_monitor = read_packet.monitor_conflict val read_monitor_start = read_packet.monitor_conflict_start val read_monitor_end = read_packet.monitor_conflict_end + val profile_start = read_packet.profile_conflict_start + val profile_end = read_packet.profile_conflict_end // Firing off TileLink read requests and allocating space inside the reservation buffer for them val get = edge.Get( @@ -245,6 +253,9 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val monitor_conflict = Output(Bool()) val monitor_conflict_start = Output(Bool()) val monitor_conflict_end = Output(Bool()) + + val profile_conflict_start = Output(Bool()) + val profile_conflict_end = Output(Bool()) } val untranslated_a = Wire(Decoupled(new TLBundleAWithInfo)) @@ -288,6 +299,28 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf assert(retry_a.ready) val tl_miss = tl.a.valid && !tl.a.ready + val tl_profile_start = translate_q.io.deq.bits.profile_conflict_start + val tl_profile_end = translate_q.io.deq.bits.profile_conflict_end + val (p_reset :: p_profile_start :: Nil) = Enum(2) + val profile_miss_counter = RegInit(0.U(7.W)) + val p_state = RegInit(s_reset) + when(p_state === p_reset){ + when(tl_profile_start){ + p_state := p_profile_start + } + } + when(p_state === p_profile_start){ + when(tl_miss){ + profile_miss_counter := profile_miss_counter + 1.U //which counter to use? + }.otherwise{ + profile_miss_counter := 0.U + } + when(tl_profile_end){ + p_state === p_reset + profile_miss_counter := 0.U + } + } + val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(io.alert_cycles) diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index d4382b3eb..881bab6ff 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -105,6 +105,8 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig io.dma.req.bits.monitor_conflict := monitor_conflict io.dma.req.bits.monitor_conflict_start := monitor_conflict_start io.dma.req.bits.monitor_conflict_end := monitor_conflict_end + io.dma.req.bits.profile_conflict_start := profile_conflict_start + io.dma.req.bits.profile_conflict_end := profile_conflict_end // Command tracker IO cmd_tracker.io.alloc.valid := control_state === waiting_for_command && cmd.valid && DoLoad diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 5008dcb93..a83847ea6 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -29,6 +29,9 @@ class ScratchpadMemReadRequest[U <: Data](local_addr_t: LocalAddr, scale_t_bits: val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict_start = Bool() + val profile_conflict_end = Bool() + override def cloneType: this.type = new ScratchpadMemReadRequest(local_addr_t, scale_t_bits).asInstanceOf[this.type] } @@ -317,6 +320,8 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, reader.module.io.req.bits.monitor_conflict := read_issue_q.io.deq.bits.monitor_conflict reader.module.io.req.bits.monitor_conflict_end := read_issue_q.io.deq.bits.monitor_conflict_end reader.module.io.req.bits.monitor_conflict_start := read_issue_q.io.deq.bits.monitor_conflict_start + reader.module.io.req.bits.profile_conflict_end := read_issue_q.io.deq.bits.profile_conflict_end + reader.module.io.req.bits.profile_conflict_start := read_issue_q.io.deq.bits.profile_conflict_start val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier( config.mvin_scale_args, From 133a4d43c0746ba69f0af139c530c187d08164fb Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 15:29:19 -0700 Subject: [PATCH 077/123] fixing bugs --- src/main/scala/gemmini/DMA.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 39a06bf69..3cc0175ec 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -303,7 +303,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_profile_end = translate_q.io.deq.bits.profile_conflict_end val (p_reset :: p_profile_start :: Nil) = Enum(2) val profile_miss_counter = RegInit(0.U(7.W)) - val p_state = RegInit(s_reset) + val p_state = RegInit(p_reset) when(p_state === p_reset){ when(tl_profile_start){ p_state := p_profile_start @@ -320,6 +320,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf profile_miss_counter := 0.U } } + dontTouch(profile_miss_counter) val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) From 125192a5f98ad7fb3e1cc7a367f9f211845d300d Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 17:33:19 -0700 Subject: [PATCH 078/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 3cc0175ec..4f3f171ec 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -267,6 +267,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf untranslated_a.bits.monitor_conflict := read_monitor untranslated_a.bits.monitor_conflict_start := read_monitor_start untranslated_a.bits.monitor_conflict_end := read_monitor_end + untranslated_a.bits.profile_conflict_end := profile_end + untranslated_a.bits.profile_conflict_start := profile_start // 0 goes to retries, 1 goes to state machine val retry_a = Wire(Decoupled(new TLBundleAWithInfo)) From 4a982ceae9ef03559f24e9deca3a8ee85ddc708e Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 18:57:35 -0700 Subject: [PATCH 079/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 4f3f171ec..daf1b7101 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -318,7 +318,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf profile_miss_counter := 0.U } when(tl_profile_end){ - p_state === p_reset + p_state := p_reset profile_miss_counter := 0.U } } From 6c5b02c241f47b59f6739c745440df847e2fee22 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 23:25:06 -0700 Subject: [PATCH 080/123] changing profiling to looploader --- src/main/scala/gemmini/DMA.scala | 9 ++++++++- src/main/scala/gemmini/LoadController.scala | 6 ++++-- src/main/scala/gemmini/LoopLoader.scala | 12 ++++++++---- src/main/scala/gemmini/LoopMatmul.scala | 5 +---- src/main/scala/gemmini/Scratchpad.scala | 2 ++ 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index daf1b7101..b450c8c1c 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -31,6 +31,7 @@ class StreamReadRequest[U <: Data](spad_rows: Int, acc_rows: Int, mvin_scale_t_b val monitor_conflict = Bool() val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict = Bool() val profile_conflict_start = Bool() val profile_conflict_end = Bool() @@ -193,6 +194,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val monitor_conflict = Bool() val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict = Bool() val profile_conflict_start = Bool() val profile_conflict_end = Bool() } @@ -218,6 +220,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf packet.monitor_conflict := req.monitor_conflict packet.monitor_conflict_start := req.monitor_conflict_start packet.monitor_conflict_end := req.monitor_conflict_end + packet.profile_conflict := req.profile_conflict packet.profile_conflict_end := req.profile_conflict_end packet.profile_conflict_start := req.profile_conflict_start @@ -234,6 +237,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val read_monitor = read_packet.monitor_conflict val read_monitor_start = read_packet.monitor_conflict_start val read_monitor_end = read_packet.monitor_conflict_end + val profile = read_packet.profile_conflict val profile_start = read_packet.profile_conflict_start val profile_end = read_packet.profile_conflict_end @@ -254,6 +258,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val monitor_conflict_start = Output(Bool()) val monitor_conflict_end = Output(Bool()) + val profile_conflict = Output(Bool()) val profile_conflict_start = Output(Bool()) val profile_conflict_end = Output(Bool()) } @@ -267,6 +272,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf untranslated_a.bits.monitor_conflict := read_monitor untranslated_a.bits.monitor_conflict_start := read_monitor_start untranslated_a.bits.monitor_conflict_end := read_monitor_end + untranslated_a.bits.profile_conflict := profile untranslated_a.bits.profile_conflict_end := profile_end untranslated_a.bits.profile_conflict_start := profile_start @@ -301,6 +307,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf assert(retry_a.ready) val tl_miss = tl.a.valid && !tl.a.ready + val tl_profile = translate_q.io.deq.bits.profile_conflict val tl_profile_start = translate_q.io.deq.bits.profile_conflict_start val tl_profile_end = translate_q.io.deq.bits.profile_conflict_end val (p_reset :: p_profile_start :: Nil) = Enum(2) @@ -312,7 +319,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } } when(p_state === p_profile_start){ - when(tl_miss){ + when(tl_miss && tl_profile){ // and here? profile_miss_counter := profile_miss_counter + 1.U //which counter to use? }.otherwise{ profile_miss_counter := 0.U diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index 881bab6ff..6d4f83bdd 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -48,8 +48,9 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val monitor_conflict_start = monitor_conflict && cmd.bits.cmd.rs2(61) val monitor_conflict_end = monitor_conflict && cmd.bits.cmd.rs2(62) //profiling - val profile_conflict_start = (cmd.bits.cmd.inst.funct === LOAD3_CMD) && cmd.bits.cmd.rs2(61) - val profile_conflict_end = (cmd.bits.cmd.inst.funct === LOAD3_CMD) && cmd.bits.cmd.rs2(62) + val profile_conflict = (cmd.bits.cmd.inst.funct === LOAD2_CMD || cmd.bits.cmd.inst.funct === LOAD_CMD) && cmd.bits.cmd.rs2(47) + val profile_conflict_start = profile_conflict && cmd.bits.cmd.rs2(45) + val profile_conflict_end = profile_conflict && cmd.bits.cmd.rs2(46) val mstatus = cmd.bits.cmd.status @@ -107,6 +108,7 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig io.dma.req.bits.monitor_conflict_end := monitor_conflict_end io.dma.req.bits.profile_conflict_start := profile_conflict_start io.dma.req.bits.profile_conflict_end := profile_conflict_end + io.dma.req.bits.profile_conflict := profile_conflict // Command tracker IO cmd_tracker.io.alloc.valid := control_state === waiting_for_command && cmd.valid && DoLoad diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 167a97c82..e48b922d9 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -83,6 +83,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val max_blocks = max_block_len.asUInt() val AB = RegInit(false.B) //false if B, true if A + val profile = RegInit(false.B) //ToDo: rotate starting address like LoopMatmul.scala val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) @@ -116,17 +117,19 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val state = RegInit(idle) val configured = RegInit(false.B) - val conflict_monitor = !((alert_cycle === 0.U) || (latency === 0.U)) + val conflict_monitor = !(latency === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) && (state === ld) + val profile_start = profile && (row_iterator === 1.U && col_iterator === 0.U) + val profile_end = profile && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) //ToDo: either load A or B (for now just do with B) val load_cmd = Wire(new RoCCCommand()) load_cmd := DontCare load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr - load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr //for conv val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow @@ -170,7 +173,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock = unlock_monitor >= unlock_cycle - 1.U // ToDo: change this number io.out.bits := Mux(configured, Mux(is_conv, Mux(state === config, config_cmd, mvin_cmd), load_cmd), - Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && conflict_monitor, fixed_loop_cmd, cmd.bits)) + Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && (conflict_monitor || profile), fixed_loop_cmd, cmd.bits)) io.out.bits.status := cmd.bits.status io.out.valid := Mux(configured, state =/= idle, cmd.valid && !is_matmul_ldconfig && !is_conv_ldconfig) cmd.ready := Mux(is_matmul_ldconfig || is_conv_ldconfig, !configured, !configured && io.out.ready) @@ -187,6 +190,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I max_row_iterator := cmd.bits.rs2(iterator_bitwidth-1, 0) AB := cmd.bits.rs1(63) + profile := cmd.bits.rs1(62) //added for profiling cache behavior col_pad := cmd.bits.rs1(iterator_bitwidth * 2 - 1, iterator_bitwidth) row_pad := cmd.bits.rs1(iterator_bitwidth-1, 0) is_conv := false.B @@ -195,7 +199,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(!pause_req || unlock) { dram_base_addr := cmd.bits.rs1 row_stride := cmd.bits.rs2 - when(conflict_monitor) { // if latency == 0, don't unroll + when(conflict_monitor || profile) { // if latency == 0, don't unroll configured := true.B state := ld }.otherwise { diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index d73353f0d..24de572f2 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -260,14 +260,11 @@ class LoopMatmulLdD(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In val cols = (blocks * block_size.U) - Mux(j + blocks >= req.max_j, req.pad_j, 0.U) val rows = block_size.U - Mux(i === req.max_i-1.U, req.pad_i, 0.U) - //for conflict monitor profiling (ToDo: turn this off, use with conv as well) - val start_profile = (i === 1.U) && (j === 0.U) - val end_profile = (i === req.max_i - 1.U) && (j + blocks >= req.max_j) val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD3_CMD mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := (end_profile << 62).asUInt() | (start_profile << 61).asUInt() | (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + mvin_cmd.rs2 := (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr io.req.ready := state === idle io.idle := state === idle diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index a83847ea6..12b8b9766 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -29,6 +29,7 @@ class ScratchpadMemReadRequest[U <: Data](local_addr_t: LocalAddr, scale_t_bits: val monitor_conflict_start = Bool() val monitor_conflict_end = Bool() + val profile_conflict = Bool() val profile_conflict_start = Bool() val profile_conflict_end = Bool() @@ -322,6 +323,7 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, reader.module.io.req.bits.monitor_conflict_start := read_issue_q.io.deq.bits.monitor_conflict_start reader.module.io.req.bits.profile_conflict_end := read_issue_q.io.deq.bits.profile_conflict_end reader.module.io.req.bits.profile_conflict_start := read_issue_q.io.deq.bits.profile_conflict_start + reader.module.io.req.bits.profile_conflict := read_issue_q.io.deq.bits.profile_conflict val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier( config.mvin_scale_args, From 8d3b2149a7e3abb1eb3f2d4e116548476fe1b1ff Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 4 Apr 2021 23:46:15 -0700 Subject: [PATCH 081/123] debugging --- src/main/scala/gemmini/LoopLoader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index e48b922d9..07d2d6ae8 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -122,7 +122,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) && (state === ld) - val profile_start = profile && (row_iterator === 1.U && col_iterator === 0.U) + val profile_start = profile && (row_iterator === 0.U && col_iterator === 0.U) val profile_end = profile && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) //ToDo: either load A or B (for now just do with B) val load_cmd = Wire(new RoCCCommand()) From cda843f5dcf4f54b4f0bcaaca4630c0bea9cc679 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 00:55:00 -0700 Subject: [PATCH 082/123] debugging --- src/main/scala/gemmini/LoadController.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoadController.scala b/src/main/scala/gemmini/LoadController.scala index 6d4f83bdd..d3a072756 100644 --- a/src/main/scala/gemmini/LoadController.scala +++ b/src/main/scala/gemmini/LoadController.scala @@ -36,7 +36,8 @@ class LoadController[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig val cmd = Queue(io.cmd, ld_queue_length) val vaddr = cmd.bits.cmd.rs1 val localaddr = cmd.bits.cmd.rs2.asTypeOf(local_addr_t) - val cols = cmd.bits.cmd.rs2(32 + mvin_cols_bits - 1, 32) // TODO magic numbers + //val cols = cmd.bits.cmd.rs2(32 + mvin_cols_bits - 1, 32) // TODO magic numbers + val cols = cmd.bits.cmd.rs2(44, 32) //val rows = cmd.bits.cmd.rs2(48 + mvin_rows_bits - 1, 48) // TODO magic numbers val rows = cmd.bits.cmd.rs2(60, 48) // TODO magic numbers val config_stride = cmd.bits.cmd.rs2 From ca0fe4f83bdfc0b9e63c04c522fc8a679b37b667 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 12:57:16 -0700 Subject: [PATCH 083/123] adding more profile function --- src/main/scala/gemmini/DMA.scala | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index b450c8c1c..f8b27fa89 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -313,6 +313,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val (p_reset :: p_profile_start :: Nil) = Enum(2) val profile_miss_counter = RegInit(0.U(7.W)) val p_state = RegInit(p_reset) + val profile_number = RegInit(0.U(5.W)) + val profile_total = RegInit(0.U(10.W)) + val profile_cycle = RegInit(0.U(6.W)) + val profile_detected = RegInit(false.B) when(p_state === p_reset){ when(tl_profile_start){ p_state := p_profile_start @@ -320,13 +324,23 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } when(p_state === p_profile_start){ when(tl_miss && tl_profile){ // and here? + when(profile_miss_counter === 5.U){ //only count those that are over 5 cycles (to avoid false detection) + profile_number := profile_number + 1.U + profile_detected := true.B + } profile_miss_counter := profile_miss_counter + 1.U //which counter to use? }.otherwise{ + when(profile_detected){ + profile_total := profile_total + profile_miss_counter + profile_detected := false.B + } profile_miss_counter := 0.U } when(tl_profile_end){ + profile_detected := false.B p_state := p_reset profile_miss_counter := 0.U + profile_cycle := profile_total / profile_number // ToDo: need to change (don't use division) } } dontTouch(profile_miss_counter) From 286d2a28b5700372615d184d61c736234078ed63 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 13:29:33 -0700 Subject: [PATCH 084/123] added dontTouch --- src/main/scala/gemmini/DMA.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index f8b27fa89..4684b5da8 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -344,6 +344,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } } dontTouch(profile_miss_counter) + dontTouch(profile_cycle) + dontTouch(profile_detected) + dontTouch(profile_total) + dontTouch(profile_number) val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) From 0f8eb29b6bcf73aeac2b3df4b2ab0f50a484ef6e Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 16:56:55 -0700 Subject: [PATCH 085/123] increase bitwidth --- src/main/scala/gemmini/DMA.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 4684b5da8..8324ee06f 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -313,9 +313,9 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val (p_reset :: p_profile_start :: Nil) = Enum(2) val profile_miss_counter = RegInit(0.U(7.W)) val p_state = RegInit(p_reset) - val profile_number = RegInit(0.U(5.W)) - val profile_total = RegInit(0.U(10.W)) - val profile_cycle = RegInit(0.U(6.W)) + val profile_number = RegInit(0.U(9.W)) + val profile_total = RegInit(0.U(12.W)) + val profile_cycle = RegInit(0.U(7.W)) val profile_detected = RegInit(false.B) when(p_state === p_reset){ when(tl_profile_start){ From 0533ccb8ff23b4aeef5dd4d9f031bbd549b9d65a Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 19:38:31 -0700 Subject: [PATCH 086/123] add maximum, fix profiling start bug --- src/main/scala/gemmini/DMA.scala | 12 +++++++++--- src/main/scala/gemmini/LoopLoader.scala | 11 ++++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 8324ee06f..a253d4412 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -315,7 +315,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val p_state = RegInit(p_reset) val profile_number = RegInit(0.U(9.W)) val profile_total = RegInit(0.U(12.W)) - val profile_cycle = RegInit(0.U(7.W)) + val profile_max = RegInit(0.U(7.W)) + val profile_average = RegInit(0.U(7.W)) + // if too far off, select average + val profile_cycle = Mux(profile_max > profile_average + 10.U, profile_average, profile_max) val profile_detected = RegInit(false.B) when(p_state === p_reset){ when(tl_profile_start){ @@ -324,7 +327,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } when(p_state === p_profile_start){ when(tl_miss && tl_profile){ // and here? - when(profile_miss_counter === 5.U){ //only count those that are over 5 cycles (to avoid false detection) + when(profile_miss_counter === 7.U){ //only count those that are over 5 cycles (to avoid false detection) profile_number := profile_number + 1.U profile_detected := true.B } @@ -332,6 +335,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf }.otherwise{ when(profile_detected){ profile_total := profile_total + profile_miss_counter + profile_max := Mux(profile_max < profile_miss_counter, profile_miss_counter, profile_max)//update to max value profile_detected := false.B } profile_miss_counter := 0.U @@ -340,11 +344,13 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf profile_detected := false.B p_state := p_reset profile_miss_counter := 0.U - profile_cycle := profile_total / profile_number // ToDo: need to change (don't use division) + profile_average := profile_total / profile_number // ToDo: need to change (don't use division) } } dontTouch(profile_miss_counter) dontTouch(profile_cycle) + dontTouch(profile_average) + dontTouch(profile_max) dontTouch(profile_detected) dontTouch(profile_total) dontTouch(profile_number) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 07d2d6ae8..7790e2b43 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -121,15 +121,18 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) && (state === ld) + val unlock_monitor = RegInit(0.U(4.W)) + val unlock_cycle = RegInit(3.U(4.W)) - val profile_start = profile && (row_iterator === 0.U && col_iterator === 0.U) - val profile_end = profile && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) + val profile_hit = profile && (unlock_cycle =/= 0.U) + val profile_start = profile_hit && (row_iterator === 0.U && col_iterator === 0.U) + val profile_end = profile_hit && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) //ToDo: either load A or B (for now just do with B) val load_cmd = Wire(new RoCCCommand()) load_cmd := DontCare load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr - load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr + load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr //for conv val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow @@ -159,8 +162,6 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I fixed_loop_cmd.rs1 := Mux(cmd.bits.inst.funct === LOOP_CONV_WS_CONFIG_5, 0.U, Mux(AB, 0.U, cmd.bits.rs1)) //if conv, weight fixed_loop_cmd.rs2 := Mux(is_conv, cmd.bits.rs2, Mux(AB, cmd.bits.rs2, 0.U)) //for now, not do input for conv - val unlock_monitor = RegInit(0.U(4.W)) - val unlock_cycle = RegInit(3.U(4.W)) unlock_monitor := floorAdd(unlock_monitor, 1.U, unlock_cycle + pause_turn - 1.U, pause_req && is_loop_ws_addr & lock_tag && cmd.fire()) when(!pause_req){ unlock_monitor := 0.U From 4e71bbcc1fade6d5a9cf799e7f74d607ad4519de Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 22:23:34 -0700 Subject: [PATCH 087/123] parameterize selection btw average and max --- src/main/scala/gemmini/DMA.scala | 6 +++--- src/main/scala/gemmini/LoopLoader.scala | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index a253d4412..ed5369cd6 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -302,6 +302,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf ///////////////////////////////////////////////////////////////////////////////////////// val conflict_detected = RegInit(false.B) + val pause_turn = RegInit(io.pause_turn) retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready || conflict_detected) retry_a.bits := translate_q.io.deq.bits assert(retry_a.ready) @@ -317,8 +318,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val profile_total = RegInit(0.U(12.W)) val profile_max = RegInit(0.U(7.W)) val profile_average = RegInit(0.U(7.W)) - // if too far off, select average - val profile_cycle = Mux(profile_max > profile_average + 10.U, profile_average, profile_max) + // either average or max + val profile_cycle = Mux(pause_turn === 1.U, profile_average + 10.U, profile_max + 1.U) //parameterize what to select val profile_detected = RegInit(false.B) when(p_state === p_reset){ when(tl_profile_start){ @@ -358,7 +359,6 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(io.alert_cycles) - val pause_turn = RegInit(io.pause_turn) val latency = RegInit(io.latency) tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 7790e2b43..99641fc34 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -124,7 +124,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock_monitor = RegInit(0.U(4.W)) val unlock_cycle = RegInit(3.U(4.W)) - val profile_hit = profile && (unlock_cycle =/= 0.U) + val profile_hit = profile && (pause_turn =/= 0.U) val profile_start = profile_hit && (row_iterator === 0.U && col_iterator === 0.U) val profile_end = profile_hit && (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks) //ToDo: either load A or B (for now just do with B) From 45e9679495ea48c3d90e81ce716ca0a5ef6e2f06 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 23:01:35 -0700 Subject: [PATCH 088/123] fix how to calculate latency --- src/main/scala/gemmini/DMA.scala | 3 ++- src/main/scala/gemmini/LoopLoader.scala | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index ed5369cd6..5bbb47300 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -359,7 +359,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(io.alert_cycles) - val latency = RegInit(io.latency) + //val latency = RegInit(io.latency) + val latency = io.latency * profile_average tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) when(tl_miss_counter >= alert_cycles){ //reached limit diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 99641fc34..62d613071 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -149,9 +149,10 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd.rs1 := dram_addr mvin_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr + val expected_tl_req = (max_addr / (2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle - io.latency := latency + io.latency := expected_tl_req//latency io.pause_turn := pause_turn // fix loop_ws command val loop_ws_state = RegInit(idle) From 3ffa0d4d28e70f30f1acb1b44943ed434209eaaf Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 23:14:32 -0700 Subject: [PATCH 089/123] debugging --- src/main/scala/gemmini/DMA.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 5bbb47300..5c3e7ea96 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -379,9 +379,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf when(m_state === s_reset) { when(translate_q.io.deq.bits.monitor_conflict_start){ // to avoid false detection m_state := s_monitor_start - //pause_monitor_start := pause_monitor_start + 1.U alert_cycles := io.alert_cycles - latency := io.latency + //latency := io.latency //delared latency above pause_turn := io.pause_turn } }.elsewhen(m_state === s_monitor_start){ From 47323af7c33c42dfd690d9a4fe73039454bbd0b8 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 5 Apr 2021 23:51:21 -0700 Subject: [PATCH 090/123] debugging, added reset signals --- src/main/scala/gemmini/Scratchpad.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/scala/gemmini/Scratchpad.scala b/src/main/scala/gemmini/Scratchpad.scala index 12b8b9766..7c2fc9269 100644 --- a/src/main/scala/gemmini/Scratchpad.scala +++ b/src/main/scala/gemmini/Scratchpad.scala @@ -324,6 +324,14 @@ class Scratchpad[T <: Data, U <: Data, V <: Data](config: GemminiArrayConfig[T, reader.module.io.req.bits.profile_conflict_end := read_issue_q.io.deq.bits.profile_conflict_end reader.module.io.req.bits.profile_conflict_start := read_issue_q.io.deq.bits.profile_conflict_start reader.module.io.req.bits.profile_conflict := read_issue_q.io.deq.bits.profile_conflict + when(reset.toBool()){ + reader.module.io.req.bits.profile_conflict := false.B + reader.module.io.req.bits.profile_conflict_start := false.B + reader.module.io.req.bits.profile_conflict_end := false.B + reader.module.io.req.bits.monitor_conflict := false.B + reader.module.io.req.bits.monitor_conflict_end := false.B + reader.module.io.req.bits.monitor_conflict_start := false.B + } val (mvin_scale_in, mvin_scale_out) = VectorScalarMultiplier( config.mvin_scale_args, From d67ebe50fadd8c281f5d47ee5022bbd0d6b72a0b Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 01:00:16 -0700 Subject: [PATCH 091/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 +- src/main/scala/gemmini/LoopLoader.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 5c3e7ea96..723d003da 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -360,7 +360,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(io.alert_cycles) //val latency = RegInit(io.latency) - val latency = io.latency * profile_average + val latency = 0.U//io.latency * profile_average tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) when(tl_miss_counter >= alert_cycles){ //reached limit diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 62d613071..3726cf6d6 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -149,7 +149,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd.rs1 := dram_addr mvin_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr - val expected_tl_req = (max_addr / (2*max_block_len)).asUInt() + val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle io.latency := expected_tl_req//latency From 667ec2da7ba9e4d0fbbff388f5ba3e2af28dc56e Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 01:16:20 -0700 Subject: [PATCH 092/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 723d003da..99316ae43 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -360,7 +360,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(io.alert_cycles) //val latency = RegInit(io.latency) - val latency = 0.U//io.latency * profile_average + val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, io.latency * profile_average, 0.U) tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) when(tl_miss_counter >= alert_cycles){ //reached limit From f2966f0af8087422010b7c6a768af418831dbf5a Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 02:28:58 -0700 Subject: [PATCH 093/123] debugging --- src/main/scala/gemmini/DMA.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 99316ae43..5bd402d99 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -319,7 +319,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val profile_max = RegInit(0.U(7.W)) val profile_average = RegInit(0.U(7.W)) // either average or max - val profile_cycle = Mux(pause_turn === 1.U, profile_average + 10.U, profile_max + 1.U) //parameterize what to select + val profile_cycle = RegInit(profile_max) val profile_detected = RegInit(false.B) when(p_state === p_reset){ when(tl_profile_start){ @@ -346,6 +346,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf p_state := p_reset profile_miss_counter := 0.U profile_average := profile_total / profile_number // ToDo: need to change (don't use division) + profile_cycle := Mux(pause_turn === 1.U, profile_average + 10.U, profile_max + 1.U) //parameterize what to select } } dontTouch(profile_miss_counter) @@ -358,7 +359,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) - val alert_cycles = RegInit(io.alert_cycles) + val alert_cycles = profile_cycle //RegInit(io.alert_cycles) //val latency = RegInit(io.latency) val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, io.latency * profile_average, 0.U) @@ -379,7 +380,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf when(m_state === s_reset) { when(translate_q.io.deq.bits.monitor_conflict_start){ // to avoid false detection m_state := s_monitor_start - alert_cycles := io.alert_cycles + //alert_cycles := io.alert_cycles //latency := io.latency //delared latency above pause_turn := io.pause_turn } From d2eeae45dc62579a874c1065dccf14a54d106905 Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 03:09:56 -0700 Subject: [PATCH 094/123] added reset for translate enq --- src/main/scala/gemmini/DMA.scala | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 5bd402d99..27f604ee2 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -306,6 +306,14 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf retry_a.valid := translate_q.io.deq.valid && (io.tlb.resp.miss || !tl.a.ready || conflict_detected) retry_a.bits := translate_q.io.deq.bits assert(retry_a.ready) + when(reset.toBool()){ + translate_q.io.enq.bits.profile_conflict := false.B + translate_q.io.enq.bits.profile_conflict_start := false.B + translate_q.io.enq.bits.profile_conflict_end := false.B + translate_q.io.enq.bits.monitor_conflict := false.B + translate_q.io.enq.bits.monitor_conflict_end := false.B + translate_q.io.enq.bits.monitor_conflict_start := false.B + } val tl_miss = tl.a.valid && !tl.a.ready val tl_profile = translate_q.io.deq.bits.profile_conflict @@ -328,7 +336,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf } when(p_state === p_profile_start){ when(tl_miss && tl_profile){ // and here? - when(profile_miss_counter === 7.U){ //only count those that are over 5 cycles (to avoid false detection) + when(profile_miss_counter === 5.U){ //only count those that are over 5 cycles (to avoid false detection) profile_number := profile_number + 1.U profile_detected := true.B } @@ -346,7 +354,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf p_state := p_reset profile_miss_counter := 0.U profile_average := profile_total / profile_number // ToDo: need to change (don't use division) - profile_cycle := Mux(pause_turn === 1.U, profile_average + 10.U, profile_max + 1.U) //parameterize what to select + profile_cycle := Mux(pause_turn === 1.U, profile_average * 2.U, profile_max + 1.U) //parameterize what to select } } dontTouch(profile_miss_counter) From ae3e381bf87a8d27575e98c135c16890a094ee28 Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 03:32:30 -0700 Subject: [PATCH 095/123] more reset --- src/main/scala/gemmini/DMA.scala | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 27f604ee2..9d813883b 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -313,6 +313,18 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf translate_q.io.enq.bits.monitor_conflict := false.B translate_q.io.enq.bits.monitor_conflict_end := false.B translate_q.io.enq.bits.monitor_conflict_start := false.B + retry_a.bits.profile_conflict := false.B + retry_a.bits.profile_conflict_start := false.B + retry_a.bits.profile_conflict_end := false.B + retry_a.bits.monitor_conflict_end := false.B + retry_a.bits.monitor_conflict_start := false.B + retry_a.bits.monitor_conflict := false.B + untranslated_a.bits.profile_conflict := false.B + untranslated_a.bits.profile_conflict_start := false.B + untranslated_a.bits.profile_conflict_end := false.B + untranslated_a.bits.monitor_conflict_end := false.B + untranslated_a.bits.monitor_conflict_start := false.B + untranslated_a.bits.monitor_conflict := false.B } val tl_miss = tl.a.valid && !tl.a.ready From 99d89c8838d5793ccb30c9a5f82f8e79458c049a Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 12:09:48 -0700 Subject: [PATCH 096/123] debugging --- src/main/scala/gemmini/DMA.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 9d813883b..15c4904b5 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -329,7 +329,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_miss = tl.a.valid && !tl.a.ready val tl_profile = translate_q.io.deq.bits.profile_conflict - val tl_profile_start = translate_q.io.deq.bits.profile_conflict_start + val tl_profile_start = translate_q.io.deq.bits.profile_conflict_start && translate_q.io.deq.valid val tl_profile_end = translate_q.io.deq.bits.profile_conflict_end val (p_reset :: p_profile_start :: Nil) = Enum(2) val profile_miss_counter = RegInit(0.U(7.W)) @@ -366,7 +366,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf p_state := p_reset profile_miss_counter := 0.U profile_average := profile_total / profile_number // ToDo: need to change (don't use division) - profile_cycle := Mux(pause_turn === 1.U, profile_average * 2.U, profile_max + 1.U) //parameterize what to select + profile_cycle := Mux(io.pause_turn === 1.U, profile_average * 2.U, profile_max + 1.U) //parameterize what to select } } dontTouch(profile_miss_counter) @@ -398,7 +398,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf io.pause := pause_detect when(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.monitor_conflict_end){ when(m_state === s_reset) { - when(translate_q.io.deq.bits.monitor_conflict_start){ // to avoid false detection + when(translate_q.io.deq.bits.monitor_conflict_start && translate_q.io.deq.valid){ // to avoid false detection m_state := s_monitor_start //alert_cycles := io.alert_cycles //latency := io.latency //delared latency above From 9e946f3cb21d711071241f05c5972bc51c5c45d3 Mon Sep 17 00:00:00 2001 From: SeahK Date: Tue, 6 Apr 2021 13:22:40 -0700 Subject: [PATCH 097/123] remove division --- src/main/scala/gemmini/DMA.scala | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 15c4904b5..af7710a7f 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -358,6 +358,9 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf profile_total := profile_total + profile_miss_counter profile_max := Mux(profile_max < profile_miss_counter, profile_miss_counter, profile_max)//update to max value profile_detected := false.B + when(profile_number === 64.U){ + profile_average := profile_total / 64.U + } } profile_miss_counter := 0.U } @@ -365,7 +368,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf profile_detected := false.B p_state := p_reset profile_miss_counter := 0.U - profile_average := profile_total / profile_number // ToDo: need to change (don't use division) + when(profile_number === 64.U){ + profile_average := profile_total / 64.U + } + //profile_average := profile_total / profile_number // ToDo: need to change (don't use division) profile_cycle := Mux(io.pause_turn === 1.U, profile_average * 2.U, profile_max + 1.U) //parameterize what to select } } From a6f4ccb055c5fc695fe5f4628a742c5a776b9b07 Mon Sep 17 00:00:00 2001 From: SeahK Date: Wed, 7 Apr 2021 22:34:28 -0700 Subject: [PATCH 098/123] add dw mapping --- src/main/scala/gemmini/LoopConv.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 8e8fa4a58..af3eb52d6 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -303,6 +303,7 @@ class LoopConvLdWeightReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val loop_id = UInt(log2Up(concurrent_loops).W) val dram_padding = Bool() val dram_stride_divide = UInt(4.W) + val depthwise = Bool() } class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, @@ -346,7 +347,7 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) // Addresses - val dram_addr = req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * och_stride +& och) * (input_w/8).U + val dram_addr = Mux(req.depthwise, req.dram_addr +& ((krow*kernel_dim +& kcol +& kch) * och_stride +& och) * (input_w/8).U, req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * och_stride +& och) * (input_w/8).U) val spad_addr = addr_start + (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch // Sizes @@ -707,6 +708,7 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_bias = Bool() val no_pool = Bool() + val depthwise = Bool() val dram_ich_padding = Bool() val dram_och_padding = Bool() val dram_och_divide = UInt(4.W) @@ -928,6 +930,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.no_bias := cmd.bits.rs1(0) loop_being_configured.no_pool := cmd.bits.rs2(0) + loop_being_configured.depthwise := cmd.bits.rs2(63) loop_being_configured.dram_ich_padding := cmd.bits.rs2(1) loop_being_configured.dram_och_padding := cmd.bits.rs2(2) @@ -993,6 +996,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_weights.io.req.bits.loop_id := loop_requesting_ld_weights_id ld_weights.io.req.bits.dram_padding := loop_requesting_ld_weights.dram_och_padding ld_weights.io.req.bits.dram_stride_divide := loop_requesting_ld_weights.dram_och_divide + ld_weights.io.req.bits.depthwise := loop_requesting_ld_weights.depthwise ld_weights.io.req.valid := !loop_requesting_ld_weights.ld_weights_started && loop_requesting_ld_weights.configured From 98aa96763a8d69d5dc1b9c66163ef36d9e8c8f4e Mon Sep 17 00:00:00 2001 From: SeahK Date: Wed, 7 Apr 2021 23:16:25 -0700 Subject: [PATCH 099/123] looploader dw --- src/main/scala/gemmini/LoopLoader.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 3726cf6d6..0e0507262 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -89,6 +89,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv val och_divide = RegInit(1.U(4.W)) + val depthwise = RegInit(false.B) val total_out_channel = out_channels * och_divide val out_channel_stride = Mux(padding, total_out_channel + max_blocks * block_size.U, total_out_channel) val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) @@ -99,8 +100,9 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val sp_addr_start = Mux(is_conv, B_sp_addr_end - B_rows + block_size.U, Mux(AB, A_sp_addr_start, B_sp_addr_end - max_row_iterator * max_col_iterator * block_size.U + block_size.U)) // Todo: need mux with 0 (skip A) + val conv_dram_addr = Mux(depthwise, dram_base_addr +& ((krow*kernel_dim +& kcol +& kch) * out_channel_stride +& och) * (input_w/8).U, dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channel_stride +& och) * (input_w/8).U) val dram_addr = Mux(!is_conv, dram_base_addr + (row_iterator * row_stride + col_iterator) * block_size.U * (input_w/8).U, - dram_base_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * out_channel_stride +& och) * (input_w/8).U) + conv_dram_addr) val sp_addr = sp_addr_start + Mux(is_conv, (och / block_size.U) * krows * kcols * kchs + krow * kcols * kchs + kcol * kchs + kch, (row_iterator * max_col_iterator + col_iterator) * block_size.U) val blocks = Mux(col_iterator + max_blocks <= max_col_iterator, max_blocks, max_col_iterator-col_iterator) @@ -236,6 +238,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I dram_base_addr := cmd.bits.rs1 padding := cmd.bits.rs2(32) och_divide := cmd.bits.rs2(33) + depthwise := cmd.bits.rs2(63) out_channels := cmd.bits.rs2(31, 16) in_channels := cmd.bits.rs2(15, 0) //can code more From cff3e4d9ec65cb227b9f80aa56b1c97ca6261427 Mon Sep 17 00:00:00 2001 From: SeahK Date: Thu, 8 Apr 2021 02:51:26 -0700 Subject: [PATCH 100/123] fix monitoring trigger signal, added in_channel divide for dw conv --- src/main/scala/gemmini/DMA.scala | 4 ++-- src/main/scala/gemmini/LoopConv.scala | 7 ++++++- src/main/scala/gemmini/LoopLoader.scala | 9 +++++---- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index af7710a7f..06f6679ae 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -385,7 +385,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict val tl_miss_counter = RegInit(0.U(6.W)) - val alert_cycles = profile_cycle //RegInit(io.alert_cycles) + val alert_cycles = RegInit(profile_cycle) //val latency = RegInit(io.latency) val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, io.latency * profile_average, 0.U) @@ -406,7 +406,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf when(m_state === s_reset) { when(translate_q.io.deq.bits.monitor_conflict_start && translate_q.io.deq.valid){ // to avoid false detection m_state := s_monitor_start - //alert_cycles := io.alert_cycles + alert_cycles := Mux(io.alert_cycles === 0.U, profile_cycle, io.alert_cycles) //latency := io.latency //delared latency above pause_turn := io.pause_turn } diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index af3eb52d6..040ae0ed6 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -178,6 +178,7 @@ class LoopConvLdInputReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val addr_start = UInt(log2Up(max_acc_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) + val dram_stride_divide = UInt(4.W) val dram_padding = Bool() } @@ -220,7 +221,9 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw val irow_padded = irow +& upad.zext() val icol_padded = icol +& lpad.zext() val is_zeros = irow < 0.S || irow >= irows_unpadded.zext() || icol < 0.S || icol >= icols_unpadded.zext() - val ich_stride = Mux(req.dram_padding, in_channels + block_size.U * max_block_len.U, in_channels) + + val total_in_channels = in_channels * req.dram_stride_divide + val ich_stride = Mux(req.dram_padding, total_in_channels + block_size.U * max_block_len.U, total_in_channels) // Addresses val dram_addr = Mux(is_zeros, 0.U, @@ -978,6 +981,8 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_input.io.req.bits.dram_addr := loop_requesting_ld_input.input_dram_addr ld_input.io.req.bits.loop_id := loop_requesting_ld_input_id ld_input.io.req.bits.dram_padding := loop_requesting_ld_input.dram_ich_padding + // for dw conv, divide input channel when output channel is divided + ld_input.io.req.bits.dram_stride_divide := Mux(loop_requesting_ld_input.depthwise, loop_requesting_ld_input.dram_och_divide, 1.U) ld_input.io.req.valid := !loop_requesting_ld_input.ld_input_started && loop_requesting_ld_input.configured diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 0e0507262..3f1eca9df 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -119,12 +119,13 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val state = RegInit(idle) val configured = RegInit(false.B) - val conflict_monitor = !(latency === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) + val unlock_monitor = RegInit(0.U(4.W)) + val unlock_cycle = RegInit(0.U(4.W)) + val conflict_monitor = !(unlock_cycle === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), (row_iterator === max_row_iterator - 1.U && col_iterator >= max_col_iterator - max_blocks)) && (state === ld) - val unlock_monitor = RegInit(0.U(4.W)) - val unlock_cycle = RegInit(3.U(4.W)) + val profile_hit = profile && (pause_turn =/= 0.U) val profile_start = profile_hit && (row_iterator === 0.U && col_iterator === 0.U) @@ -174,7 +175,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I pause_req := io.pause_monitor } - val unlock = unlock_monitor >= unlock_cycle - 1.U // ToDo: change this number + val unlock = unlock_monitor + 1.U >= unlock_cycle // ToDo: change this number io.out.bits := Mux(configured, Mux(is_conv, Mux(state === config, config_cmd, mvin_cmd), load_cmd), Mux(lock_tag && is_loop_ws_addr && (!pause_req || unlock) && (conflict_monitor || profile), fixed_loop_cmd, cmd.bits)) From 4de86ce7c885bde486c4f737c3ae972ffd0430e1 Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 9 Apr 2021 02:20:42 -0700 Subject: [PATCH 101/123] divide output channel for squeezenet concat --- src/main/scala/gemmini/LoopConv.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 040ae0ed6..a9c929c32 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -565,6 +565,7 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val loop_id = UInt(log2Up(concurrent_loops).W) val dram_padding = Bool() val dram_stride_divide = UInt(4.W) + val out_channel_split = Bool() // for squeezenet } class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { @@ -604,7 +605,8 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val ocol = Reg(UInt(small_iterator_bitwidth.W)) val och = Reg(UInt(large_iterator_bitwidth.W)) - val total_out_channels = out_channels * req.dram_stride_divide + //further divide due to squeezenet fire module concatenation + val total_out_channels = Mux(req.out_channel_split, out_channels * req.dram_stride_divide * 2.U, out_channels * req.dram_stride_divide) val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) // Addresses val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U @@ -715,6 +717,7 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val dram_ich_padding = Bool() val dram_och_padding = Bool() val dram_och_divide = UInt(4.W) + val dram_out_split = Bool() //for squeezenet fire output val configured = Bool() @@ -938,6 +941,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.dram_ich_padding := cmd.bits.rs2(1) loop_being_configured.dram_och_padding := cmd.bits.rs2(2) loop_being_configured.dram_och_divide := cmd.bits.rs2(6,3) + loop_being_configured.dram_out_split := cmd.bits.rs2(12) loop_being_configured.configured := true.B } @@ -1043,6 +1047,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.loop_id := loop_requesting_st_id st.io.req.bits.dram_padding := loop_requesting_st.dram_och_padding st.io.req.bits.dram_stride_divide := loop_requesting_st.dram_och_divide + st.io.req.bits.out_channel_split := loop_requesting_st.dram_out_split st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured From 45f60d65125e07f8195fbe93ece09683fc743016 Mon Sep 17 00:00:00 2001 From: SeahK Date: Wed, 14 Apr 2021 13:08:26 -0700 Subject: [PATCH 102/123] change stride --- src/main/scala/gemmini/LoopConv.scala | 39 ++++++------------------- src/main/scala/gemmini/LoopLoader.scala | 8 ++--- 2 files changed, 11 insertions(+), 36 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index a9c929c32..33540c594 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -13,6 +13,9 @@ class LoopConvOuterBounds(val large_iterator_bitwidth: Int, val small_iterator_b val in_dim = UInt(small_iterator_bitwidth.W) val in_channels = UInt(large_iterator_bitwidth.W) val out_channels = UInt(large_iterator_bitwidth.W) + val out_stride = UInt(large_iterator_bitwidth.W) //stride for output activation + val in_stride = UInt(large_iterator_bitwidth.W) //stride for input activation + val weight_stride = UInt(large_iterator_bitwidth.W) //stride for weight val out_dim = UInt(small_iterator_bitwidth.W) val pool_out_dim = UInt(small_iterator_bitwidth.W) val stride = UInt(tiny_iterator_bitwidth.W) @@ -178,8 +181,6 @@ class LoopConvLdInputReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val addr_start = UInt(log2Up(max_acc_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) - val dram_stride_divide = UInt(4.W) - val dram_padding = Bool() } class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_addr: Int, input_w: Int, @@ -222,8 +223,7 @@ class LoopConvLdInput(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitw val icol_padded = icol +& lpad.zext() val is_zeros = irow < 0.S || irow >= irows_unpadded.zext() || icol < 0.S || icol >= icols_unpadded.zext() - val total_in_channels = in_channels * req.dram_stride_divide - val ich_stride = Mux(req.dram_padding, total_in_channels + block_size.U * max_block_len.U, total_in_channels) + val ich_stride = in_stride // Addresses val dram_addr = Mux(is_zeros, 0.U, @@ -304,8 +304,6 @@ class LoopConvLdWeightReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: val addr_end = UInt(log2Up(max_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val loop_id = UInt(log2Up(concurrent_loops).W) - val dram_padding = Bool() - val dram_stride_divide = UInt(4.W) val depthwise = Bool() } @@ -346,8 +344,7 @@ class LoopConvLdWeight(block_size: Int, coreMaxAddrBits: Int, large_iterator_bit val kcol = Reg(UInt(tiny_iterator_bitwidth.W)) val kch = Reg(UInt(large_iterator_bitwidth.W)) - val total_out_channels = out_channels * req.dram_stride_divide - val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) + val och_stride = weight_stride // Addresses val dram_addr = Mux(req.depthwise, req.dram_addr +& ((krow*kernel_dim +& kcol +& kch) * och_stride +& och) * (input_w/8).U, req.dram_addr +& ((krow*kernel_dim*in_channels +& kcol*in_channels +& kch) * och_stride +& och) * (input_w/8).U) @@ -563,9 +560,6 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val dram_addr = UInt(coreMaxAddrBits.W) val no_pool = Bool() val loop_id = UInt(log2Up(concurrent_loops).W) - val dram_padding = Bool() - val dram_stride_divide = UInt(4.W) - val out_channel_split = Bool() // for squeezenet } class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { @@ -606,8 +600,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val och = Reg(UInt(large_iterator_bitwidth.W)) //further divide due to squeezenet fire module concatenation - val total_out_channels = Mux(req.out_channel_split, out_channels * req.dram_stride_divide * 2.U, out_channels * req.dram_stride_divide) - val och_stride = Mux(req.dram_padding, total_out_channels + block_size.U * max_block_len.U, total_out_channels) + val och_stride = out_stride // Addresses val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol @@ -714,10 +707,6 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_bias = Bool() val no_pool = Bool() val depthwise = Bool() - val dram_ich_padding = Bool() - val dram_och_padding = Bool() - val dram_och_divide = UInt(4.W) - val dram_out_split = Bool() //for squeezenet fire output val configured = Bool() @@ -917,6 +906,9 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.inner_bounds.pupad := cmd.bits.rs1(31, 16) loop_being_configured.inner_bounds.pdpad := cmd.bits.rs1(15, 0) + loop_being_configured.outer_bounds.in_stride := cmd.bits.rs2(63, 48) + loop_being_configured.outer_bounds.weight_stride := cmd.bits.rs2(47, 32) + loop_being_configured.outer_bounds.out_stride := cmd.bits.rs2(31, 16) loop_being_configured.inner_bounds.ocols := cmd.bits.rs2(15, 0) } @@ -938,10 +930,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.no_pool := cmd.bits.rs2(0) loop_being_configured.depthwise := cmd.bits.rs2(63) - loop_being_configured.dram_ich_padding := cmd.bits.rs2(1) - loop_being_configured.dram_och_padding := cmd.bits.rs2(2) - loop_being_configured.dram_och_divide := cmd.bits.rs2(6,3) - loop_being_configured.dram_out_split := cmd.bits.rs2(12) loop_being_configured.configured := true.B } @@ -984,10 +972,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_input.io.req.bits.addr_start := loop_requesting_ld_input.a_addr_start ld_input.io.req.bits.dram_addr := loop_requesting_ld_input.input_dram_addr ld_input.io.req.bits.loop_id := loop_requesting_ld_input_id - ld_input.io.req.bits.dram_padding := loop_requesting_ld_input.dram_ich_padding - // for dw conv, divide input channel when output channel is divided - ld_input.io.req.bits.dram_stride_divide := Mux(loop_requesting_ld_input.depthwise, loop_requesting_ld_input.dram_och_divide, 1.U) - ld_input.io.req.valid := !loop_requesting_ld_input.ld_input_started && loop_requesting_ld_input.configured when (ld_input.io.req.fire()) { @@ -1003,8 +987,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_weights.io.req.bits.addr_end := loop_requesting_ld_weights.b_addr_end ld_weights.io.req.bits.dram_addr := loop_requesting_ld_weights.weights_dram_addr ld_weights.io.req.bits.loop_id := loop_requesting_ld_weights_id - ld_weights.io.req.bits.dram_padding := loop_requesting_ld_weights.dram_och_padding - ld_weights.io.req.bits.dram_stride_divide := loop_requesting_ld_weights.dram_och_divide ld_weights.io.req.bits.depthwise := loop_requesting_ld_weights.depthwise ld_weights.io.req.valid := !loop_requesting_ld_weights.ld_weights_started && loop_requesting_ld_weights.configured @@ -1045,9 +1027,6 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr st.io.req.bits.no_pool := loop_requesting_st.no_pool st.io.req.bits.loop_id := loop_requesting_st_id - st.io.req.bits.dram_padding := loop_requesting_st.dram_och_padding - st.io.req.bits.dram_stride_divide := loop_requesting_st.dram_och_divide - st.io.req.bits.out_channel_split := loop_requesting_st.dram_out_split st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 3f1eca9df..56db2eec8 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -73,7 +73,6 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val kcols = RegInit(0.U(4.W)) val kchs = RegInit(0.U(16.W)) val ochs = RegInit(0.U(16.W)) - val padding = RegInit(false.B) // SW padding for bank conflict // conv Iterators val och = RegInit(0.U(16.W)) @@ -88,10 +87,8 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val A_sp_addr_start = Mux(loop_tag, (max_addr/2).U, 0.U)//RegInit(0.U(log2Up(max_addr).W)) val B_sp_addr_end = Mux(loop_tag, (max_addr - block_size).U, (max_addr/2 - block_size).U)//RegInit((max_addr/2).U(log2Up(max_addr).W)) //for conv - val och_divide = RegInit(1.U(4.W)) val depthwise = RegInit(false.B) - val total_out_channel = out_channels * och_divide - val out_channel_stride = Mux(padding, total_out_channel + max_blocks * block_size.U, total_out_channel) + val out_channel_stride = RegInit(0.U(coreMaxAddrBits.W)) val max_ochs_per_mvin = Mux(ochs < (max_block_len * block_size).U, ochs, (max_block_len * block_size).U) val out_channels_per_bank = WireInit(0.U(8.W)) out_channels_per_bank := ochs / block_size.U +& (ochs % block_size.U =/= 0.U) @@ -237,8 +234,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I is(LOOP_CONV_LD_CONFIG_ADDRS){ when(!pause_req || unlock) { dram_base_addr := cmd.bits.rs1 - padding := cmd.bits.rs2(32) - och_divide := cmd.bits.rs2(33) + out_channel_stride := cmd.bits.rs2(47, 32) depthwise := cmd.bits.rs2(63) out_channels := cmd.bits.rs2(31, 16) in_channels := cmd.bits.rs2(15, 0) From 91505b451547464427fef9637e855b6acd4b1ab5 Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 16 Apr 2021 03:27:10 -0700 Subject: [PATCH 103/123] bump rocc-tests --- software/gemmini-rocc-tests | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/software/gemmini-rocc-tests b/software/gemmini-rocc-tests index c4903ac3d..e32f5be38 160000 --- a/software/gemmini-rocc-tests +++ b/software/gemmini-rocc-tests @@ -1 +1 @@ -Subproject commit c4903ac3deead09541182adaea0104bf868915f1 +Subproject commit e32f5be388e851b7cee073b39f848e8173872700 From 5ebb22493b3de74b55070f0d38cb16af37079880 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 18 Apr 2021 14:01:54 -0700 Subject: [PATCH 104/123] start working on 2 out --- src/main/scala/gemmini/LoopConv.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 33540c594..e60fbbdb5 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -558,7 +558,9 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val derived_params = new LoopConvDerivedParams(large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth) val addr_start = UInt(log2Up(max_acc_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) + val dram_addr_pool = UInt(coreMaxAddrBits.W) val no_pool = Bool() + val both_out = Bool() // output both pooled and unpooled val loop_id = UInt(log2Up(concurrent_loops).W) } @@ -703,9 +705,11 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val weights_dram_addr = UInt(coreMaxAddrBits.W) val input_dram_addr = UInt(coreMaxAddrBits.W) val output_dram_addr = UInt(coreMaxAddrBits.W) + val pool_output_dram_addr = UInt(coreMaxAddrBits.W) val no_bias = Bool() val no_pool = Bool() + val both_out = Bool() // both pool and not pool val depthwise = Bool() val configured = Bool() @@ -925,9 +929,11 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I } is (LOOP_CONV_WS) { - loop_being_configured.no_bias := cmd.bits.rs1(0) + loop_being_configured.pool_output_dram_addr := cmd.bits.rs1 // added for 2 mvout + loop_being_configured.no_bias := cmd.bits.rs2(61) loop_being_configured.no_pool := cmd.bits.rs2(0) + loop_being_configured.both_out := cmd.bits.rs2(62) loop_being_configured.depthwise := cmd.bits.rs2(63) @@ -1026,7 +1032,10 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.addr_start := st_addr_start st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr st.io.req.bits.no_pool := loop_requesting_st.no_pool + st.io.req.bits.both_out := loop_requesting_st.both_out st.io.req.bits.loop_id := loop_requesting_st_id + // added for 2 mvout + st.io.req.bits.dram_addr_pool := loop_requesting_st.pool_output_dram_addr st.io.req.valid := !loop_requesting_st.st_started && loop_requesting_st.ex_started && loop_requesting_st.configured From 31c8ce0fd17d65693462ec1235c95c62ac0fafa1 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 18 Apr 2021 15:03:32 -0700 Subject: [PATCH 105/123] adding pooling after normal output --- src/main/scala/gemmini/LoopConv.scala | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index e60fbbdb5..a371a7b9e 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -652,11 +652,12 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: io.cmd.bits := MuxLookup(state.asUInt, mvout_cmd, Seq(pre_pool_config.asUInt -> pre_pool_config_cmd, pool.asUInt -> pool_cmd, post_pool_config.asUInt -> post_pool_config_cmd)) + val second_pool = RegInit(false.B) //need to output pool next // Sending outputs when (skip) { state := idle }.elsewhen(io.cmd.fire()) { - when (req.no_pool) { + when (req.no_pool || (req.both_out && !second_pool)) { // needs normal output first before pool val next_och = floorAdd(och, block_size.U, ochs) val next_ocol = floorAdd(ocol, block_size.U, ocols, next_och === 0.U) val next_orow = floorAdd(orow, 1.U, orows, next_ocol === 0.U && next_och === 0.U) @@ -666,9 +667,11 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: ocol := next_ocol orow := next_orow b := next_b - - state := Mux(next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U, - idle, st) + val next_all_zero = next_b === 0.U && next_orow === 0.U && next_ocol === 0.U && next_och === 0.U + state := Mux(next_all_zero, Mux(req.both_out, pre_pool_config, idle), st) + when(next_all_zero && !second_pool && req.both_out){ + second_pool := true.B + } }.elsewhen(state === pre_pool_config) { state := pool }.elsewhen(state === post_pool_config) { @@ -682,6 +685,9 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: state := Mux(next_b === 0.U && next_och === 0.U, post_pool_config, pool) + when(next_b === 0.U && next_och === 0.U && req.both_out){ + second_pool := false.B + } } } From ccea00cfecf4a8f6c094cf2958a28888cf4f489c Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 18 Apr 2021 18:32:59 -0700 Subject: [PATCH 106/123] debugging --- src/main/scala/gemmini/LoopConv.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index a371a7b9e..359fbc70a 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -676,6 +676,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: state := pool }.elsewhen(state === post_pool_config) { state := idle + second_pool := false.B }.otherwise { val next_och = floorAdd(och, block_size.U, ochs) val next_b = floorAdd(b, 1.U, batches, next_och === 0.U) @@ -685,9 +686,6 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: state := Mux(next_b === 0.U && next_och === 0.U, post_pool_config, pool) - when(next_b === 0.U && next_och === 0.U && req.both_out){ - second_pool := false.B - } } } From 6994f372c35584ec8936ad1f65b3058f4b6e927b Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 18 Apr 2021 19:37:58 -0700 Subject: [PATCH 107/123] debugging --- src/main/scala/gemmini/LoopConv.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 359fbc70a..98e99885b 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -607,7 +607,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol - val pool_dram_addr = req.dram_addr + ((b * pool_out_dim * pool_out_dim) * och_stride + och) * (input_w/8).U + val pool_dram_addr = Mux(req.both_out, req.dram_addr_pool, req.dram_addr) + ((b * pool_out_dim * pool_out_dim) * och_stride + och) * (input_w/8).U val pool_spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols // Sizes From 3086b0402826d4a5c44d3d9b4e271793cd186f27 Mon Sep 17 00:00:00 2001 From: SeahK Date: Sun, 18 Apr 2021 20:06:30 -0700 Subject: [PATCH 108/123] debugging --- src/main/scala/gemmini/LoopConv.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 98e99885b..efc1d3400 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -692,7 +692,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: // Accepting requests when (io.req.fire()) { req := io.req.bits - state := Mux(io.req.bits.no_pool, st, pre_pool_config) + state := Mux(io.req.bits.no_pool || io.req.bits.both_out, st, pre_pool_config) b := 0.U orow := 0.U From 6570b111a4a4962554cd0990a90040ef16448088 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 00:27:29 -0700 Subject: [PATCH 109/123] fixing looploader --- src/main/scala/gemmini/DMA.scala | 5 ++++- src/main/scala/gemmini/LoopLoader.scala | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 06f6679ae..3b58cc01b 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -387,7 +387,10 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(profile_cycle) //val latency = RegInit(io.latency) - val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, io.latency * profile_average, 0.U) + val max_block_len = (maxBytes / (meshRows * spadWidth / 8)) max 1 + val expected_tl_req = (spad_rows / (2*2*max_block_len)).asUInt() + val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, + Mux(io.latency === 0.U, expected_tl_req * profile_average, 0.U), 0.U) //if latency not give, use profiled one tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) when(tl_miss_counter >= alert_cycles){ //reached limit diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 56db2eec8..d7186d0bf 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -118,6 +118,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock_monitor = RegInit(0.U(4.W)) val unlock_cycle = RegInit(0.U(4.W)) + val enable_bubble = RegInit(false.B) val conflict_monitor = !(unlock_cycle === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), @@ -132,7 +133,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I load_cmd := DontCare load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr - load_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr + load_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr //for conv val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow @@ -147,12 +148,12 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD2_CMD // for now, only weight mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := (conflict_monitor << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr + mvin_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle - io.latency := expected_tl_req//latency + io.latency := Mux(latency === 0.U, expected_tl_req, latency) io.pause_turn := pause_turn // fix loop_ws command val loop_ws_state = RegInit(idle) @@ -184,6 +185,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(cmd.valid && is_matmul_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_LD_CONFIG_BOUNDS){ + enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(iterator_bitwidth * 3 + 12, iterator_bitwidth * 3 + 10) alert_cycle := cmd.bits.rs2(iterator_bitwidth * 3 + 5, iterator_bitwidth * 3) latency := cmd.bits.rs2(iterator_bitwidth * 3 - 1, iterator_bitwidth * 2) //ToDo: give this to DMA From 7ec19bc07a4fc4bc673449f157377f34a876fe2a Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 01:02:40 -0700 Subject: [PATCH 110/123] debugging --- src/main/scala/gemmini/LoopLoader.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index d7186d0bf..276e8d951 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -215,6 +215,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I }.elsewhen(cmd.valid && is_conv_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_CONV_LD_CONFIG_BOUNDS){ + enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(60, 58) unlock_cycle := cmd.bits.rs2(57, 54) alert_cycle := cmd.bits.rs2(53, 48) From 9f61b44f68ae3da52d9e96f1ea515457a1eeb115 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 02:43:27 -0700 Subject: [PATCH 111/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 3b58cc01b..f53d8ef9c 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -390,7 +390,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val max_block_len = (maxBytes / (meshRows * spadWidth / 8)) max 1 val expected_tl_req = (spad_rows / (2*2*max_block_len)).asUInt() val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, - Mux(io.latency === 0.U, expected_tl_req * profile_average, 0.U), 0.U) //if latency not give, use profiled one + Mux(io.latency === 0.U, expected_tl_req * profile_average, io.latency), 0.U) //if latency not give, use profiled one tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) when(tl_miss_counter >= alert_cycles){ //reached limit From c107cb5fa53b233a514a273261742df08a7b1493 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 02:44:47 -0700 Subject: [PATCH 112/123] deleted counter --- src/main/scala/gemmini/DMA.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index f53d8ef9c..0ec23fe6d 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -446,6 +446,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss && !conflict_detected tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr + /* val cycles = freechips.rocketchip.util.WideCounter(32) when(tl.a.fire()){ printf("GEMMINI_MEM %x %x %x %x\n", cycles.value, p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address, tl.a.bits.size) @@ -456,6 +457,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf //printf(midas.targetutils.SynthesizePrintf("GEMMINI_BLOCK: %x %x \n", p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address)) } + + */ ///////////////////////////////////////////////////////////////////////////////////////// //tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss From 1a7fa4a6d9ee02b8b6563d5b6ed29b8ab76efcd2 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 03:17:52 -0700 Subject: [PATCH 113/123] debugging --- src/main/scala/gemmini/DMA.scala | 2 +- src/main/scala/gemmini/LoopLoader.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 0ec23fe6d..5e8bd78fb 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -457,7 +457,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf //printf(midas.targetutils.SynthesizePrintf("GEMMINI_BLOCK: %x %x \n", p(freechips.rocketchip.tile.TileKey).hartId.U, tl.a.bits.address)) } - + */ ///////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 276e8d951..7e6e85ff8 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -150,10 +150,10 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd.rs1 := dram_addr mvin_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr - val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() + //val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle - io.latency := Mux(latency === 0.U, expected_tl_req, latency) + io.latency := latency//Mux(latency === 0.U, expected_tl_req, latency) io.pause_turn := pause_turn // fix loop_ws command val loop_ws_state = RegInit(idle) From 2cde9ce154116d123220ee6e459a00819cfce8ed Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 12:27:03 -0700 Subject: [PATCH 114/123] changed latency --- src/main/scala/gemmini/DMA.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 5e8bd78fb..70b6528fd 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -387,8 +387,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(profile_cycle) //val latency = RegInit(io.latency) - val max_block_len = (maxBytes / (meshRows * spadWidth / 8)) max 1 - val expected_tl_req = (spad_rows / (2*2*max_block_len)).asUInt() + //val max_block_len = (maxBytes / (meshRows * spadWidth / 8)) max 1 + val expected_tl_req = (spad_rows / (2*2*4)).asUInt() val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, Mux(io.latency === 0.U, expected_tl_req * profile_average, io.latency), 0.U) //if latency not give, use profiled one From 60a1103b8ef2131538d71bf6754e867d418ae002 Mon Sep 17 00:00:00 2001 From: SeahK Date: Mon, 26 Apr 2021 13:16:45 -0700 Subject: [PATCH 115/123] changed latency --- src/main/scala/gemmini/DMA.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 70b6528fd..2cc6ccc2a 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -356,7 +356,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf }.otherwise{ when(profile_detected){ profile_total := profile_total + profile_miss_counter - profile_max := Mux(profile_max < profile_miss_counter, profile_miss_counter, profile_max)//update to max value + profile_max := Mux(profile_max < profile_miss_counter && !translate_q.io.deq.bits.profile_conflict_start, profile_miss_counter, profile_max)//update to max value profile_detected := false.B when(profile_number === 64.U){ profile_average := profile_total / 64.U From 123a980c0f9225e5f18c356bec633363acc210a3 Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 30 Apr 2021 13:18:41 -0700 Subject: [PATCH 116/123] code for partial sum move out --- src/main/scala/gemmini/LoopConv.scala | 6 +++++- src/main/scala/gemmini/LoopMatmul.scala | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index efc1d3400..0eb91caaf 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -561,6 +561,7 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val dram_addr_pool = UInt(coreMaxAddrBits.W) val no_pool = Bool() val both_out = Bool() // output both pooled and unpooled + val partial_sum = Bool() // move out 32 bits of partial sum val loop_id = UInt(log2Up(concurrent_loops).W) } @@ -621,7 +622,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: mvout_cmd := DontCare mvout_cmd.inst.funct := STORE_CMD mvout_cmd.rs1 := dram_addr - mvout_cmd.rs2 := (I << 48.U) | (J << 32.U) | spad_addr + mvout_cmd.rs2 := Mux(req.partial_sum, (I << 48.U) | (J << 32.U) | spad_addr | (1.U << 30), (I << 48.U) | (J << 32.U) | spad_addr) val pre_pool_config_cmd = Wire(new RoCCCommand) pre_pool_config_cmd := DontCare @@ -714,6 +715,7 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_bias = Bool() val no_pool = Bool() val both_out = Bool() // both pool and not pool + val partial_sum = Bool() val depthwise = Bool() val configured = Bool() @@ -938,6 +940,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I loop_being_configured.no_pool := cmd.bits.rs2(0) loop_being_configured.both_out := cmd.bits.rs2(62) + loop_being_configured.partial_sum := cmd.bits.rs2(60) loop_being_configured.depthwise := cmd.bits.rs2(63) @@ -1037,6 +1040,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr st.io.req.bits.no_pool := loop_requesting_st.no_pool st.io.req.bits.both_out := loop_requesting_st.both_out + st.io.req.bits.partial_sum := loop_requesting_st.partial_sum st.io.req.bits.loop_id := loop_requesting_st_id // added for 2 mvout st.io.req.bits.dram_addr_pool := loop_requesting_st.pool_output_dram_addr diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 24de572f2..7303c69f1 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -447,6 +447,7 @@ class LoopMatmulStCReq(val block_size: Int, val coreMaxAddrBits: Int, val iterat val dram_addr = UInt(coreMaxAddrBits.W) val dram_stride = UInt(coreMaxAddrBits.W) val full_c = Bool() + val partial_sum = Bool() // to move out partial sum val addr_start = UInt(log2Up(max_acc_addr).W) val loop_id = UInt(log2Up(concurrent_loops).W) } @@ -497,7 +498,7 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In mvout_cmd := DontCare mvout_cmd.inst.funct := STORE_CMD mvout_cmd.rs1 := dram_addr - mvout_cmd.rs2 := (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr + mvout_cmd.rs2 := Mux(req.partial_sum, (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr | (1.U << 30).asUInt(), (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr) io.req.ready := state === idle io.j := j @@ -563,6 +564,7 @@ class LoopMatmulState(val iterator_bitwidth: Int, val coreMaxAddrBits: Int, val val b_transpose = Bool() val low_d = Bool() + val partial_sum = Bool() //to moveout partial sum val full_c = Bool() val ex_accumulate = Bool() @@ -737,6 +739,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: loop_being_configured.ex_accumulate := cmd.bits.rs1(0) loop_being_configured.full_c := cmd.bits.rs1(1) loop_being_configured.low_d := cmd.bits.rs1(2) + loop_being_configured.partial_sum := cmd.bits.rs1(3) loop_being_configured.a_transpose := cmd.bits.rs2(0) loop_being_configured.b_transpose := cmd.bits.rs2(1) @@ -851,6 +854,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: stC.io.req.bits.dram_addr := loop_requesting_st.c_dram_addr stC.io.req.bits.dram_stride := loop_requesting_st.c_dram_stride stC.io.req.bits.full_c := loop_requesting_st.full_c + stC.io.req.bits.partial_sum := loop_requesting_st.partial_sum stC.io.req.bits.addr_start := st_c_addr_start stC.io.req.bits.loop_id := loop_requesting_st_id From e7feb86aa14551f3a7ea34d83db4e15bf7974a2d Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 30 Apr 2021 13:23:58 -0700 Subject: [PATCH 117/123] stride for partial sum --- src/main/scala/gemmini/LoopConv.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 0eb91caaf..9ed416e7e 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -565,7 +565,7 @@ class LoopConvStReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: Int, val loop_id = UInt(log2Up(concurrent_loops).W) } -class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { +class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: Int, small_iterator_bitwidth: Int, tiny_iterator_bitwidth: Int, max_acc_addr: Int, input_w: Int, acc_w: Int, max_block_len: Int, concurrent_loops: Int)(implicit p: Parameters) extends Module { val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow val io = IO(new Bundle { @@ -636,7 +636,8 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: post_pool_config_cmd := DontCare post_pool_config_cmd.inst.funct := CONFIG_CMD post_pool_config_cmd.rs1 := CONFIG_STORE - post_pool_config_cmd.rs2 := och_stride * (input_w / 8).U + post_pool_config_cmd.rs2 := Mux(req.partial_sum, och_stride * (acc_w / 8).U, och_stride * (input_w / 8).U) + //need 32 bits stride to move out partial sum val pool_cmd = Wire(new RoCCCommand) pool_cmd := DontCare @@ -820,7 +821,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I val ld_input = Module(new LoopConvLdInput(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) val ld_weights = Module(new LoopConvLdWeight(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, input_w, max_block_len, concurrent_loops)) val ex = Module(new LoopConvExecute(block_size, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_addr, max_acc_addr, concurrent_loops)) - val st = Module(new LoopConvSt(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, input_w, max_block_len, concurrent_loops)) + val st = Module(new LoopConvSt(block_size, coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth, max_acc_addr, input_w, acc_w, max_block_len, concurrent_loops)) // Create command queue val cmd = Queue(io.in) From f2d1841146e7b5b615484cfb77dc084ccae561cc Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 30 Apr 2021 14:59:00 -0700 Subject: [PATCH 118/123] debugging --- src/main/scala/gemmini/LoopConv.scala | 4 ++-- src/main/scala/gemmini/LoopMatmul.scala | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 9ed416e7e..438354a1c 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -605,7 +605,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: //further divide due to squeezenet fire module concatenation val och_stride = out_stride // Addresses - val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * (input_w/8).U + val dram_addr = req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * och_stride + och) * Mux(req.partial_sum, (acc_w/8).U, (input_w/8).U) val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol val pool_dram_addr = Mux(req.both_out, req.dram_addr_pool, req.dram_addr) + ((b * pool_out_dim * pool_out_dim) * och_stride + och) * (input_w/8).U @@ -622,7 +622,7 @@ class LoopConvSt(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwidth: mvout_cmd := DontCare mvout_cmd.inst.funct := STORE_CMD mvout_cmd.rs1 := dram_addr - mvout_cmd.rs2 := Mux(req.partial_sum, (I << 48.U) | (J << 32.U) | spad_addr | (1.U << 30), (I << 48.U) | (J << 32.U) | spad_addr) + mvout_cmd.rs2 := Mux(req.partial_sum, (I << 48.U) | (J << 32.U) | spad_addr | (1.U << 29), (I << 48.U) | (J << 32.U) | spad_addr) val pre_pool_config_cmd = Wire(new RoCCCommand) pre_pool_config_cmd := DontCare diff --git a/src/main/scala/gemmini/LoopMatmul.scala b/src/main/scala/gemmini/LoopMatmul.scala index 7303c69f1..3da100806 100644 --- a/src/main/scala/gemmini/LoopMatmul.scala +++ b/src/main/scala/gemmini/LoopMatmul.scala @@ -447,7 +447,7 @@ class LoopMatmulStCReq(val block_size: Int, val coreMaxAddrBits: Int, val iterat val dram_addr = UInt(coreMaxAddrBits.W) val dram_stride = UInt(coreMaxAddrBits.W) val full_c = Bool() - val partial_sum = Bool() // to move out partial sum + //val partial_sum = Bool() // to move out partial sum val addr_start = UInt(log2Up(max_acc_addr).W) val loop_id = UInt(log2Up(concurrent_loops).W) } @@ -498,7 +498,7 @@ class LoopMatmulStC(block_size: Int, coreMaxAddrBits: Int, iterator_bitwidth: In mvout_cmd := DontCare mvout_cmd.inst.funct := STORE_CMD mvout_cmd.rs1 := dram_addr - mvout_cmd.rs2 := Mux(req.partial_sum, (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr | (1.U << 30).asUInt(), (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr) + mvout_cmd.rs2 := (rows << 48).asUInt() | (cols << 32).asUInt() | sp_addr io.req.ready := state === idle io.j := j @@ -564,7 +564,7 @@ class LoopMatmulState(val iterator_bitwidth: Int, val coreMaxAddrBits: Int, val val b_transpose = Bool() val low_d = Bool() - val partial_sum = Bool() //to moveout partial sum + //val partial_sum = Bool() //to moveout partial sum val full_c = Bool() val ex_accumulate = Bool() @@ -739,7 +739,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: loop_being_configured.ex_accumulate := cmd.bits.rs1(0) loop_being_configured.full_c := cmd.bits.rs1(1) loop_being_configured.low_d := cmd.bits.rs1(2) - loop_being_configured.partial_sum := cmd.bits.rs1(3) + //loop_being_configured.partial_sum := cmd.bits.rs1(3) loop_being_configured.a_transpose := cmd.bits.rs2(0) loop_being_configured.b_transpose := cmd.bits.rs2(1) @@ -854,7 +854,7 @@ class LoopMatmul(block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: stC.io.req.bits.dram_addr := loop_requesting_st.c_dram_addr stC.io.req.bits.dram_stride := loop_requesting_st.c_dram_stride stC.io.req.bits.full_c := loop_requesting_st.full_c - stC.io.req.bits.partial_sum := loop_requesting_st.partial_sum + //stC.io.req.bits.partial_sum := loop_requesting_st.partial_sum stC.io.req.bits.addr_start := st_c_addr_start stC.io.req.bits.loop_id := loop_requesting_st_id From d54d314de1c7d30bcf4caf323eda72014b5050af Mon Sep 17 00:00:00 2001 From: SeahK Date: Fri, 30 Apr 2021 15:49:34 -0700 Subject: [PATCH 119/123] fixing bias for loopconv --- src/main/scala/gemmini/LoopConv.scala | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/scala/gemmini/LoopConv.scala b/src/main/scala/gemmini/LoopConv.scala index 438354a1c..7cf24a47a 100644 --- a/src/main/scala/gemmini/LoopConv.scala +++ b/src/main/scala/gemmini/LoopConv.scala @@ -71,6 +71,7 @@ class LoopConvLdBiasReq(val coreMaxAddrBits: Int, val large_iterator_bitwidth: I val addr_start = UInt(log2Up(max_acc_addr).W) val dram_addr = UInt(coreMaxAddrBits.W) val no_bias = Bool() + val partial_sum_mvin = Bool() //for partial sum move-in val loop_id = UInt(log2Up(concurrent_loops).W) } @@ -97,6 +98,7 @@ class LoopConvLdBias(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwi val req = Reg(new LoopConvLdBiasReq(coreMaxAddrBits, large_iterator_bitwidth, small_iterator_bitwidth, tiny_iterator_bitwidth: Int, max_acc_addr, concurrent_loops)) import req.inner_bounds._ + import req.outer_bounds._ import req.derived_params._ val acc_addr_start = (BigInt(1) << 31).U | req.addr_start @@ -113,7 +115,11 @@ class LoopConvLdBias(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwi val och = Reg(UInt(large_iterator_bitwidth.W)) // Addresses - val dram_addr = req.dram_addr +& och * (acc_w/8).U +// val dram_addr = req.dram_addr +& och * (acc_w/8).U + val dram_addr = Mux(req.partial_sum_mvin, req.dram_addr + ((b*out_dim*out_dim + orow*out_dim + ocol) * out_channels + och) * (acc_w/8).U, + req.dram_addr +& och * (acc_w/8).U) + //stride for partial sum: out_channels (not och_stride) + val spad_addr = acc_addr_start +& (och / block_size.U) * batches * orows * ocols +& b * orows * ocols +& orow * ocols +& ocol // Sizes @@ -125,7 +131,8 @@ class LoopConvLdBias(block_size: Int, coreMaxAddrBits: Int, large_iterator_bitwi config_cmd := DontCare config_cmd.inst.funct := CONFIG_CMD config_cmd.rs1 := (MVIN_SCALE_IDENTITY << 32.U) | (req.derived_params.bias_spad_stride << 16.U) | (2.U << 3) | 1.U - config_cmd.rs2 := 0.U + config_cmd.rs2 := Mux(req.partial_sum_mvin, out_channels * (acc_w/8).U, 0.U) + // to move in partial sum, need stride val mvin_cmd = Wire(new RoCCCommand) mvin_cmd := DontCare @@ -716,7 +723,8 @@ class LoopConvState(val block_size: Int, val large_iterator_bitwidth: Int, val s val no_bias = Bool() val no_pool = Bool() val both_out = Bool() // both pool and not pool - val partial_sum = Bool() + val partial_sum_mvout = Bool() + val partial_sum_mvin = Bool() val depthwise = Bool() val configured = Bool() @@ -938,10 +946,11 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I is (LOOP_CONV_WS) { loop_being_configured.pool_output_dram_addr := cmd.bits.rs1 // added for 2 mvout loop_being_configured.no_bias := cmd.bits.rs2(61) + loop_being_configured.partial_sum_mvin := cmd.bits.rs2(59) loop_being_configured.no_pool := cmd.bits.rs2(0) loop_being_configured.both_out := cmd.bits.rs2(62) - loop_being_configured.partial_sum := cmd.bits.rs2(60) + loop_being_configured.partial_sum_mvout := cmd.bits.rs2(60) loop_being_configured.depthwise := cmd.bits.rs2(63) @@ -963,6 +972,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I ld_bias.io.req.bits.addr_start := ld_bias_addr_start ld_bias.io.req.bits.dram_addr := loop_requesting_ld_bias.bias_dram_addr ld_bias.io.req.bits.no_bias := loop_requesting_ld_bias.no_bias + ld_bias.io.req.bits.partial_sum_mvin := loop_requesting_ld_bias.partial_sum_mvin ld_bias.io.req.bits.loop_id := loop_requesting_ld_bias_id @@ -1041,7 +1051,7 @@ class LoopConv (block_size: Int, coreMaxAddrBits: Int, rob_size: Int, max_lds: I st.io.req.bits.dram_addr := loop_requesting_st.output_dram_addr st.io.req.bits.no_pool := loop_requesting_st.no_pool st.io.req.bits.both_out := loop_requesting_st.both_out - st.io.req.bits.partial_sum := loop_requesting_st.partial_sum + st.io.req.bits.partial_sum := loop_requesting_st.partial_sum_mvout st.io.req.bits.loop_id := loop_requesting_st_id // added for 2 mvout st.io.req.bits.dram_addr_pool := loop_requesting_st.pool_output_dram_addr From 520268e29efdf538e1e7ca953b400b641c50e18f Mon Sep 17 00:00:00 2001 From: Seah Kim Date: Tue, 4 May 2021 14:25:02 -0700 Subject: [PATCH 120/123] changing bubble insertion part --- src/main/scala/gemmini/DMA.scala | 73 ++++++++++++++++++++----- src/main/scala/gemmini/LoopLoader.scala | 7 ++- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/src/main/scala/gemmini/DMA.scala b/src/main/scala/gemmini/DMA.scala index 2cc6ccc2a..2383f4e50 100644 --- a/src/main/scala/gemmini/DMA.scala +++ b/src/main/scala/gemmini/DMA.scala @@ -383,41 +383,89 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf dontTouch(profile_total) dontTouch(profile_number) - val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict + val tl_counter_trigger = tl_miss && translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.monitor_conflict_end val tl_miss_counter = RegInit(0.U(6.W)) val alert_cycles = RegInit(profile_cycle) - //val latency = RegInit(io.latency) + val latency = RegInit(io.latency) + val enable_bubble = RegInit(false.B) //val max_block_len = (maxBytes / (meshRows * spadWidth / 8)) max 1 val expected_tl_req = (spad_rows / (2*2*4)).asUInt() - val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, - Mux(io.latency === 0.U, expected_tl_req * profile_average, io.latency), 0.U) //if latency not give, use profiled one - - tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) + //val latency = Mux(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.profile_conflict, + // Mux(io.latency === 0.U, expected_tl_req * profile_average, io.latency), 0.U) //if latency not give, use profiled one + //tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_counter_trigger) + /* when(tl_miss_counter >= alert_cycles){ //reached limit conflict_detected := true.B }.elsewhen(!tl_counter_trigger){ tl_miss_counter := 0.U } + + */ // pause monitoring detecting logic val (s_reset :: s_monitor_start :: s_conflict_detected :: Nil) = Enum(3) val m_state = RegInit(s_reset) val pause_detect = RegInit(false.B) val pause_count = RegInit(0.U(2.W)) //Todo: parameterize it? + val tl_miss_timer = RegInit(0.U(16.W)) + /* + tl_miss_timer := floorAdd(tl_miss_timer, 1.U, latency + 1.U, conflict_detected) + when(tl_miss_timer === latency){ //resolve miss counter temporary + tl_miss_counter := 0.U //reset miss counter + conflict_detected := false.B + } + */ //val pause_monitor_start = RegInit(0.U(6.W)) io.pause := pause_detect when(translate_q.io.deq.bits.monitor_conflict && !translate_q.io.deq.bits.monitor_conflict_end){ when(m_state === s_reset) { + tl_miss_counter := 0.U + tl_miss_timer := 0.U when(translate_q.io.deq.bits.monitor_conflict_start && translate_q.io.deq.valid){ // to avoid false detection m_state := s_monitor_start alert_cycles := Mux(io.alert_cycles === 0.U, profile_cycle, io.alert_cycles) - //latency := io.latency //delared latency above + when(io.latency === 0.U){ + latency := expected_tl_req * profile_average // use profiled one + enable_bubble := true.B + }.elsewhen(io.latency === 1.U){ + latency := 0.U + enable_bubble := false.B + }.otherwise{ + enable_bubble := true.B + latency := io.latency + } pause_turn := io.pause_turn } }.elsewhen(m_state === s_monitor_start){ + tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_miss) when(tl_miss_counter >= alert_cycles){ m_state := s_conflict_detected + when(enable_bubble){ + conflict_detected := true.B + }.otherwise{ + tl_miss_counter := 0.U //resolve miss counter immediately (no bubbbles) + conflict_detected := false.B + } + }.elsewhen(!tl_miss){ + tl_miss_counter := 0.U } //pause_monitor_start := 0.U + }.elsewhen(m_state === s_conflict_detected){ + tl_miss_counter := satAdd(tl_miss_counter, 1.U, alert_cycles + 2.U, tl_miss) + tl_miss_timer := floorAdd(tl_miss_timer, 1.U, latency + 1.U, conflict_detected) + when(tl_miss_counter >= alert_cycles){ + when(enable_bubble){ + conflict_detected := true.B + }.otherwise{ + tl_miss_counter := 0.U //resolve miss counter immediately (no bubbbles) + conflict_detected := false.B + } + }.elsewhen(!tl_miss){ + tl_miss_counter := 0.U + } + when(tl_miss_timer === latency){ //resolve miss counter temporary + tl_miss_counter := 0.U //reset miss counter + conflict_detected := false.B + } } }.elsewhen(translate_q.io.deq.bits.monitor_conflict_end) { when(m_state === s_conflict_detected) { @@ -426,7 +474,7 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf pause_detect := false.B }.elsewhen(m_state === s_monitor_start) { // no detection during time window when(pause_count === pause_turn) { // pause monitoring - pause_detect := true.B // on 3rd time (ToDo: parameterize this?) + pause_detect := true.B m_state := s_reset pause_count := 0.U // reset pause counter }.otherwise { @@ -437,13 +485,8 @@ class StreamReaderCore[T <: Data, U <: Data, V <: Data](config: GemminiArrayConf //pause_monitor_start := 0.U } //ToDo: how to restart monitoring after pausing - val tl_miss_timer = RegInit(0.U(16.W)) - tl_miss_timer := floorAdd(tl_miss_timer, 1.U, latency + 1.U, conflict_detected) - when(tl_miss_timer === latency){ //resolve miss counter temporary - tl_miss_counter := 0.U //reset miss counter - conflict_detected := false.B - } - tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss && !conflict_detected + + tl.a.valid := translate_q.io.deq.valid && !io.tlb.resp.miss && !(conflict_detected) tl.a.bits := translate_q.io.deq.bits.tl_a tl.a.bits.address := io.tlb.resp.paddr /* diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 7e6e85ff8..3f1e5998c 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -133,7 +133,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I load_cmd := DontCare load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr - load_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr + load_cmd.rs2 := ((conflict_monitor) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr //for conv val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow @@ -148,12 +148,13 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD2_CMD // for now, only weight mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr + mvin_cmd.rs2 := ((conflict_monitor) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr //val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle - io.latency := latency//Mux(latency === 0.U, expected_tl_req, latency) + io.latency := Mux(enable_bubble, latency, 1.U) // latency + // not enable bubble -> DMA latency 1 (loopld+FSM without bubble) io.pause_turn := pause_turn // fix loop_ws command val loop_ws_state = RegInit(idle) From e18f5c3126505914f748fd411b6c3f231e2cd833 Mon Sep 17 00:00:00 2001 From: Seah Kim Date: Tue, 4 May 2021 17:49:50 -0700 Subject: [PATCH 121/123] debugging --- src/main/scala/gemmini/LoopLoader.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 3f1e5998c..dfd0f49b4 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -153,7 +153,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I //val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle - io.latency := Mux(enable_bubble, latency, 1.U) // latency + io.latency := latency//Mux(enable_bubble, latency, 1.U) // latency // not enable bubble -> DMA latency 1 (loopld+FSM without bubble) io.pause_turn := pause_turn // fix loop_ws command From b1c38c43971277f39edc30c1959b42526bffdcbf Mon Sep 17 00:00:00 2001 From: Seah Kim Date: Tue, 4 May 2021 18:23:39 -0700 Subject: [PATCH 122/123] delete bubble enable signal --- src/main/scala/gemmini/LoopLoader.scala | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index dfd0f49b4..104837996 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -118,7 +118,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock_monitor = RegInit(0.U(4.W)) val unlock_cycle = RegInit(0.U(4.W)) - val enable_bubble = RegInit(false.B) + //val enable_bubble = WireInit(false.B) val conflict_monitor = !(unlock_cycle === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), @@ -154,7 +154,8 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I io.busy := cmd.valid || configured io.alert_cycle := alert_cycle io.latency := latency//Mux(enable_bubble, latency, 1.U) // latency - // not enable bubble -> DMA latency 1 (loopld+FSM without bubble) + // enable_bubble := (latency =/= 0.U) //if latency == 0, disable bubble + // not enable bubble (loopld+FSM without bubble) io.pause_turn := pause_turn // fix loop_ws command val loop_ws_state = RegInit(idle) @@ -186,7 +187,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(cmd.valid && is_matmul_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_LD_CONFIG_BOUNDS){ - enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion + //enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(iterator_bitwidth * 3 + 12, iterator_bitwidth * 3 + 10) alert_cycle := cmd.bits.rs2(iterator_bitwidth * 3 + 5, iterator_bitwidth * 3) latency := cmd.bits.rs2(iterator_bitwidth * 3 - 1, iterator_bitwidth * 2) //ToDo: give this to DMA @@ -216,7 +217,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I }.elsewhen(cmd.valid && is_conv_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_CONV_LD_CONFIG_BOUNDS){ - enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion + //enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(60, 58) unlock_cycle := cmd.bits.rs2(57, 54) alert_cycle := cmd.bits.rs2(53, 48) From 0fcbe86405a8d5134b915fd68286ed544eda1c14 Mon Sep 17 00:00:00 2001 From: Seah Kim Date: Tue, 4 May 2021 18:29:52 -0700 Subject: [PATCH 123/123] add enable signal again --- src/main/scala/gemmini/LoopLoader.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/scala/gemmini/LoopLoader.scala b/src/main/scala/gemmini/LoopLoader.scala index 104837996..0f7cabe3d 100644 --- a/src/main/scala/gemmini/LoopLoader.scala +++ b/src/main/scala/gemmini/LoopLoader.scala @@ -118,7 +118,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I val unlock_monitor = RegInit(0.U(4.W)) val unlock_cycle = RegInit(0.U(4.W)) - //val enable_bubble = WireInit(false.B) + val enable_bubble = RegInit(false.B) // enable monitoring for cache hits val conflict_monitor = !(unlock_cycle === 0.U)//!((alert_cycle === 0.U) || (latency === 0.U)) val conflict_monitor_start = conflict_monitor && Mux(is_conv, (och === 0.U && kch === 0.U && kcol === 0.U && krow === 0.U), (row_iterator === 0.U && col_iterator === 0.U)) && (state === ld) //ToDo: with conv val conflict_monitor_end = conflict_monitor && Mux(is_conv, (kch + block_size.U >= kchs && kcol === kcols - 1.U && krow === krows - 1.U && och + max_ochs_per_mvin >= ochs), @@ -133,7 +133,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I load_cmd := DontCare load_cmd.inst.funct := Mux(AB, LOAD_CMD, LOAD2_CMD) load_cmd.rs1 := dram_addr - load_cmd.rs2 := ((conflict_monitor) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr + load_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (rows << 48).asUInt() | (profile_hit << 47).asUInt() | (profile_end << 46).asUInt() | (profile_start << 45).asUInt() | (cols << 32).asUInt() | sp_addr //for conv val MVIN_SCALE_IDENTITY = 0x3f800000.U // TODO get this from configs somehow @@ -148,13 +148,13 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I mvin_cmd := DontCare mvin_cmd.inst.funct := LOAD2_CMD // for now, only weight mvin_cmd.rs1 := dram_addr - mvin_cmd.rs2 := ((conflict_monitor) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr + mvin_cmd.rs2 := ((conflict_monitor && enable_bubble) << 63).asUInt() | (conflict_monitor_end << 62).asUInt() | (conflict_monitor_start << 61).asUInt() | (K << 48.U).asUInt() | (J << 32.U).asUInt() | sp_addr //val expected_tl_req = (max_addr / (2*2*max_block_len)).asUInt() io.busy := cmd.valid || configured io.alert_cycle := alert_cycle io.latency := latency//Mux(enable_bubble, latency, 1.U) // latency - // enable_bubble := (latency =/= 0.U) //if latency == 0, disable bubble + //enable_bubble := (latency =/= 0.U) //if latency == 0, disable bubble // not enable bubble (loopld+FSM without bubble) io.pause_turn := pause_turn // fix loop_ws command @@ -187,7 +187,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I when(cmd.valid && is_matmul_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_LD_CONFIG_BOUNDS){ - //enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion + enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(iterator_bitwidth * 3 + 12, iterator_bitwidth * 3 + 10) alert_cycle := cmd.bits.rs2(iterator_bitwidth * 3 + 5, iterator_bitwidth * 3) latency := cmd.bits.rs2(iterator_bitwidth * 3 - 1, iterator_bitwidth * 2) //ToDo: give this to DMA @@ -217,7 +217,7 @@ class LoopLoader(block_size: Int, coreMaxAddrBits:Int, max_addr: Int, input_w: I }.elsewhen(cmd.valid && is_conv_ldconfig && state === idle){ switch(cmd.bits.inst.funct){ is(LOOP_CONV_LD_CONFIG_BOUNDS){ - //enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion + enable_bubble := cmd.bits.rs2(63) //diable: just loop B without bubble insertion pause_turn := cmd.bits.rs2(60, 58) unlock_cycle := cmd.bits.rs2(57, 54) alert_cycle := cmd.bits.rs2(53, 48)