diff --git a/hardware/src/vlsu/align_stage.sv b/hardware/src/vlsu/align_stage.sv index cf18898..e656410 100644 --- a/hardware/src/vlsu/align_stage.sv +++ b/hardware/src/vlsu/align_stage.sv @@ -171,6 +171,8 @@ assign axi_resp_i_cut[0].r_valid = axi_resp_i.r_valid; // Handle Vector Loads // ///////////////////////// +logic is_load_aligned, is_load_resp_valid; + always_comb begin // Initialize state @@ -239,14 +241,11 @@ always_comb begin ///// Handling unaligned data using byte enable ///// - // If a stage receives a valid packet, shift the byte enable - for (int s=0; s < NumStages; s++) begin - if (axi_resp_o_cut[s].r_valid) begin - be_d[s+1] = tracker_q[rd_resp_pnt_q[s]].shift_en[s] ? be_q[s] >> (1 << s) : be_q[s]; - end - end be_final_d = be_q[NumStages]; + is_load_aligned = be_final_d[AxiDataBytes-1]; + is_load_resp_valid = axi_resp_i_cut[NumStages].r_valid; + ///// Handle incoming AXI responses ///// // Track the previous data packet and along with the byte enable @@ -259,7 +258,7 @@ always_comb begin axi_req_cut_ready[NumStages] = axi_req_i.r_ready; // For a valid handshake assign to buffer to be used later - if (axi_resp_i_cut[NumStages].r_valid && axi_req_i.r_ready) begin + if (is_load_resp_valid && axi_req_i.r_ready) begin // Buffer data in this cycle data_d = axi_resp_i_cut[NumStages].r.data; data_valid_d = 1'b1; @@ -277,7 +276,7 @@ always_comb begin // If misaligned, make sure you have a valid beat in the current cycle // or if the transaction is short that is check if previous beat was the last beat // Otherwise, we have a valid data if the request is aligned - automatic logic valid_data = (~be_final_d[AxiDataBytes-1] & (axi_resp_i_cut[NumStages].r_valid | last_q)) | be_final_d[AxiDataBytes-1]; + automatic logic valid_data = (~is_load_aligned & (is_load_resp_valid | last_q)) | is_load_aligned; for (int b=0; b> (1 << s) : be_q[s]; + end + end + ///// Pointer updates to align stages ///// // Update read pointer of each stage diff --git a/hardware/src/vlsu/global_ldst.sv b/hardware/src/vlsu/global_ldst.sv index 12c517f..c41d57a 100644 --- a/hardware/src/vlsu/global_ldst.sv +++ b/hardware/src/vlsu/global_ldst.sv @@ -78,6 +78,7 @@ vew_e vew_d, vew_q; vlen_cluster_t vl_ldst_rd_d, vl_ldst_rd_q, vl_ldst_wr_d, vl_ldst_wr_q; logic fifo_push_d, fifo_push_q; +logic w_fifo_push_d, w_fifo_push_q; // These are updated only when a new aw/ar is accepted assign cluster_metadata_o.vew = vew_d; @@ -103,6 +104,7 @@ always_ff @(posedge clk_i or negedge rst_ni) begin r_resp_lane_q <= '0; r_resp_rem_q <= '0; fifo_push_q <= 1'b0; + w_fifo_push_q <= 1'b0; axi_resp_q <= '0; end else begin r_req_valid_q <= r_req_valid_d; @@ -119,6 +121,7 @@ always_ff @(posedge clk_i or negedge rst_ni) begin r_resp_lane_q <= r_resp_lane_d; r_resp_rem_q <= r_resp_rem_d; fifo_push_q <= fifo_push_d; + w_fifo_push_q <= w_fifo_push_d; axi_resp_q <= axi_resp_d; end end @@ -247,6 +250,7 @@ always_comb begin : p_global_ldst fifo_push_i = 1'b0; b_fifo_push_i = 1'b0; fifo_push_d = fifo_push_q; + w_fifo_push_d = w_fifo_push_q; // Initialize cluster pointers and request counters cluster_aw_d = cluster_aw_q; @@ -328,24 +332,24 @@ always_comb begin : p_global_ldst vl_ldst_wr_d -= vl_w_done; req_d.aw.addr = wr_aligned_next_start_addr_d; // Update request state w_req_valid_d = 1'b1; - b_fifo_push_i = fifo_push_q ? 1'b0 : 1'b1; - fifo_push_d = 1'b1; + b_fifo_push_i = w_fifo_push_q ? 1'b0 : 1'b1; + w_fifo_push_d = 1'b1; end else begin vl_ldst_wr_d = '0; w_req_valid_d = 1'b0; - b_fifo_push_i = fifo_push_q ? 1'b0 : 1'b1; - fifo_push_d = 1'b0; + b_fifo_push_i = w_fifo_push_q ? 1'b0 : 1'b1; + w_fifo_push_d = 1'b0; end end else begin vl_ldst_wr_d -= 1; // If request already pushed to fifo don't push again - b_fifo_push_i = fifo_push_q ? 1'b0 : 1'b1; - fifo_push_d = 1'b1; + b_fifo_push_i = w_fifo_push_q ? 1'b0 : 1'b1; + w_fifo_push_d = 1'b1; if (vl_ldst_wr_d == 0) begin // If we have sent all the requests for this indexed vector store, we can move the AW pointer back to cluster 0. cluster_aw_d = '0; lane_aw_d = '0; - fifo_push_d = 1'b0; + w_fifo_push_d = 1'b0; end w_req_valid_d = 1'b0; end diff --git a/hardware/src/vlsu/shuffle_stage.sv b/hardware/src/vlsu/shuffle_stage.sv index b5c1027..1f4732a 100644 --- a/hardware/src/vlsu/shuffle_stage.sv +++ b/hardware/src/vlsu/shuffle_stage.sv @@ -46,6 +46,10 @@ module shuffle_stage import ara_pkg::*; import rvv_pkg::*; #( `include "common_cells/registers.svh" +////////////////// +// Params/Types // +////////////////// + // There are 2 dapaths in this unit // 1) Shuffle - to shuffle the data coming from memory to the required cluster based on element width // 2) Buffer - to buffer the data coming from memory if the element width is 64b and ClusterAxiDataWidth is 32N, since in this case, the data coming from memory needs to be stored and sent in 2 cycles to the clusters. @@ -54,20 +58,14 @@ localparam int unsigned NUM_DATAPATHS = 2; localparam int unsigned NumTrackers=16; typedef enum logic { SHUFFLE, BUFFER } datapath_t; -typedef logic [$clog2(NumTrackers)-1:0] pnt_t; -typedef logic [$clog2(NumTrackers):0] cnt_t; -typedef axi_w_t [NrClusters-1:0] stage_w_t; - -logic [NrClusters-1:0] buf_sel_d, buf_sel_q; -logic cluster_sel_d, cluster_sel_q; -logic cluster_buf_ready, cluster_buf_valid; -pnt_t [NumStages-1:0] wr_issue_pnt_d, wr_issue_pnt_q; +typedef logic [$clog2(NumTrackers)-1:0] pnt_t; +typedef logic [$clog2(NumTrackers):0] cnt_t; -logic [NrClusters-1:0] wr_cluster_completed_d, wr_cluster_status_completed; +typedef axi_w_t [NrClusters-1:0] stage_w_t; +typedef axi_r_t [NrClusters-1:0] stage_r_t; -`FF(buf_sel_q, buf_sel_d, '0, clk_i, rst_ni) -`FF(cluster_sel_q, cluster_sel_d, '0, clk_i, rst_ni) +typedef logic [NrClusters-1:0] cluster_bitmask_t; // This is the main tracking structure for the requests coming into the shuffle stage. // It keeps track of the status of each request and is used to configure the shuffle and buffer datapath. @@ -91,24 +89,102 @@ typedef struct packed { ara_op_e op; } req_track_t; -req_track_t [NumTrackers-1:0] rd_tracker_d, rd_tracker_q; -pnt_t rd_accept_pnt_d, rd_accept_pnt_q; +////////////////////// +// Helper Functions // +////////////////////// + +// Helper function to compute vector length for a specific cluster +function automatic int unsigned compute_cluster_vl( + input int unsigned vl, + input logic use_eew1, + input int unsigned cluster_idx, + input int unsigned nr_lanes, + input int unsigned total_nr_lanes +); + automatic int unsigned vl_tot = use_eew1 ? vl << 3 : vl; + automatic int unsigned vl_rem = vl_tot & (total_nr_lanes - 1); + automatic int unsigned vl_base = vl_tot >> $clog2(total_nr_lanes); + automatic int unsigned vl_rem_diff = vl_rem - (cluster_idx * nr_lanes); + return (vl_base << $clog2(nr_lanes)) + ((vl_rem >= (cluster_idx + 1) * nr_lanes) ? nr_lanes : (vl_rem >= (cluster_idx * nr_lanes)) ? vl_rem_diff : '0); +endfunction + +// Helper function to advance lane and cluster pointers cyclically +function automatic void advance_lane_cluster( + inout logic [$clog2(NrLanes):0] lane_ptr, + inout logic [$clog2(NrClusters):0] cluster_ptr +); + lane_ptr += 1; + if (lane_ptr == NrLanes) begin + lane_ptr = '0; + cluster_ptr += 1; + if (cluster_ptr == NrClusters) begin + cluster_ptr = '0; + end + end +endfunction + +// Helper function to initialize tracker entry with metadata and request info +function automatic void init_tracker_entry( + output req_track_t tracker, + input cluster_metadata_t cluster_metadata, + input int unsigned len_value, + input logic isRead +); + automatic int unsigned vl_computed; + + tracker.vew = cluster_metadata.vew; + tracker.use_eew1 = cluster_metadata.use_eew1; + tracker.op = cluster_metadata.op; + + for (int c=0; c> 3 : vl_computed; + tracker.len[c] = len_value; + end + + for (int s=0; s= (3 + cluster_metadata.vew)) ? 1'b1 : 1'b0; + end else begin + // Write path: reversed scale + tracker.shuffle_en[s] = ((NumStages - s - 1) >= (3 + cluster_metadata.vew)) ? 1'b1 : 1'b0; + end + end + + tracker.datapath = NumStages < (3 + cluster_metadata.vew) ? BUFFER : SHUFFLE; + tracker.second_buffer_unused = cluster_metadata.vl <= (NrLanes * NrClusters / 2) && tracker.datapath; +endfunction + +cluster_bitmask_t buf_sel_d, buf_sel_q; +logic cluster_sel_d, cluster_sel_q; +logic cluster_buf_ready, cluster_buf_valid; + +pnt_t [NumStages-1:0] wr_issue_pnt_d, wr_issue_pnt_q; pnt_t [NumStages-1:0] rd_issue_pnt_d, rd_issue_pnt_q; +pnt_t rd_accept_pnt_d, rd_accept_pnt_q; +pnt_t wr_accept_pnt_d, wr_accept_pnt_q; + cnt_t rd_cnt_d, rd_cnt_q; +cnt_t wr_cnt_d, wr_cnt_q; +cluster_bitmask_t wr_cluster_completed_d, wr_cluster_status_completed; + +req_track_t [NumTrackers-1:0] rd_tracker_d, rd_tracker_q; req_track_t [NumTrackers-1:0] wr_tracker_d, wr_tracker_q; -pnt_t wr_accept_pnt_d, wr_accept_pnt_q; -cnt_t wr_cnt_d, wr_cnt_q; -typedef axi_r_t [NrClusters-1:0] stage_r_t; stage_r_t [NumStages-1:0] r_data_in, r_data_out; - stage_w_t [NumStages-1:0] w_data_in, w_data_out; logic [NumStages-1:0] r_valid, r_ready, w_valid, w_ready; logic [NumStages-1:0] r_shuffle_en, w_shuffle_en; -logic [NrClusters-1:0] r_ready_i, r_valid_o; +cluster_bitmask_t r_ready_i, r_valid_o; +cluster_bitmask_t axi_wr_buffer_valid, axi_wr_buffer_ready; +cluster_bitmask_t axi_wr_shuffle_valid, axi_wr_shuffle_ready; +cluster_bitmask_t axi_wr_shuffle_ready_inp, axi_wr_shuffle_valid_inp; logic rd_full, wr_full; assign rd_full = (rd_cnt_q == NumTrackers); @@ -136,10 +212,6 @@ stream_fork #( .ready_o(r_ready[NumStages-1] ) ); -logic [NrClusters-1:0] axi_wr_buffer_valid, axi_wr_buffer_ready; -logic [NrClusters-1:0] axi_wr_shuffle_valid, axi_wr_shuffle_ready; -logic [NrClusters-1:0] axi_wr_shuffle_ready_inp, axi_wr_shuffle_valid_inp; - // To handle cases where write data does not come simultaneously // from all the clusters stream_join #( @@ -246,8 +318,7 @@ end /////////////// // Read Responses -typedef axi_r_t [NrClusters-1:0] axi_resp_ext_t; -axi_resp_ext_t [NumBuffers-1:0] buf_d, buf_q; +stage_r_t [NumBuffers-1:0] buf_d, buf_q; axi_resp_t [NrClusters-1:0] axi_resp_buf_out; logic rdbuf_pnt_q, rdbuf_pnt_d; @@ -257,14 +328,61 @@ logic r_ready_buf, r_ready_buf_q; datapath_t rd_datapath; ara_op_e rd_op; +req_track_t rd_tracker_first, rd_tracker_last, rd_tracker_second; -// If responses switch from BUFFER to SHUFFLE datapath, we stall to ensure that all responses for the BUFFER datapath have been handled -// Otherwise the responses that need to use the BUFFER datapath erroneously go into the SHUFFLE datpath. -logic stall_resp; -assign stall_resp = (rd_tracker_q[rd_issue_pnt_q[1]].datapath == BUFFER) && (rd_tracker_q[rd_issue_pnt_q[0]].datapath == SHUFFLE) && (rd_cnt_q != 0); +logic stall_rd_resp; +logic is_shuffle_ongoing, is_buffer_ongoing; -assign rd_datapath = stall_resp ? BUFFER : rd_tracker_q[rd_issue_pnt_q[0]].datapath; -assign rd_op = stall_resp ? rd_tracker_q[rd_issue_pnt_q[1]].op : rd_tracker_q[rd_issue_pnt_q[0]].op; +always_comb begin + rd_tracker_first = rd_tracker_q[rd_issue_pnt_q[0]]; + rd_tracker_second = rd_tracker_q[rd_issue_pnt_q[1]]; + rd_tracker_last = rd_tracker_q[rd_issue_pnt_q[NumStages-1]]; + + is_shuffle_ongoing = (rd_tracker_last.datapath == SHUFFLE) && (rd_tracker_first.datapath != SHUFFLE); + is_buffer_ongoing = (rd_tracker_last.datapath == BUFFER) && !(rd_tracker_last.op inside {VLXE, VLSE}) && (rd_tracker_first.op inside {VLXE, VLSE} || rd_tracker_second.op inside {VLXE, VLSE}); + + stall_rd_resp = (is_shuffle_ongoing || is_buffer_ongoing) && (rd_cnt_q > 1); + + rd_datapath = rd_tracker_last.datapath; + rd_op = rd_tracker_last.op; +end + +////////////////////////////////// +// Mux/Demux for read responses // +////////////////////////////////// + +cluster_bitmask_t axi_rd_unit_stride_ready, axi_rd_unit_stride_valid; +cluster_bitmask_t axi_rd_buffer_ready, axi_rd_buffer_valid; +cluster_bitmask_t axi_req_out_r_ready; +cluster_bitmask_t axi_rd_shuffle_valid, axi_rd_shuffle_ready; +cluster_bitmask_t axi_rd_indexed_valid, axi_rd_indexed_ready; + +for (genvar c=0; c> 3 : vl; - end + // Initialize tracker entry with metadata and request length (isRead=1) + init_tracker_entry(rd_tracker_d[rd_accept_pnt_q], cluster_metadata, axi_req_i[cluster_ar_q].ar.len+1, 1'b1); + // Update pnt to accept next request rd_accept_pnt_d = (rd_accept_pnt_q == NumTrackers-1) ? '0 : rd_accept_pnt_q + 1; rd_cnt_d += 1; - // To enable certain shuffle stages based on element width - for (int s=0; s= (3 + cluster_metadata.vew)) ? 1'b1 : 1'b0; - end - // To enable buffer for 64b element widths - rd_tracker_d[rd_accept_pnt_q].datapath = NumStages < (3 + cluster_metadata.vew) ? BUFFER : SHUFFLE; - rd_tracker_d[rd_accept_pnt_q].second_buffer_unused = cluster_metadata.vl <= (NrLanes * NrClusters / 2) && rd_tracker_d[rd_accept_pnt_q].datapath; - rd_tracker_d[rd_accept_pnt_q].op = cluster_metadata.op; // If it is a VLXE/VLSE request, take from the desired cluster // and switch clusters for every NrLanes requests if (cluster_metadata.op inside {VLXE, VLSE}) begin - lane_ar_d += 1; - if (lane_ar_q == NrLanes - 1) begin - cluster_ar_d += 1; - if (cluster_ar_q == NrClusters - 1) begin - cluster_ar_d = '0; - end - lane_ar_d = '0; - end - + // Update lane and cluster pointers cyclically + advance_lane_cluster(lane_ar_d, cluster_ar_d); + // If a valid request is sent, track it for synchronization if (axi_req_o[cluster_ar_q].ar_valid & axi_resp_o[cluster_ar_q].ar_ready) begin vl_idx_cluster_d = vl_idx_cluster_q + 1; @@ -563,49 +654,23 @@ always_comb begin automatic cluster_metadata_t cluster_metadata = cluster_metadata_i[cluster_aw_q]; if (!wr_idx_accepted_q) begin - // Store element width - wr_tracker_d[wr_accept_pnt_q].vew = cluster_metadata.vew; - wr_tracker_d[wr_accept_pnt_q].use_eew1 = cluster_metadata.use_eew1; - // Track number of beats and vl - for (int c=0; c> $clog2(TotalNrLanes); - automatic int unsigned vl_rem_diff = vl_rem - (c * NrLanes); - automatic int unsigned vl = (vl_base << $clog2(NrLanes)) + ((vl_rem >= (c + 1) * NrLanes) ? NrLanes : (vl_rem >= (c * NrLanes)) ? vl_rem_diff : '0); - - wr_tracker_d[wr_accept_pnt_q].vl[c] = vl; - wr_tracker_d[wr_accept_pnt_q].len[c] = axi_req_i[cluster_aw_q].aw.len+1; - end + // Initialize tracker entry with metadata and request length (isRead=0) + init_tracker_entry(wr_tracker_d[wr_accept_pnt_q], cluster_metadata, axi_req_i[cluster_aw_q].aw.len+1, 1'b0); + // Update pnt to accept next request wr_accept_pnt_d = (wr_accept_pnt_q == NumTrackers-1) ? '0 : wr_accept_pnt_q + 1; wr_cnt_d += 1; - + // If indexed/strided request, write to tracker only once wr_idx_accepted_d = (cluster_metadata.op inside {VSXE, VSSE}); - - // To enable certain shuffle stages based on element width - for (int s=0; s= (3 + cluster_metadata.vew)) ? 1'b1 : 1'b0); - end - // To enable buffer for 64b element widths - wr_tracker_d[wr_accept_pnt_q].datapath = NumStages < (3 + cluster_metadata.vew) ? BUFFER : SHUFFLE; - wr_tracker_d[wr_accept_pnt_q].second_buffer_unused = cluster_metadata.vl <= (NrLanes * NrClusters / 2) && wr_tracker_d[wr_accept_pnt_q].datapath; - wr_tracker_d[wr_accept_pnt_q].op = cluster_metadata.op; end // If it is a VSXE/VSSE request, take from the desired cluster // and switch clusters for every NrLanes requests if (cluster_metadata.op inside {VSXE, VSSE}) begin - lane_aw_d += 1; - if (lane_aw_q == NrLanes - 1) begin - cluster_aw_d += 1; - if (cluster_aw_q == NrClusters - 1) begin - cluster_aw_d = '0; - end - lane_aw_d = '0; - end - + // Update lane and cluster pointers cyclically + advance_lane_cluster(lane_aw_d, cluster_aw_d); + // If a valid request is sent, track it for synchronization if (axi_req_o[cluster_aw_q].aw_valid & axi_resp_i[cluster_aw_q].aw_ready) begin vl_idx_cluster_d = vl_idx_cluster_q + 1; @@ -681,30 +746,27 @@ always_comb begin rdbuf_pnt_d = rdbuf_pnt_q; shift_d = shift_q; r_ready_buf = r_ready_buf_q; - buffer_ld_resp_accepted = 1'b0; + + axi_rd_buffer_ready = '0; + axi_rd_indexed_ready = '0; rd_cluster_completed_d = rd_cluster_completed_q; rd_buffer_completed_d = rd_buffer_completed_q; - // If there is an existing valid data in the buffer and the next request does not use the buffer datapath - // we need to stall until the buffer responses are committed to the clusters - pending_resp = ((rd_datapath == SHUFFLE) || (rd_op inside {VLXE, VLSE})) && (|buf_valid_q); - - if ((((rd_datapath == BUFFER) || (|buf_valid_q)) && (rd_op == VLE)) || pending_resp) begin - + if ((rd_datapath == BUFFER) && (rd_op == VLE)) begin ///// UNIT STRIDE LOADS ///// ///// 64b precision ///// // If have a valid handshake on response add to the buffer // If have a valid response from L2 after aligning buffer it first pointed by rdbuf_pnt_q // Set we have a valid data - if (axi_resp_i[0].r_valid & r_ready_buf_q & (&r_ready_i)) begin + if (axi_rd_buffer_valid[0] & r_ready_buf_q & (&r_ready_i)) begin for (int c=0; c