[Feature] Support heterogeneous multi-CGRAs.

cgra/CgraTemplateRTL.py

@@ -79,7 +79,17 @@ def construct(s, CgraPayloadType,
                 total_steps, mem_access_is_combinational,
                 FunctionUnit, FuList, TileList, LinkList,
                 dataSPM, controller2addr_map, idTo2d_map,
-                is_multi_cgra = True, cgra_id = 0):
+                is_multi_cgra = True, cgra_id = 0,
+                provided_max_per_cgra_rows = None,
+                provided_max_per_cgra_cols = None,
+                provided_max_num_rd_tiles = None,
+                provided_max_num_wr_tiles = None):
+    """
+    provided_max_per_cgra_rows: the row number of the largest cgra in the multi heterogeneous cgra architecture. None for single cgra arch or Homogeneous multi-cgra arch.
+    provided_max_per_cgra_cols: the column number of the largest cgra in the multi heterogeneous cgra architecture. None for single cgra arch or Homogeneous multi-cgra arch.
+    provided_max_num_rd_tiles: the number of read ports of the largest cgra in the multi heterogeneous cgra architecture. None for single cgra arch or Homogeneous multi-cgra arch.
+    provided_max_num_wr_tiles: the number of write ports of the largest cgra in the multi heterogeneous cgra architecture. None for single cgra arch or Homogeneous multi-cgra arch.
+    """
 
     DataType = CgraPayloadType.get_field_type(kAttrData)
     PredicateType = DataType.get_field_type(kAttrPredicate)
@@ -89,22 +99,31 @@ def construct(s, CgraPayloadType,
     CgraIdType = mk_cgra_id_type(multi_cgra_columns, multi_cgra_rows)
 
     # Reconstructs packet types.
-    num_tiles = len(TileList)
-    # Calculates num_rd_tiles from TileList (number of tiles with read ports).
-    num_rd_tiles = dataSPM.getNumOfValidReadPorts()
-
+    # In the case of heterogeneous multi-cgra, `max_num_tiles` means the tile number of the largest cgra.
+    # In the case of single cgra, it is the tile number of the current cgra.
+    max_per_cgra_rows = provided_max_per_cgra_rows if provided_max_per_cgra_rows is not None else per_cgra_rows
+    max_per_cgra_cols = provided_max_per_cgra_cols if provided_max_per_cgra_cols is not None else per_cgra_columns
+    max_num_tiles = max_per_cgra_rows * max_per_cgra_cols
+    # In the case of heterogeneous multi-cgra, `max_num_rd_tiles` means the number of read ports of the largest cgra.
+    # In the case of single cgra, it is the number of read ports of the current cgra.
+    max_num_rd_tiles = provided_max_num_rd_tiles if provided_max_num_rd_tiles is not None else dataSPM.getNumOfValidReadPorts()
+    max_num_wr_tiles = provided_max_num_wr_tiles if provided_max_num_wr_tiles is not None else dataSPM.getNumOfValidWritePorts()
+
+
+    # Use largest CGRA shape(max_num_tiles) to set CtrlPktType/NocPktType for compatibility.
     CtrlPktType = mk_intra_cgra_pkt(multi_cgra_columns, multi_cgra_rows,
-                                    num_tiles, CgraPayloadType)
+                                    max_num_tiles, CgraPayloadType)
 
     NocPktType = mk_inter_cgra_pkt(multi_cgra_columns, multi_cgra_rows,
-                                   num_tiles, num_rd_tiles,
+                                   max_num_tiles, max_num_rd_tiles,
                                    CgraPayloadType)
 
     s.num_mesh_ports = 8
+    # tile number of the current cgra.
     s.num_tiles = len(TileList)
     num_cgras = multi_cgra_rows * multi_cgra_columns
     # An additional router for controller to receive CMD_COMPLETE signal from Ring to CPU.
-    CtrlRingPos = mk_ring_pos(s.num_tiles + 1)
+    CtrlRingPos = mk_ring_pos(max_num_tiles + 1)
     CtrlAddrType = mk_bits(clog2(ctrl_mem_size))
     DataAddrType = mk_bits(clog2(data_mem_size_global))
     assert(data_mem_size_per_bank * num_banks_per_cgra <= \
@@ -117,14 +136,17 @@ def construct(s, CgraPayloadType,
     s.send_to_inter_cgra_noc = SendIfcRTL(NocPktType)
 
     if is_multi_cgra:
-      s.recv_data_on_boundary_north = [RecvIfcRTL(DataType) for _ in range(per_cgra_columns)]
-      s.send_data_on_boundary_north = [SendIfcRTL(DataType) for _ in range(per_cgra_columns)]
-      s.recv_data_on_boundary_south = [RecvIfcRTL(DataType) for _ in range(per_cgra_columns)]
-      s.send_data_on_boundary_south = [SendIfcRTL(DataType) for _ in range(per_cgra_columns)]
-      s.recv_data_on_boundary_west  = [RecvIfcRTL(DataType) for _ in range(per_cgra_rows)]
-      s.send_data_on_boundary_west  = [SendIfcRTL(DataType) for _ in range(per_cgra_rows)]
-      s.recv_data_on_boundary_east  = [RecvIfcRTL(DataType) for _ in range(per_cgra_rows)]
-      s.send_data_on_boundary_east  = [SendIfcRTL(DataType) for _ in range(per_cgra_rows)]
+      # Use the largest CGRA shape to set the boundary ports for compatibility in the case of heterogeneous multi-cgra.
+      # Remember to ground the remaining boundary ports of the current CGRA when the current CGRA has fewer rows or columns than the largest CGRA.
+      # See also: 
+      s.recv_data_on_boundary_north = [RecvIfcRTL(DataType) for _ in range(max_per_cgra_cols)]
+      s.send_data_on_boundary_north = [SendIfcRTL(DataType) for _ in range(max_per_cgra_cols)]
+      s.recv_data_on_boundary_south = [RecvIfcRTL(DataType) for _ in range(max_per_cgra_cols)]
+      s.send_data_on_boundary_south = [SendIfcRTL(DataType) for _ in range(max_per_cgra_cols)]
+      s.recv_data_on_boundary_west  = [RecvIfcRTL(DataType) for _ in range(max_per_cgra_rows)]
+      s.send_data_on_boundary_west  = [SendIfcRTL(DataType) for _ in range(max_per_cgra_rows)]
+      s.recv_data_on_boundary_east  = [RecvIfcRTL(DataType) for _ in range(max_per_cgra_rows)]
+      s.send_data_on_boundary_east  = [SendIfcRTL(DataType) for _ in range(max_per_cgra_rows)]
 
     # Components
     s.tile = [TileRTL(CtrlPktType,
@@ -140,22 +162,22 @@ def construct(s, CgraPayloadType,
                                       data_mem_size_global,
                                       data_mem_size_per_bank,
                                       num_banks_per_cgra,
-                                      dataSPM.getNumOfValidReadPorts(),
-                                      dataSPM.getNumOfValidWritePorts(),
+                                      max_num_rd_tiles,
+                                      max_num_wr_tiles,
                                       multi_cgra_rows,
                                       multi_cgra_columns,
-                                      s.num_tiles,
+                                      max_num_tiles,
                                       mem_access_is_combinational,
                                       idTo2d_map)
     s.cgra_id = InPort(CgraIdType)
     s.controller = ControllerRTL(NocPktType,
                                   multi_cgra_rows, multi_cgra_columns,
-                                  s.num_tiles, controller2addr_map, idTo2d_map)
+                                  max_num_tiles, controller2addr_map, idTo2d_map)
     # Connects controller id.
     s.controller.cgra_id //= s.cgra_id
     # An additional router for controller to receive CMD_COMPLETE signal from Ring to CPU.
     # The last argument of 1 is for the latency per hop.
-    s.ctrl_ring = RingNetworkRTL(CtrlPktType, CtrlRingPos, s.num_tiles + 1, 1)
+    s.ctrl_ring = RingNetworkRTL(CtrlPktType, CtrlRingPos, max_num_tiles + 1, 1)
 
     # Address lower and upper bound.
     s.address_lower = InPort(DataAddrType)
@@ -196,11 +218,17 @@ def construct(s, CgraPayloadType,
     # Connects ring with each control memory.
     for i in range(s.num_tiles):
       s.ctrl_ring.send[i] //= s.tile[i].recv_from_controller_pkt
-    for i in range(s.num_tiles):
       s.ctrl_ring.recv[i] //= s.tile[i].send_to_controller_pkt
+
     s.ctrl_ring.recv[s.num_tiles] //= s.controller.send_to_ctrl_ring_pkt
     s.ctrl_ring.send[s.num_tiles] //= s.controller.recv_from_ctrl_ring_pkt
 
+    # Grounds the remaining ports of the ring.
+    for i in range(s.num_tiles + 1, max_num_tiles + 1):
+      s.ctrl_ring.send[i].rdy //= 0
+      s.ctrl_ring.recv[i].val //= 0
+      s.ctrl_ring.recv[i].msg //= CtrlPktType()
+
     for link in LinkList:
 
       if link.isFromMem():
@@ -233,8 +261,16 @@ def construct(s, CgraPayloadType,
           s.tile[srcTileIndex].send_data[link.srcPort] //= s.tile[dstTileIndex].recv_data[link.dstPort]
 
     # (cgra_idx_x, cgra_idx_y) is the coordinate of the current cgra in multi-cgra(Cartesian coordinate system).
+    """
+    ^ y
+    |
+    |  cgra2   cgra3
+    |  cgra0   cgra1
+    +---------------> x
+
+    """
     cgra_idx_x = cgra_id % multi_cgra_columns
-    cgra_idx_y = cgra_id // multi_cgra_rows
+    cgra_idx_y = cgra_id // multi_cgra_columns
 
     """
     row ^

lib/util/cgra/cgra_helper.py

@@ -131,8 +131,8 @@ def set_port_validity(tile, port, is_valid = True):
 
 
 def configure_boundary_ports(cgra_id, tiles_flat,
-                                num_cgra_rows, num_cgra_columns,
-                                per_cgra_rows, per_cgra_columns,
+                                num_cgra_rows, num_cgra_cols,
+                                id2shape_map,
                                 is_valid = True):
     """
     Enable boundary ports for tiles on adjacent CGRAs.
@@ -141,49 +141,69 @@ def configure_boundary_ports(cgra_id, tiles_flat,
     - cgra_id: ID of the current CGRA (0-indexed, bottom-left to top-right)
     - tiles_flat: Flat list of tiles for this CGRA (reshaped from 2D)
     - num_cgra_rows: Number of CGRA rows in the mesh
-    - num_cgra_columns: Number of CGRA columns in the mesh
-    - per_cgra_rows: Number of tile rows in each CGRA
-    - per_cgra_columns: Number of tile columns in each CGRA
+    - num_cgra_cols: Number of CGRA columns in the mesh
+    - id2shape_map: Map of each CGRA id to its shape: (num_tile_rows x num_tile_columns) tiles
     - is_valid: If true, enable ports, otherwise disable ports
-
-    CGRA ID mapping (example for 2x2):
-    CGRA 2: [row=0, col=0] CGRA 3: [row=0, col=1]  (top row, row=0)
-    CGRA 0: [row=1, col=0] CGRA 1: [row=1, col=1]  (bottom row, row=1)
     """
     # Converts CGRA ID to 2D coordinates
-    cgra_row = (num_cgra_rows - 1) - (cgra_id // num_cgra_columns)
-    cgra_col = cgra_id % num_cgra_columns
+    cgra_x = cgra_id % num_cgra_cols
+    cgra_y = cgra_id // num_cgra_cols
+    # The number of tile rows and columns in the current CGRA.
+    num_tile_rows, num_tile_cols = id2shape_map[cgra_id]
 
-    # Helper to get tile from flat list using row/col indices
-    def get_tile(row, col):
-        return tiles_flat[row * per_cgra_columns + col]
+    # Helper to get tile from flat list using x/y indices
+    def get_tile(x, y):
+        return tiles_flat[y * num_tile_cols + x]
 
     # Enables NORTH ports if there's a neighbor to the north
-    if cgra_row > 0:
+    if cgra_y < num_cgra_rows - 1:
         # This CGRA has a neighbor above
+        # Gets the tile shape of the neighbor CGRA.
+        neighbor_cgra_id = cgra_id + num_cgra_cols
+        num_neighbor_tile_rows, num_neighbor_tile_cols = id2shape_map[neighbor_cgra_id]
+
         # Top row of tiles in this CGRA should have NORTH ports enabled
-        top_row_idx = per_cgra_rows - 1
-        for tile_col in range(per_cgra_columns):
-            set_port_validity(get_tile(top_row_idx, tile_col), PORT_INDEX_NORTH, is_valid)
+        # y axis of the top row tiles of the current CGRA.
+        top_row_y = num_tile_rows - 1
+        valid_port_num = min(num_tile_cols, num_neighbor_tile_cols)
+        for tile_x in range(valid_port_num):
+            set_port_validity(get_tile(tile_x, top_row_y), PORT_INDEX_NORTH, is_valid)
 
     # Enables SOUTH ports if there's a neighbor to the south
-    if cgra_row < num_cgra_rows - 1:
+    if cgra_y > 0:
         # This CGRA has a neighbor below
+        # Gets the tile shape of the neighbor CGRA.
+        neighbor_cgra_id = cgra_id - num_cgra_cols
+        num_neighbor_tile_rows, num_neighbor_tile_cols = id2shape_map[neighbor_cgra_id]
+
         # Bottom row of tiles in this CGRA should have SOUTH ports enabled
-        bottom_row_idx = 0
-        for tile_col in range(per_cgra_columns):
-            set_port_validity(get_tile(bottom_row_idx, tile_col), PORT_INDEX_SOUTH, is_valid)
+        bottom_row_y = 0
+        valid_port_num = min(num_tile_cols, num_neighbor_tile_cols)
+        for tile_x in range(valid_port_num):
+            set_port_validity(get_tile(tile_x, bottom_row_y), PORT_INDEX_SOUTH, is_valid)
 
     # Enables EAST ports if there's a neighbor to the east
-    if cgra_col < num_cgra_columns - 1:
+    if cgra_x < num_cgra_cols - 1:
+        # This CGRA has a neighbor to the right.
+        # Gets the tile shape of the neighbor CGRA.
+        neighbor_cgra_id = cgra_id + 1
+        num_neighbor_tile_rows, num_neighbor_tile_cols = id2shape_map[neighbor_cgra_id]
+
         # Rightmost column of tiles in this CGRA should have EAST ports enabled
-        east_col_idx = per_cgra_columns - 1
-        for tile_row in range(per_cgra_rows):
-            set_port_validity(get_tile(tile_row, east_col_idx), PORT_INDEX_EAST, is_valid)
+        east_col_x = num_tile_cols - 1
+        valid_port_num = min(num_tile_rows, num_neighbor_tile_rows)
+        for tile_y in range(valid_port_num):
+            set_port_validity(get_tile(east_col_x, tile_y), PORT_INDEX_EAST, is_valid)
 
     # Enables WEST ports if there's a neighbor to the west
-    if cgra_col > 0:
+    if cgra_x > 0:
+        # This CGRA has a neighbor to the left.
+        # Gets the tile shape of the neighbor CGRA.
+        neighbor_cgra_id = cgra_id - 1
+        num_neighbor_tile_rows, num_neighbor_tile_cols = id2shape_map[neighbor_cgra_id]
+
         # Leftmost column of tiles in this CGRA should have WEST ports enabled
-        west_col_idx = 0
-        for tile_row in range(per_cgra_rows):
-            set_port_validity(get_tile(tile_row, west_col_idx), PORT_INDEX_WEST, is_valid)
+        west_col_x = 0
+        valid_port_num = min(num_tile_rows, num_neighbor_tile_rows)
+        for tile_y in range(valid_port_num):
+            set_port_validity(get_tile(west_col_x, tile_y), PORT_INDEX_WEST, is_valid)

mem/ctrl/CtrlMemDynamicRTL.py

@@ -30,6 +30,7 @@ def construct(s, IntraCgraPktType,
 
     CgraPayloadType = IntraCgraPktType.get_field_type(kAttrPayload)
     CtrlType = CgraPayloadType.get_field_type(kAttrCtrl)
+    IntraPktTileIdType = IntraCgraPktType.get_field_type(kAttrSrc)
     # The total_ctrl_steps indicates the number of steps the ctrl
     # signals should proceed. For example, if the number of ctrl
     # signals is 4 and they need to repeat 5 times, then the total
@@ -179,7 +180,7 @@ def update_send_pkt_to_controller():
       if s.start_iterate_ctrl == b1(1):
         if s.recv_from_element_queue.send.val & (~s.sent_complete):
           s.send_pkt_to_controller.msg @= \
-              IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0,
+              IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0,
                                s.recv_from_element_queue.send.msg)
           s.send_pkt_to_controller.val @= 1
           s.recv_from_element_queue.send.rdy @= s.send_pkt_to_controller.rdy
@@ -188,7 +189,7 @@ def update_send_pkt_to_controller():
           # Sends COMPLETE signal to Controller when the last ctrl signal is done.
           if ~s.sent_complete & (s.total_ctrl_steps_val > 0) & (s.times == s.total_ctrl_steps_val) & s.start_iterate_ctrl:
             s.send_pkt_to_controller.msg @= \
-                IntraCgraPktType(s.tile_id, num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0))
+                IntraCgraPktType(zext(s.tile_id, IntraPktTileIdType), num_tiles, 0, 0, 0, 0, 0, 0, 0, 0, CgraPayloadType(CMD_COMPLETE, 0, 0, 0, 0))
             s.send_pkt_to_controller.val @= 1
 
     @update