From 2555cea353ad2352b7ce491ebc5a12105b60f0f6 Mon Sep 17 00:00:00 2001 From: Kateryna Muts Date: Wed, 18 Mar 2026 02:29:35 -0600 Subject: [PATCH 1/2] [AIE2P][NFC] Add acqrel-scheduling test (pre-no_fence baseline) Add acqrel-scheduling.mir with scheduling tests covering memory (store/load vs ACQ/REL/DONE), scalar streams (SS/MS), cascade streams (SCD/MCD), and lock sequences (ACQ/ACQ, ACQ/REL, REL/ACQ, REL/REL). CHECK lines reflect the existing guarded-only scheduler behavior, before lock no_fence semantics are implemented. --- .../AIE/aie2p/schedule/acqrel-scheduling.mir | 833 ++++++++++++++++++ 1 file changed, 833 insertions(+) create mode 100644 llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir diff --git a/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir new file mode 100644 index 000000000000..7e0232034401 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir @@ -0,0 +1,833 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +# RUN: llc -march=aie2p %topdown-multi -run-pass=postmisched %s -o - \ +# RUN: | FileCheck %s + +# Test ACQ/REL/DONE <-> MEM/stream scheduling gaps. +# +# Memory (load/store): +# - MEM -> ACQ/REL/DONE: memory before lock must be separated by the stall gap. +# - ACQ/REL -> MEM: lock no_fence semantics allow subsequent memory +# immediately after the lock (0 cycles gap). +# - DONE -> MEM: conservative forward latency still applied (no_fence +# does not apply to DONE). +# +# Streams (SS/MS/SCD/MCD): +# Kept away from locks via reserved functional units, not LockDelays. +# lock no_fence semantics do NOT affect stream<->lock gaps. + +# --------------------------------------------------------------------------- +# Memory: ACQ/REL -> store (no prior mem, lock no_fence applies) +# --------------------------------------------------------------------------- + +--- +name: acq_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_store + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +--- +name: rel_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_store + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +# --------------------------------------------------------------------------- +# Memory: store -> ACQ/REL (MEM->ACQ gap enforced) and store after (no_fence) +# --------------------------------------------------------------------------- + +--- +name: store_acq_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_acq_store + ; CHECK: ST_dms_sts_idx_imm $r6, $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + ACQ_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 4 +... + +--- +name: store_rel_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_rel_store + ; CHECK: ST_dms_sts_idx_imm $r6, $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + REL_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 4 +... + +# --------------------------------------------------------------------------- +# Memory: ACQ/REL -> load (lock no_fence: 0 gap) +# --------------------------------------------------------------------------- + +--- +name: acq_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_load + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +--- +name: rel_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_load + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +# --------------------------------------------------------------------------- +# Memory: load -> ACQ/REL (MEM->ACQ gap enforced) +# --------------------------------------------------------------------------- + +--- +name: load_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_acq + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: load_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_rel + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Scalar streams: ACQ/REL -> SS read / MS write (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: acq_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: rel_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: acq_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +--- +name: rel_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +# --------------------------------------------------------------------------- +# Scalar streams: SS read / MS write -> ACQ/REL (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: ss_read_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_acq + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: ss_read_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_rel + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + REL_mLockId_imm 0, $r0 +... + +--- +name: ms_write_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_acq + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: ms_write_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_rel + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Cascade streams: ACQ/REL -> SCD read / MCD write (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: acq_then_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $x0 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: rel_then_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $x0 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: acq_then_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_mcd_write + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +--- +name: rel_then_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_mcd_write + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +# --------------------------------------------------------------------------- +# Cascade streams: SCD read / MCD write -> ACQ/REL (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: scd_read_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_then_acq + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + ACQ_mLockId_imm 0, $r0 +... + +--- +name: scd_read_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_then_rel + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + REL_mLockId_imm 0, $r0 +... + +--- +name: mcd_write_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_then_acq + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + ACQ_mLockId_imm 0, $r0 +... + +--- +name: mcd_write_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_then_rel + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Cascade streams: stream -> lock -> stream (from *_double variants) +# --------------------------------------------------------------------------- + +--- +name: scd_read_acq_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_acq_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: $x1 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + ACQ_mLockId_imm 0, $r0 + $x1 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: mcd_write_acq_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_acq_mcd_write + ; CHECK: BUNDLE implicit $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + ACQ_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +--- +name: scd_read_rel_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_rel_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: $x1 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + REL_mLockId_imm 0, $r0 + $x1 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: mcd_write_rel_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_rel_mcd_write + ; CHECK: BUNDLE implicit $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + REL_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +# --------------------------------------------------------------------------- +# Lock sequences +# --------------------------------------------------------------------------- + +--- +name: acq_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_acq + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + ACQ_mLockId_imm 0, $r1 +... + +--- +name: acq_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_rel + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + REL_mLockId_imm 0, $r1 +... + +--- +name: rel_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_acq + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + ACQ_mLockId_imm 0, $r1 +... + +--- +name: rel_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_rel + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + REL_mLockId_imm 0, $r1 +... + +# --------------------------------------------------------------------------- +# DONE: memory interactions (no_fence does NOT apply to DONE; conservative +# forward latency is kept — both DONE -> MEM and MEM -> DONE enforce the +# stall gap). +# --------------------------------------------------------------------------- + +--- +name: done_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_store + ; CHECK: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + DONE + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +--- +name: store_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_then_done + ; CHECK: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + DONE +... + +--- +name: done_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_load + ; CHECK: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +--- +name: load_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_done + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + DONE +... + +# --------------------------------------------------------------------------- +# DONE: stream interactions (FU reservation, unaffected by lock no_fence) +# --------------------------------------------------------------------------- + +--- +name: done_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: ss_read_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_done + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + DONE +... + +--- +name: done_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +--- +name: ms_write_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_done + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + DONE +... From 1c17bf725c73353bd033db7e54bfd3462ae9c799 Mon Sep 17 00:00:00 2001 From: Kateryna Muts Date: Wed, 18 Mar 2026 02:33:18 -0600 Subject: [PATCH 2/2] [AIE2P] Implement lock no_fence scheduling semantics The AIE2P spec defines two variants of the acquire/release lock instruction depending on context: - guarded: used when memory operations are in-flight before the lock. Enforces a minimum gap between the preceding memory op and any subsequent one. - no_fence: used when no memory is in-flight before the lock. Omits all memory lock token constraints, allowing subsequent memory ops to be scheduled immediately (0 cycles gap). Previously the scheduler always applied the guarded latency forward from any lock to subsequent memory, matching the guarded variant regardless of context. This patch implements no_fence semantics for AIE2P by: 1. Adding a virtual hasLockNoFenceSemantics() hook to AIEBaseInstrInfo (default: false), overridden to return true in AIE2PInstrInfo. 2. In LockDelays::apply, skipping the forward ACQ/REL->MEM latency when the target supports no_fence semantics. 3. In AIERegMemEventTracker::getSafeOperandsDistanceFromBottom, skipping the forward lock-to-mem distance for the same reason. The MEM->ACQ/REL direction (predecessor latencies) is unchanged. The fix is gated on hasLockNoFenceSemantics() and does not affect AIE1, AIE2, or AIE2PS. Update acqrel-scheduling.mir CHECK lines to reflect no_fence behavior. --- llvm/lib/Target/AIE/AIEBaseInstrInfo.h | 4 ++ llvm/lib/Target/AIE/AIEBaseSubtarget.cpp | 8 ++++ llvm/lib/Target/AIE/AIERegMemEventTracker.cpp | 8 +++- llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h | 1 + .../AIE/aie2p/schedule/acqrel-scheduling.mir | 46 ++++++++++--------- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h index b96182060f5b..6ab53d739f16 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h @@ -656,6 +656,10 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { virtual int getCoreStallCycleAfterLock() const; /// Return cycles for core to resume after lock instruction. virtual int getCoreResumeCycleAfterLock() const; + /// Return true if the target supports lock no_fence semantics: when a + /// lock has no in-flight memory predecessor, no forward latency to subsequent + /// memory ops is required. + virtual bool hasLockNoFenceSemantics() const { return false; } /// Return the schedclass for the given instruction descriptor based on /// operand regclass. diff --git a/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp b/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp index ea851ab55d6d..db63cfdb5fd8 100644 --- a/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp +++ b/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp @@ -220,6 +220,14 @@ class LockDelays : public ScheduleDAGMutation { if (SuccEdge.getKind() != SDep::Order || !LdSt->mayLoadOrStore()) { continue; } + // lock no_fence: when the target supports it, no forward latency is + // needed for ACQ/REL. The predecessor loop above already enforces the + // MEM->lock gap (draining any in-flight memory before the lock issues), + // so no memory can be in-flight at lock issue regardless of whether a + // memory pred exists in the DAG. + // Note: DONE is excluded — it keeps conservative forward latency. + if (TII->isLock(MI->getOpcode()) && TII->hasLockNoFenceSemantics()) + continue; auto OptFirstMemCycle = TII->getFirstMemoryCycle(LdSt->getDesc().SchedClass); assert(!ExactLatencies || OptFirstMemCycle); diff --git a/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp b/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp index 0a75d96d782b..d9c508017977 100644 --- a/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp +++ b/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp @@ -403,8 +403,12 @@ unsigned AIERegMemEventTracker::getSafeOperandsDistanceFromBottom( MaxLatency = checkMemoryDependency(MaxLatency, LoadStoreCycle, MIMemoryCycle, true); } - // Lock instructions: all subsequent memory operations must wait - if (TII->isLock(MI.getOpcode())) { + // Lock instructions: all subsequent memory operations must wait. + // lock no_fence: for targets that support it, no forward latency is + // needed — LockDelays already enforces MEM->lock gaps large enough to drain + // the pipeline before the lock issues, so no memory is ever in-flight at + // lock issue time. + if (TII->isLock(MI.getOpcode()) && !TII->hasLockNoFenceSemantics()) { if (getFirstMemoryAccessCycle() > INT_MIN) { const int CoreResumeCycle = TII->getCoreResumeCycleAfterLock(); const int MemCycle = getFirstMemoryAccessCycle(); diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h index 134a27a70109..8d4fc1a47593 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h @@ -65,6 +65,7 @@ class AIE2PInstrInfo : public AIE2PGenInstrInfo { unsigned getGenericExtractSubvectorOpcode() const override; unsigned getGenericIntegerComparisonOpcode() const override; bool isLock(unsigned Opc) const override; + bool hasLockNoFenceSemantics() const override { return true; } std::optional getDoneLatency(unsigned) const override; bool isDelayedSchedBarrier(const MachineInstr &MI) const override; bool isSchedBarrier(const MachineInstr &MI) const override; diff --git a/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir index 7e0232034401..beb7db289108 100644 --- a/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir +++ b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir @@ -31,11 +31,12 @@ alignment: 16 body: | bb.0.entry: ; CHECK-LABEL: name: acq_then_store - ; CHECK: ACQ_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP + ; CHECK: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 ; CHECK-NEXT: NOP ACQ_mLockId_imm 0, $r0 ST_dms_sts_idx_imm $r6, $p0, 0 @@ -47,11 +48,12 @@ alignment: 16 body: | bb.0.entry: ; CHECK-LABEL: name: rel_then_store - ; CHECK: REL_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP + ; CHECK: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 ; CHECK-NEXT: NOP REL_mLockId_imm 0, $r0 ST_dms_sts_idx_imm $r6, $p0, 0 @@ -71,11 +73,12 @@ body: | ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP + ; CHECK-NEXT: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 ; CHECK-NEXT: NOP ST_dms_sts_idx_imm $r6, $p0, 0 ACQ_mLockId_imm 0, $r0 @@ -92,11 +95,12 @@ body: | ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP + ; CHECK-NEXT: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP - ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 ; CHECK-NEXT: NOP ST_dms_sts_idx_imm $r6, $p0, 0 REL_mLockId_imm 0, $r0 @@ -113,11 +117,10 @@ alignment: 16 body: | bb.0.entry: ; CHECK-LABEL: name: acq_then_load - ; CHECK: ACQ_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP - ; CHECK-NEXT: NOP - ; CHECK-NEXT: NOP - ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK: BUNDLE implicit-def $r6, implicit killed $p1, implicit killed $r0 { + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP @@ -134,11 +137,10 @@ alignment: 16 body: | bb.0.entry: ; CHECK-LABEL: name: rel_then_load - ; CHECK: REL_mLockId_imm 0, killed $r0 - ; CHECK-NEXT: NOP - ; CHECK-NEXT: NOP - ; CHECK-NEXT: NOP - ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK: BUNDLE implicit-def $r6, implicit killed $p1, implicit killed $r0 { + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP ; CHECK-NEXT: NOP