diff --git a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h index b96182060f5b..6ab53d739f16 100644 --- a/llvm/lib/Target/AIE/AIEBaseInstrInfo.h +++ b/llvm/lib/Target/AIE/AIEBaseInstrInfo.h @@ -656,6 +656,10 @@ struct AIEBaseInstrInfo : public TargetInstrInfo { virtual int getCoreStallCycleAfterLock() const; /// Return cycles for core to resume after lock instruction. virtual int getCoreResumeCycleAfterLock() const; + /// Return true if the target supports lock no_fence semantics: when a + /// lock has no in-flight memory predecessor, no forward latency to subsequent + /// memory ops is required. + virtual bool hasLockNoFenceSemantics() const { return false; } /// Return the schedclass for the given instruction descriptor based on /// operand regclass. diff --git a/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp b/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp index ea851ab55d6d..db63cfdb5fd8 100644 --- a/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp +++ b/llvm/lib/Target/AIE/AIEBaseSubtarget.cpp @@ -220,6 +220,14 @@ class LockDelays : public ScheduleDAGMutation { if (SuccEdge.getKind() != SDep::Order || !LdSt->mayLoadOrStore()) { continue; } + // lock no_fence: when the target supports it, no forward latency is + // needed for ACQ/REL. The predecessor loop above already enforces the + // MEM->lock gap (draining any in-flight memory before the lock issues), + // so no memory can be in-flight at lock issue regardless of whether a + // memory pred exists in the DAG. + // Note: DONE is excluded — it keeps conservative forward latency. + if (TII->isLock(MI->getOpcode()) && TII->hasLockNoFenceSemantics()) + continue; auto OptFirstMemCycle = TII->getFirstMemoryCycle(LdSt->getDesc().SchedClass); assert(!ExactLatencies || OptFirstMemCycle); diff --git a/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp b/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp index 0a75d96d782b..d9c508017977 100644 --- a/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp +++ b/llvm/lib/Target/AIE/AIERegMemEventTracker.cpp @@ -403,8 +403,12 @@ unsigned AIERegMemEventTracker::getSafeOperandsDistanceFromBottom( MaxLatency = checkMemoryDependency(MaxLatency, LoadStoreCycle, MIMemoryCycle, true); } - // Lock instructions: all subsequent memory operations must wait - if (TII->isLock(MI.getOpcode())) { + // Lock instructions: all subsequent memory operations must wait. + // lock no_fence: for targets that support it, no forward latency is + // needed — LockDelays already enforces MEM->lock gaps large enough to drain + // the pipeline before the lock issues, so no memory is ever in-flight at + // lock issue time. + if (TII->isLock(MI.getOpcode()) && !TII->hasLockNoFenceSemantics()) { if (getFirstMemoryAccessCycle() > INT_MIN) { const int CoreResumeCycle = TII->getCoreResumeCycleAfterLock(); const int MemCycle = getFirstMemoryAccessCycle(); diff --git a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h index 134a27a70109..8d4fc1a47593 100644 --- a/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h +++ b/llvm/lib/Target/AIE/aie2p/AIE2PInstrInfo.h @@ -65,6 +65,7 @@ class AIE2PInstrInfo : public AIE2PGenInstrInfo { unsigned getGenericExtractSubvectorOpcode() const override; unsigned getGenericIntegerComparisonOpcode() const override; bool isLock(unsigned Opc) const override; + bool hasLockNoFenceSemantics() const override { return true; } std::optional getDoneLatency(unsigned) const override; bool isDelayedSchedBarrier(const MachineInstr &MI) const override; bool isSchedBarrier(const MachineInstr &MI) const override; diff --git a/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir new file mode 100644 index 000000000000..beb7db289108 --- /dev/null +++ b/llvm/test/CodeGen/AIE/aie2p/schedule/acqrel-scheduling.mir @@ -0,0 +1,835 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# +# This file is licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# (c) Copyright 2026 Advanced Micro Devices, Inc. or its affiliates +# RUN: llc -march=aie2p %topdown-multi -run-pass=postmisched %s -o - \ +# RUN: | FileCheck %s + +# Test ACQ/REL/DONE <-> MEM/stream scheduling gaps. +# +# Memory (load/store): +# - MEM -> ACQ/REL/DONE: memory before lock must be separated by the stall gap. +# - ACQ/REL -> MEM: lock no_fence semantics allow subsequent memory +# immediately after the lock (0 cycles gap). +# - DONE -> MEM: conservative forward latency still applied (no_fence +# does not apply to DONE). +# +# Streams (SS/MS/SCD/MCD): +# Kept away from locks via reserved functional units, not LockDelays. +# lock no_fence semantics do NOT affect stream<->lock gaps. + +# --------------------------------------------------------------------------- +# Memory: ACQ/REL -> store (no prior mem, lock no_fence applies) +# --------------------------------------------------------------------------- + +--- +name: acq_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_store + ; CHECK: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +--- +name: rel_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_store + ; CHECK: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +# --------------------------------------------------------------------------- +# Memory: store -> ACQ/REL (MEM->ACQ gap enforced) and store after (no_fence) +# --------------------------------------------------------------------------- + +--- +name: store_acq_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_acq_store + ; CHECK: ST_dms_sts_idx_imm $r6, $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + ACQ_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 4 +... + +--- +name: store_rel_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_rel_store + ; CHECK: ST_dms_sts_idx_imm $r6, $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: BUNDLE implicit killed $r6, implicit killed $p0, implicit killed $r0 { + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 4 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + REL_mLockId_imm 0, $r0 + ST_dms_sts_idx_imm $r6, $p0, 4 +... + +# --------------------------------------------------------------------------- +# Memory: ACQ/REL -> load (lock no_fence: 0 gap) +# --------------------------------------------------------------------------- + +--- +name: acq_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_load + ; CHECK: BUNDLE implicit-def $r6, implicit killed $p1, implicit killed $r0 { + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +--- +name: rel_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_load + ; CHECK: BUNDLE implicit-def $r6, implicit killed $p1, implicit killed $r0 { + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +# --------------------------------------------------------------------------- +# Memory: load -> ACQ/REL (MEM->ACQ gap enforced) +# --------------------------------------------------------------------------- + +--- +name: load_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_acq + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: load_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_rel + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Scalar streams: ACQ/REL -> SS read / MS write (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: acq_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: rel_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: acq_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +--- +name: rel_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +# --------------------------------------------------------------------------- +# Scalar streams: SS read / MS write -> ACQ/REL (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: ss_read_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_acq + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: ss_read_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_rel + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0, implicit killed $r0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + REL_mLockId_imm 0, $r0 +... + +--- +name: ms_write_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_acq + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + ACQ_mLockId_imm 0, $r0 +... + +--- +name: ms_write_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_rel + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6, implicit killed $r0 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Cascade streams: ACQ/REL -> SCD read / MCD write (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: acq_then_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + $x0 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: rel_then_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + $x0 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: acq_then_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_then_mcd_write + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +--- +name: rel_then_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_then_mcd_write + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +# --------------------------------------------------------------------------- +# Cascade streams: SCD read / MCD write -> ACQ/REL (FU reservation) +# --------------------------------------------------------------------------- + +--- +name: scd_read_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_then_acq + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + ACQ_mLockId_imm 0, $r0 +... + +--- +name: scd_read_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_then_rel + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + REL_mLockId_imm 0, $r0 +... + +--- +name: mcd_write_then_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_then_acq + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + ACQ_mLockId_imm 0, $r0 +... + +--- +name: mcd_write_then_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_then_rel + ; CHECK: BUNDLE implicit killed $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + REL_mLockId_imm 0, $r0 +... + +# --------------------------------------------------------------------------- +# Cascade streams: stream -> lock -> stream (from *_double variants) +# --------------------------------------------------------------------------- + +--- +name: scd_read_acq_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_acq_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: $x1 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + ACQ_mLockId_imm 0, $r0 + $x1 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: mcd_write_acq_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_acq_mcd_write + ; CHECK: BUNDLE implicit $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x $x0, implicit $crmcden + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + ACQ_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +--- +name: scd_read_rel_scd_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: scd_read_rel_scd_read + ; CHECK: BUNDLE implicit-def $x0, implicit-def $wl0, implicit-def $wh0, implicit $crscden, implicit killed $r0 { + ; CHECK-NEXT: $x0 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: $x1 = VMOV_lda_mv_scd_x implicit $crscden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $x0 = VMOV_lda_mv_scd_x implicit $crscden + REL_mLockId_imm 0, $r0 + $x1 = VMOV_lda_mv_scd_x implicit $crscden +... + +--- +name: mcd_write_rel_mcd_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: mcd_write_rel_mcd_write + ; CHECK: BUNDLE implicit $x0, implicit $crmcden, implicit killed $r0 { + ; CHECK-NEXT: VMOV_st_mv_mcd_x $x0, implicit $crmcden + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: } + ; CHECK-NEXT: VMOV_st_mv_mcd_x killed $x0, implicit $crmcden + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + VMOV_st_mv_mcd_x $x0, implicit $crmcden + REL_mLockId_imm 0, $r0 + VMOV_st_mv_mcd_x $x0, implicit $crmcden +... + +# --------------------------------------------------------------------------- +# Lock sequences +# --------------------------------------------------------------------------- + +--- +name: acq_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_acq + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + ACQ_mLockId_imm 0, $r1 +... + +--- +name: acq_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: acq_rel + ; CHECK: ACQ_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ACQ_mLockId_imm 0, $r0 + REL_mLockId_imm 0, $r1 +... + +--- +name: rel_acq +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_acq + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ACQ_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + ACQ_mLockId_imm 0, $r1 +... + +--- +name: rel_rel +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: rel_rel + ; CHECK: REL_mLockId_imm 0, killed $r0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: REL_mLockId_imm 0, killed $r1 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + REL_mLockId_imm 0, $r0 + REL_mLockId_imm 0, $r1 +... + +# --------------------------------------------------------------------------- +# DONE: memory interactions (no_fence does NOT apply to DONE; conservative +# forward latency is kept — both DONE -> MEM and MEM -> DONE enforce the +# stall gap). +# --------------------------------------------------------------------------- + +--- +name: done_then_store +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_store + ; CHECK: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + DONE + ST_dms_sts_idx_imm $r6, $p0, 0 +... + +--- +name: store_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: store_then_done + ; CHECK: ST_dms_sts_idx_imm killed $r6, killed $p0, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ST_dms_sts_idx_imm $r6, $p0, 0 + DONE +... + +--- +name: done_then_load +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_load + ; CHECK: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + $r6 = LDA_dms_lda_idx_imm $p1, 0 +... + +--- +name: load_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: load_then_done + ; CHECK: $r6 = LDA_dms_lda_idx_imm killed $p1, 0 + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: DONE + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = LDA_dms_lda_idx_imm $p1, 0 + DONE +... + +# --------------------------------------------------------------------------- +# DONE: stream interactions (FU reservation, unaffected by lock no_fence) +# --------------------------------------------------------------------------- + +--- +name: done_then_ss_read +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_ss_read + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + $r6 = MOV_lda implicit-def $srss0 +... + +--- +name: ss_read_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ss_read_then_done + ; CHECK: BUNDLE implicit-def $r6, implicit-def $srss0 { + ; CHECK-NEXT: $r6 = MOV_lda implicit-def $srss0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + $r6 = MOV_lda implicit-def $srss0 + DONE +... + +--- +name: done_then_ms_write +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: done_then_ms_write + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + DONE + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 +... + +--- +name: ms_write_then_done +alignment: 16 +body: | + bb.0.entry: + ; CHECK-LABEL: name: ms_write_then_done + ; CHECK: BUNDLE implicit-def $srms0, implicit killed $r6 { + ; CHECK-NEXT: MOV_st_mMStream_tlast_imm killed $r6, implicit-def $srms0 + ; CHECK-NEXT: DONE + ; CHECK-NEXT: } + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + ; CHECK-NEXT: NOP + MOV_st_mMStream_tlast_imm $r6, implicit-def $srms0 + DONE +...