1 files changed, 88 insertions, 71 deletions

diff --git a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
index ba5432459d12..886137d86f87 100644
--- a/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/contrib/llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -747,9 +747,9 @@ void ScheduleDAGMI::finishBlock() {
   ScheduleDAGInstrs::finishBlock();
 }
 
-/// enterRegion - Called back from MachineScheduler::runOnMachineFunction after
-/// crossing a scheduling boundary. [begin, end) includes all instructions in
-/// the region, including the boundary itself and single-instruction regions
+/// enterRegion - Called back from PostMachineScheduler::runOnMachineFunction
+/// after crossing a scheduling boundary. [begin, end) includes all instructions
+/// in the region, including the boundary itself and single-instruction regions
 /// that don't get scheduled.
 void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb,
                                      MachineBasicBlock::iterator begin,
@@ -793,9 +793,9 @@ bool ScheduleDAGMI::checkSchedLimit() {
 }
 
 /// Per-region scheduling driver, called back from
-/// MachineScheduler::runOnMachineFunction. This is a simplified driver that
-/// does not consider liveness or register pressure. It is useful for PostRA
-/// scheduling and potentially other custom schedulers.
+/// PostMachineScheduler::runOnMachineFunction. This is a simplified driver
+/// that does not consider liveness or register pressure. It is useful for
+/// PostRA scheduling and potentially other custom schedulers.
 void ScheduleDAGMI::schedule() {
   LLVM_DEBUG(dbgs() << "ScheduleDAGMI::schedule starting\n");
   LLVM_DEBUG(SchedImpl->dumpPolicy());
@@ -980,8 +980,8 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceTopDown() const {
     for (TargetSchedModel::ProcResIter PI = SchedModel.getWriteProcResBegin(SC),
                                        PE = SchedModel.getWriteProcResEnd(SC);
          PI != PE; ++PI) {
-      if (SU->TopReadyCycle + PI->Cycles - 1 > LastCycle)
-        LastCycle = SU->TopReadyCycle + PI->Cycles - 1;
+      if (SU->TopReadyCycle + PI->ReleaseAtCycle - 1 > LastCycle)
+        LastCycle = SU->TopReadyCycle + PI->ReleaseAtCycle - 1;
     }
   }
   // Print the header with the cycles
@@ -1017,19 +1017,20 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceTopDown() const {
       llvm::stable_sort(ResourcesIt,
                         [](const MCWriteProcResEntry &LHS,
                            const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.StartAtCycle < RHS.StartAtCycle ||
-                                 (LHS.StartAtCycle == RHS.StartAtCycle &&
-                                  LHS.Cycles < RHS.Cycles);
+                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
+                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
+                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
                         });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
           SchedModel.getResourceName(PI.ProcResourceIdx);
       dbgs() << llvm::right_justify(ResName + " ", HeaderColWidth);
-      for (; C < SU->TopReadyCycle + PI.StartAtCycle; ++C) {
+      for (; C < SU->TopReadyCycle + PI.AcquireAtCycle; ++C) {
         dbgs() << llvm::left_justify("|", ColWidth);
       }
-      for (unsigned I = 0, E = PI.Cycles - PI.StartAtCycle; I != E; ++I, ++C)
+      for (unsigned I = 0, E = PI.ReleaseAtCycle - PI.AcquireAtCycle; I != E;
+           ++I, ++C)
         dbgs() << llvm::left_justify("| x", ColWidth);
       while (C++ <= LastCycle)
         dbgs() << llvm::left_justify("|", ColWidth);
@@ -1061,8 +1062,8 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const {
     for (TargetSchedModel::ProcResIter PI = SchedModel.getWriteProcResBegin(SC),
                                        PE = SchedModel.getWriteProcResEnd(SC);
          PI != PE; ++PI) {
-      if ((int)SU->BotReadyCycle - PI->Cycles + 1 < LastCycle)
-        LastCycle = (int)SU->BotReadyCycle - PI->Cycles + 1;
+      if ((int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1 < LastCycle)
+        LastCycle = (int)SU->BotReadyCycle - PI->ReleaseAtCycle + 1;
     }
   }
   // Print the header with the cycles
@@ -1097,19 +1098,20 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const {
       llvm::stable_sort(ResourcesIt,
                         [](const MCWriteProcResEntry &LHS,
                            const MCWriteProcResEntry &RHS) -> bool {
-                          return LHS.StartAtCycle < RHS.StartAtCycle ||
-                                 (LHS.StartAtCycle == RHS.StartAtCycle &&
-                                  LHS.Cycles < RHS.Cycles);
+                          return LHS.AcquireAtCycle < RHS.AcquireAtCycle ||
+                                 (LHS.AcquireAtCycle == RHS.AcquireAtCycle &&
+                                  LHS.ReleaseAtCycle < RHS.ReleaseAtCycle);
                         });
     for (const MCWriteProcResEntry &PI : ResourcesIt) {
       C = FirstCycle;
       const std::string ResName =
           SchedModel.getResourceName(PI.ProcResourceIdx);
       dbgs() << llvm::right_justify(ResName + " ", HeaderColWidth);
-      for (; C > ((int)SU->BotReadyCycle - (int)PI.StartAtCycle); --C) {
+      for (; C > ((int)SU->BotReadyCycle - (int)PI.AcquireAtCycle); --C) {
         dbgs() << llvm::left_justify("|", ColWidth);
       }
-      for (unsigned I = 0, E = PI.Cycles - PI.StartAtCycle; I != E; ++I, --C)
+      for (unsigned I = 0, E = PI.ReleaseAtCycle - PI.AcquireAtCycle; I != E;
+           ++I, --C)
         dbgs() << llvm::left_justify("| x", ColWidth);
       while (C-- >= LastCycle)
         dbgs() << llvm::left_justify("|", ColWidth);
@@ -1696,11 +1698,12 @@ class BaseMemOpClusterMutation : public ScheduleDAGMutation {
     SmallVector<const MachineOperand *, 4> BaseOps;
     int64_t Offset;
     unsigned Width;
+    bool OffsetIsScalable;
 
     MemOpInfo(SUnit *SU, ArrayRef<const MachineOperand *> BaseOps,
-              int64_t Offset, unsigned Width)
+              int64_t Offset, bool OffsetIsScalable, unsigned Width)
         : SU(SU), BaseOps(BaseOps.begin(), BaseOps.end()), Offset(Offset),
-          Width(Width) {}
+          Width(Width), OffsetIsScalable(OffsetIsScalable) {}
 
     static bool Compare(const MachineOperand *const &A,
                         const MachineOperand *const &B) {
@@ -1829,8 +1832,10 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps(
           SUnit2ClusterInfo[MemOpa.SU->NodeNum].second + MemOpb.Width;
     }
 
-    if (!TII->shouldClusterMemOps(MemOpa.BaseOps, MemOpb.BaseOps, ClusterLength,
-                                  CurrentClusterBytes))
+    if (!TII->shouldClusterMemOps(MemOpa.BaseOps, MemOpa.Offset,
+                                  MemOpa.OffsetIsScalable, MemOpb.BaseOps,
+                                  MemOpb.Offset, MemOpb.OffsetIsScalable,
+                                  ClusterLength, CurrentClusterBytes))
       continue;
 
     SUnit *SUa = MemOpa.SU;
@@ -1897,7 +1902,8 @@ void BaseMemOpClusterMutation::collectMemOpRecords(
     unsigned Width;
     if (TII->getMemOperandsWithOffsetWidth(MI, BaseOps, Offset,
                                            OffsetIsScalable, Width, TRI)) {
-      MemOpRecords.push_back(MemOpInfo(&SU, BaseOps, Offset, Width));
+      MemOpRecords.push_back(
+          MemOpInfo(&SU, BaseOps, Offset, OffsetIsScalable, Width));
 
       LLVM_DEBUG(dbgs() << "Num BaseOps: " << BaseOps.size() << ", Offset: "
                         << Offset << ", OffsetIsScalable: " << OffsetIsScalable
@@ -2237,8 +2243,9 @@ init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel) {
            PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
       unsigned PIdx = PI->ProcResourceIdx;
       unsigned Factor = SchedModel->getResourceFactor(PIdx);
-      assert(PI->Cycles >= PI->StartAtCycle);
-      RemainingCounts[PIdx] += (Factor * (PI->Cycles - PI->StartAtCycle));
+      assert(PI->ReleaseAtCycle >= PI->AcquireAtCycle);
+      RemainingCounts[PIdx] +=
+          (Factor * (PI->ReleaseAtCycle - PI->AcquireAtCycle));
     }
   }
 }
@@ -2291,15 +2298,15 @@ unsigned SchedBoundary::getLatencyStallCycles(SUnit *SU) {
 /// Compute the next cycle at which the given processor resource unit
 /// can be scheduled.
 unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
-                                                       unsigned Cycles,
-                                                       unsigned StartAtCycle) {
+                                                       unsigned ReleaseAtCycle,
+                                                       unsigned AcquireAtCycle) {
   if (SchedModel && SchedModel->enableIntervals()) {
     if (isTop())
       return ReservedResourceSegments[InstanceIdx].getFirstAvailableAtFromTop(
-          CurrCycle, StartAtCycle, Cycles);
+          CurrCycle, AcquireAtCycle, ReleaseAtCycle);
 
     return ReservedResourceSegments[InstanceIdx].getFirstAvailableAtFromBottom(
-        CurrCycle, StartAtCycle, Cycles);
+        CurrCycle, AcquireAtCycle, ReleaseAtCycle);
   }
 
   unsigned NextUnreserved = ReservedCycles[InstanceIdx];
@@ -2308,7 +2315,7 @@ unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
     return CurrCycle;
   // For bottom-up scheduling add the cycles needed for the current operation.
   if (!isTop())
-    NextUnreserved = std::max(CurrCycle, NextUnreserved + Cycles);
+    NextUnreserved = std::max(CurrCycle, NextUnreserved + ReleaseAtCycle);
   return NextUnreserved;
 }
 
@@ -2317,7 +2324,8 @@ unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
 /// instance in the reserved cycles vector.
 std::pair<unsigned, unsigned>
 SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
-                                    unsigned Cycles, unsigned StartAtCycle) {
+                                    unsigned ReleaseAtCycle,
+                                    unsigned AcquireAtCycle) {
   if (MischedDetailResourceBooking) {
     LLVM_DEBUG(dbgs() << "  Resource booking (@" << CurrCycle << "c): \n");
     LLVM_DEBUG(dumpReservedCycles());
@@ -2331,26 +2339,30 @@ SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
          "Cannot have zero instances of a ProcResource");
 
   if (isUnbufferedGroup(PIdx)) {
-    // If any subunits are used by the instruction, report that the resource
-    // group is available at 0, effectively removing the group record from
-    // hazarding and basing the hazarding decisions on the subunit records.
-    // Otherwise, choose the first available instance from among the subunits.
-    // Specifications which assign cycles to both the subunits and the group or
-    // which use an unbuffered group with buffered subunits will appear to
-    // schedule strangely. In the first case, the additional cycles for the
-    // group will be ignored.  In the second, the group will be ignored
-    // entirely.
+    // If any subunits are used by the instruction, report that the
+    // subunits of the resource group are available at the first cycle
+    // in which the unit is available, effectively removing the group
+    // record from hazarding and basing the hazarding decisions on the
+    // subunit records. Otherwise, choose the first available instance
+    // from among the subunits.  Specifications which assign cycles to
+    // both the subunits and the group or which use an unbuffered
+    // group with buffered subunits will appear to schedule
+    // strangely. In the first case, the additional cycles for the
+    // group will be ignored.  In the second, the group will be
+    // ignored entirely.
     for (const MCWriteProcResEntry &PE :
          make_range(SchedModel->getWriteProcResBegin(SC),
                     SchedModel->getWriteProcResEnd(SC)))
       if (ResourceGroupSubUnitMasks[PIdx][PE.ProcResourceIdx])
-        return std::make_pair(0u, StartIndex);
+        return std::make_pair(getNextResourceCycleByInstance(
+                                  StartIndex, ReleaseAtCycle, AcquireAtCycle),
+                              StartIndex);
 
     auto SubUnits = SchedModel->getProcResource(PIdx)->SubUnitsIdxBegin;
     for (unsigned I = 0, End = NumberOfInstances; I < End; ++I) {
       unsigned NextUnreserved, NextInstanceIdx;
       std::tie(NextUnreserved, NextInstanceIdx) =
-          getNextResourceCycle(SC, SubUnits[I], Cycles, StartAtCycle);
+          getNextResourceCycle(SC, SubUnits[I], ReleaseAtCycle, AcquireAtCycle);
       if (MinNextUnreserved > NextUnreserved) {
         InstanceIdx = NextInstanceIdx;
         MinNextUnreserved = NextUnreserved;
@@ -2362,7 +2374,7 @@ SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
   for (unsigned I = StartIndex, End = StartIndex + NumberOfInstances; I < End;
        ++I) {
     unsigned NextUnreserved =
-        getNextResourceCycleByInstance(I, Cycles, StartAtCycle);
+        getNextResourceCycleByInstance(I, ReleaseAtCycle, AcquireAtCycle);
     if (MischedDetailResourceBooking)
       LLVM_DEBUG(dbgs() << "    Instance " << I - StartIndex << " available @"
                         << NextUnreserved << "c\n");
@@ -2419,14 +2431,14 @@ bool SchedBoundary::checkHazard(SUnit *SU) {
           make_range(SchedModel->getWriteProcResBegin(SC),
                      SchedModel->getWriteProcResEnd(SC))) {
       unsigned ResIdx = PE.ProcResourceIdx;
-      unsigned Cycles = PE.Cycles;
-      unsigned StartAtCycle = PE.StartAtCycle;
+      unsigned ReleaseAtCycle = PE.ReleaseAtCycle;
+      unsigned AcquireAtCycle = PE.AcquireAtCycle;
       unsigned NRCycle, InstanceIdx;
       std::tie(NRCycle, InstanceIdx) =
-          getNextResourceCycle(SC, ResIdx, Cycles, StartAtCycle);
+          getNextResourceCycle(SC, ResIdx, ReleaseAtCycle, AcquireAtCycle);
       if (NRCycle > CurrCycle) {
 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
-        MaxObservedStall = std::max(Cycles, MaxObservedStall);
+        MaxObservedStall = std::max(ReleaseAtCycle, MaxObservedStall);
 #endif
         LLVM_DEBUG(dbgs() << "  SU(" << SU->NodeNum << ") "
                           << SchedModel->getResourceName(ResIdx)
@@ -2568,18 +2580,22 @@ void SchedBoundary::incExecutedResources(unsigned PIdx, unsigned Count) {
 
 /// Add the given processor resource to this scheduled zone.
 ///
-/// \param Cycles indicates the number of consecutive (non-pipelined) cycles
-/// during which this resource is consumed.
+/// \param ReleaseAtCycle indicates the number of consecutive (non-pipelined)
+/// cycles during which this resource is released.
+///
+/// \param AcquireAtCycle indicates the number of consecutive (non-pipelined)
+/// cycles at which the resource is aquired after issue (assuming no stalls).
 ///
 /// \return the next cycle at which the instruction may execute without
 /// oversubscribing resources.
 unsigned SchedBoundary::countResource(const MCSchedClassDesc *SC, unsigned PIdx,
-                                      unsigned Cycles, unsigned NextCycle,
-                                      unsigned StartAtCycle) {
+                                      unsigned ReleaseAtCycle,
+                                      unsigned NextCycle,
+                                      unsigned AcquireAtCycle) {
   unsigned Factor = SchedModel->getResourceFactor(PIdx);
-  unsigned Count = Factor * (Cycles - StartAtCycle);
+  unsigned Count = Factor * (ReleaseAtCycle- AcquireAtCycle);
   LLVM_DEBUG(dbgs() << "  " << SchedModel->getResourceName(PIdx) << " +"
-                    << Cycles << "x" << Factor << "u\n");
+                    << ReleaseAtCycle << "x" << Factor << "u\n");
 
   // Update Executed resources counts.
   incExecutedResources(PIdx, Count);
@@ -2598,7 +2614,7 @@ unsigned SchedBoundary::countResource(const MCSchedClassDesc *SC, unsigned PIdx,
   // For reserved resources, record the highest cycle using the resource.
   unsigned NextAvailable, InstanceIdx;
   std::tie(NextAvailable, InstanceIdx) =
-      getNextResourceCycle(SC, PIdx, Cycles, StartAtCycle);
+      getNextResourceCycle(SC, PIdx, ReleaseAtCycle, AcquireAtCycle);
   if (NextAvailable > CurrCycle) {
     LLVM_DEBUG(dbgs() << "  Resource conflict: "
                       << SchedModel->getResourceName(PIdx)
@@ -2677,8 +2693,9 @@ void SchedBoundary::bumpNode(SUnit *SU) {
     for (TargetSchedModel::ProcResIter
            PI = SchedModel->getWriteProcResBegin(SC),
            PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
-      unsigned RCycle = countResource(SC, PI->ProcResourceIdx, PI->Cycles,
-                                      NextCycle, PI->StartAtCycle);
+      unsigned RCycle =
+          countResource(SC, PI->ProcResourceIdx, PI->ReleaseAtCycle, NextCycle,
+                        PI->AcquireAtCycle);
       if (RCycle > NextCycle)
         NextCycle = RCycle;
     }
@@ -2695,27 +2712,27 @@ void SchedBoundary::bumpNode(SUnit *SU) {
 
           if (SchedModel && SchedModel->enableIntervals()) {
             unsigned ReservedUntil, InstanceIdx;
-            std::tie(ReservedUntil, InstanceIdx) =
-                getNextResourceCycle(SC, PIdx, PI->Cycles, PI->StartAtCycle);
+            std::tie(ReservedUntil, InstanceIdx) = getNextResourceCycle(
+                SC, PIdx, PI->ReleaseAtCycle, PI->AcquireAtCycle);
             if (isTop()) {
               ReservedResourceSegments[InstanceIdx].add(
                   ResourceSegments::getResourceIntervalTop(
-                      NextCycle, PI->StartAtCycle, PI->Cycles),
+                      NextCycle, PI->AcquireAtCycle, PI->ReleaseAtCycle),
                   MIResourceCutOff);
             } else {
               ReservedResourceSegments[InstanceIdx].add(
                   ResourceSegments::getResourceIntervalBottom(
-                      NextCycle, PI->StartAtCycle, PI->Cycles),
+                      NextCycle, PI->AcquireAtCycle, PI->ReleaseAtCycle),
                   MIResourceCutOff);
             }
           } else {
 
             unsigned ReservedUntil, InstanceIdx;
-            std::tie(ReservedUntil, InstanceIdx) =
-                getNextResourceCycle(SC, PIdx, PI->Cycles, PI->StartAtCycle);
+            std::tie(ReservedUntil, InstanceIdx) = getNextResourceCycle(
+                SC, PIdx, PI->ReleaseAtCycle, PI->AcquireAtCycle);
             if (isTop()) {
               ReservedCycles[InstanceIdx] =
-                  std::max(ReservedUntil, NextCycle + PI->Cycles);
+                  std::max(ReservedUntil, NextCycle + PI->ReleaseAtCycle);
             } else
               ReservedCycles[InstanceIdx] = NextCycle;
           }
@@ -2913,9 +2930,9 @@ initResourceDelta(const ScheduleDAGMI *DAG,
          PI = SchedModel->getWriteProcResBegin(SC),
          PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
     if (PI->ProcResourceIdx == Policy.ReduceResIdx)
-      ResDelta.CritResources += PI->Cycles;
+      ResDelta.CritResources += PI->ReleaseAtCycle;
     if (PI->ProcResourceIdx == Policy.DemandResIdx)
-      ResDelta.DemandedResources += PI->Cycles;
+      ResDelta.DemandedResources += PI->ReleaseAtCycle;
   }
 }
 
@@ -3956,7 +3973,7 @@ struct ILPOrder {
       if (ScheduledTrees->test(SchedTreeA) != ScheduledTrees->test(SchedTreeB))
         return ScheduledTrees->test(SchedTreeB);
 
-      // Trees with shallower connections have have lower priority.
+      // Trees with shallower connections have lower priority.
       if (DFSResult->getSubtreeLevel(SchedTreeA)
           != DFSResult->getSubtreeLevel(SchedTreeB)) {
         return DFSResult->getSubtreeLevel(SchedTreeA)
@@ -4243,7 +4260,7 @@ static bool sortIntervals(const ResourceSegments::IntervalTy &A,
 }
 
 unsigned ResourceSegments::getFirstAvailableAt(
-    unsigned CurrCycle, unsigned StartAtCycle, unsigned Cycle,
+    unsigned CurrCycle, unsigned AcquireAtCycle, unsigned Cycle,
     std::function<ResourceSegments::IntervalTy(unsigned, unsigned, unsigned)>
         IntervalBuilder) const {
   assert(std::is_sorted(std::begin(_Intervals), std::end(_Intervals),
@@ -4251,7 +4268,7 @@ unsigned ResourceSegments::getFirstAvailableAt(
          "Cannot execute on an un-sorted set of intervals.");
   unsigned RetCycle = CurrCycle;
   ResourceSegments::IntervalTy NewInterval =
-      IntervalBuilder(RetCycle, StartAtCycle, Cycle);
+      IntervalBuilder(RetCycle, AcquireAtCycle, Cycle);
   for (auto &Interval : _Intervals) {
     if (!intersects(NewInterval, Interval))
       continue;
@@ -4261,7 +4278,7 @@ unsigned ResourceSegments::getFirstAvailableAt(
     assert(Interval.second > NewInterval.first &&
            "Invalid intervals configuration.");
     RetCycle += (unsigned)Interval.second - (unsigned)NewInterval.first;
-    NewInterval = IntervalBuilder(RetCycle, StartAtCycle, Cycle);
+    NewInterval = IntervalBuilder(RetCycle, AcquireAtCycle, Cycle);
   }
   return RetCycle;
 }