1 files changed, 85 insertions, 59 deletions

diff --git a/lib/Target/SystemZ/SystemZHazardRecognizer.cpp b/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
index f37216022762..d300d1d88abc 100644
--- a/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
+++ b/lib/Target/SystemZ/SystemZHazardRecognizer.cpp
@@ -59,10 +59,18 @@ getNumDecoderSlots(SUnit *SU) const {
   return 1; // Normal instruction
 }
 
-unsigned SystemZHazardRecognizer::getCurrCycleIdx() {
+unsigned SystemZHazardRecognizer::getCurrCycleIdx(SUnit *SU) const {
   unsigned Idx = CurrGroupSize;
   if (GrpCount % 2)
     Idx += 3;
+
+  if (SU != nullptr && !fitsIntoCurrentGroup(SU)) {
+    if (Idx == 1 || Idx == 2)
+      Idx = 3;
+    else if (Idx == 4 || Idx == 5)
+      Idx = 0;
+  }
+
   return Idx;
 }
 
@@ -77,7 +85,7 @@ void SystemZHazardRecognizer::Reset() {
   GrpCount = 0;
   LastFPdOpCycleIdx = UINT_MAX;
   LastEmittedMI = nullptr;
-  DEBUG(CurGroupDbg = "";);
+  LLVM_DEBUG(CurGroupDbg = "";);
 }
 
 bool
@@ -100,30 +108,30 @@ SystemZHazardRecognizer::fitsIntoCurrentGroup(SUnit *SU) const {
   return true;
 }
 
-void SystemZHazardRecognizer::nextGroup(bool DbgOutput) {
-  if (CurrGroupSize > 0) {
-    DEBUG(dumpCurrGroup("Completed decode group"));
-    DEBUG(CurGroupDbg = "";);
+void SystemZHazardRecognizer::nextGroup() {
+  if (CurrGroupSize == 0)
+    return;
+
+  LLVM_DEBUG(dumpCurrGroup("Completed decode group"));
+  LLVM_DEBUG(CurGroupDbg = "";);
 
-    GrpCount++;
+  GrpCount++;
 
-    // Reset counter for next group.
-    CurrGroupSize = 0;
+  // Reset counter for next group.
+  CurrGroupSize = 0;
 
-    // Decrease counters for execution units by one.
-    for (unsigned i = 0; i < SchedModel->getNumProcResourceKinds(); ++i)
-      if (ProcResourceCounters[i] > 0)
-        ProcResourceCounters[i]--;
+  // Decrease counters for execution units by one.
+  for (unsigned i = 0; i < SchedModel->getNumProcResourceKinds(); ++i)
+    if (ProcResourceCounters[i] > 0)
+      ProcResourceCounters[i]--;
 
-    // Clear CriticalResourceIdx if it is now below the threshold.
-    if (CriticalResourceIdx != UINT_MAX &&
-        (ProcResourceCounters[CriticalResourceIdx] <=
-         ProcResCostLim))
-      CriticalResourceIdx = UINT_MAX;
-  }
+  // Clear CriticalResourceIdx if it is now below the threshold.
+  if (CriticalResourceIdx != UINT_MAX &&
+      (ProcResourceCounters[CriticalResourceIdx] <=
+       ProcResCostLim))
+    CriticalResourceIdx = UINT_MAX;
 
-  DEBUG(if (DbgOutput)
-          dumpProcResourceCounters(););
+  LLVM_DEBUG(dumpState(););
 }
 
 #ifndef NDEBUG // Debug output
@@ -143,7 +151,11 @@ void SystemZHazardRecognizer::dumpSU(SUnit *SU, raw_ostream &OS) const {
     std::string FU(PRD.Name);
     // trim e.g. Z13_FXaUnit -> FXa
     FU = FU.substr(FU.find("_") + 1);
-    FU.resize(FU.find("Unit"));
+    size_t Pos = FU.find("Unit");
+    if (Pos != std::string::npos)
+      FU.resize(Pos);
+    if (FU == "LS") // LSUnit -> LSU
+      FU = "LSU";
     OS << "/" << FU;
 
     if (PI->Cycles > 1)
@@ -163,7 +175,7 @@ void SystemZHazardRecognizer::dumpSU(SUnit *SU, raw_ostream &OS) const {
 }
 
 void SystemZHazardRecognizer::dumpCurrGroup(std::string Msg) const {
-  dbgs() << "+++ " << Msg;
+  dbgs() << "++ " << Msg;
   dbgs() << ": ";
 
   if (CurGroupDbg.empty())
@@ -188,15 +200,28 @@ void SystemZHazardRecognizer::dumpProcResourceCounters() const {
   if (!any)
     return;
 
-  dbgs() << "+++ Resource counters:\n";
+  dbgs() << "++ | Resource counters: ";
   for (unsigned i = 0; i < SchedModel->getNumProcResourceKinds(); ++i)
-    if (ProcResourceCounters[i] > 0) {
-      dbgs() << "+++ Extra schedule for execution unit "
-             << SchedModel->getProcResource(i)->Name
-             << ": " << ProcResourceCounters[i] << "\n";
-      any = true;
-    }
+    if (ProcResourceCounters[i] > 0)
+      dbgs() << SchedModel->getProcResource(i)->Name
+             << ":" << ProcResourceCounters[i] << " ";
+  dbgs() << "\n";
+
+  if (CriticalResourceIdx != UINT_MAX)
+    dbgs() << "++ | Critical resource: "
+           << SchedModel->getProcResource(CriticalResourceIdx)->Name
+           << "\n";
+}
+
+void SystemZHazardRecognizer::dumpState() const {
+  dumpCurrGroup("| Current decoder group");
+  dbgs() << "++ | Current cycle index: "
+         << getCurrCycleIdx() << "\n";
+  dumpProcResourceCounters();
+  if (LastFPdOpCycleIdx != UINT_MAX)
+    dbgs() << "++ | Last FPd cycle index: " << LastFPdOpCycleIdx << "\n";
 }
+
 #endif //NDEBUG
 
 void SystemZHazardRecognizer::clearProcResCounters() {
@@ -213,30 +238,25 @@ static inline bool isBranchRetTrap(MachineInstr *MI) {
 void SystemZHazardRecognizer::
 EmitInstruction(SUnit *SU) {
   const MCSchedClassDesc *SC = getSchedClass(SU);
-  DEBUG( dumpCurrGroup("Decode group before emission"););
+  LLVM_DEBUG(dbgs() << "++ HazardRecognizer emitting "; dumpSU(SU, dbgs());
+             dbgs() << "\n";);
+  LLVM_DEBUG(dumpCurrGroup("Decode group before emission"););
 
   // If scheduling an SU that must begin a new decoder group, move on
   // to next group.
   if (!fitsIntoCurrentGroup(SU))
     nextGroup();
 
-  DEBUG( dbgs() << "+++ HazardRecognizer emitting "; dumpSU(SU, dbgs());
-         dbgs() << "\n";
-         raw_string_ostream cgd(CurGroupDbg);
-         if (CurGroupDbg.length())
-           cgd << ", ";
-         dumpSU(SU, cgd););
+  LLVM_DEBUG(raw_string_ostream cgd(CurGroupDbg);
+             if (CurGroupDbg.length()) cgd << ", "; dumpSU(SU, cgd););
 
   LastEmittedMI = SU->getInstr();
 
   // After returning from a call, we don't know much about the state.
   if (SU->isCall) {
-    DEBUG (dbgs() << "+++ Clearing state after call.\n";);
-    clearProcResCounters();
-    LastFPdOpCycleIdx = UINT_MAX;
-    CurrGroupSize += getNumDecoderSlots(SU);
-    assert (CurrGroupSize <= 3);
-    nextGroup();
+    LLVM_DEBUG(dbgs() << "++ Clearing state after call.\n";);
+    Reset();
+    LastEmittedMI = SU->getInstr();
     return;
   }
 
@@ -256,23 +276,21 @@ EmitInstruction(SUnit *SU) {
          (PI->ProcResourceIdx != CriticalResourceIdx &&
           CurrCounter >
           ProcResourceCounters[CriticalResourceIdx]))) {
-      DEBUG( dbgs() << "+++ New critical resource: "
-             << SchedModel->getProcResource(PI->ProcResourceIdx)->Name
-             << "\n";);
+      LLVM_DEBUG(
+          dbgs() << "++ New critical resource: "
+                 << SchedModel->getProcResource(PI->ProcResourceIdx)->Name
+                 << "\n";);
       CriticalResourceIdx = PI->ProcResourceIdx;
     }
   }
 
   // Make note of an instruction that uses a blocking resource (FPd).
   if (SU->isUnbuffered) {
-    LastFPdOpCycleIdx = getCurrCycleIdx();
-    DEBUG (dbgs() << "+++ Last FPd cycle index: "
-           << LastFPdOpCycleIdx << "\n";);
+    LastFPdOpCycleIdx = getCurrCycleIdx(SU);
+    LLVM_DEBUG(dbgs() << "++ Last FPd cycle index: " << LastFPdOpCycleIdx
+                      << "\n";);
   }
 
-  bool GroupEndingBranch =
-    (CurrGroupSize >= 1 && isBranchRetTrap(SU->getInstr()));
-
   // Insert SU into current group by increasing number of slots used
   // in current group.
   CurrGroupSize += getNumDecoderSlots(SU);
@@ -280,7 +298,7 @@ EmitInstruction(SUnit *SU) {
 
   // Check if current group is now full/ended. If so, move on to next
   // group to be ready to evaluate more candidates.
-  if (CurrGroupSize == 3 || SC->EndGroup || GroupEndingBranch)
+  if (CurrGroupSize == 3 || SC->EndGroup)
     nextGroup();
 }
 
@@ -311,7 +329,7 @@ int SystemZHazardRecognizer::groupingCost(SUnit *SU) const {
   return 0;
 }
 
-bool SystemZHazardRecognizer::isFPdOpPreferred_distance(const SUnit *SU) {
+bool SystemZHazardRecognizer::isFPdOpPreferred_distance(SUnit *SU) const {
   assert (SU->isUnbuffered);
   // If this is the first FPd op, it should be scheduled high.
   if (LastFPdOpCycleIdx == UINT_MAX)
@@ -320,9 +338,10 @@ bool SystemZHazardRecognizer::isFPdOpPreferred_distance(const SUnit *SU) {
   // of the processor to use the other FPd unit there. This should
   // generally happen if two FPd ops are placed with 2 other
   // instructions between them (modulo 6).
-  if (LastFPdOpCycleIdx > getCurrCycleIdx())
-    return ((LastFPdOpCycleIdx - getCurrCycleIdx()) == 3);
-  return ((getCurrCycleIdx() - LastFPdOpCycleIdx) == 3);
+  unsigned SUCycleIdx = getCurrCycleIdx(SU);
+  if (LastFPdOpCycleIdx > SUCycleIdx)
+    return ((LastFPdOpCycleIdx - SUCycleIdx) == 3);
+  return ((SUCycleIdx - LastFPdOpCycleIdx) == 3);
 }
 
 int SystemZHazardRecognizer::
@@ -373,10 +392,17 @@ void SystemZHazardRecognizer::emitInstruction(MachineInstr *MI,
     }
   }
 
+  unsigned GroupSizeBeforeEmit = CurrGroupSize;
   EmitInstruction(&SU);
 
+  if (!TakenBranch && isBranchRetTrap(MI)) {
+    // NT Branch on second slot ends group.
+    if (GroupSizeBeforeEmit == 1)
+      nextGroup();
+  }
+
   if (TakenBranch && CurrGroupSize > 0)
-    nextGroup(false /*DbgOutput*/);
+    nextGroup();
 
   assert ((!MI->isTerminator() || isBranchRetTrap(MI)) &&
           "Scheduler: unhandled terminator!");
@@ -386,7 +412,7 @@ void SystemZHazardRecognizer::
 copyState(SystemZHazardRecognizer *Incoming) {
   // Current decoder group
   CurrGroupSize = Incoming->CurrGroupSize;
-  DEBUG (CurGroupDbg = Incoming->CurGroupDbg;);
+  LLVM_DEBUG(CurGroupDbg = Incoming->CurGroupDbg;);
 
   // Processor resources
   ProcResourceCounters = Incoming->ProcResourceCounters;