summaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp1874
1 files changed, 1874 insertions, 0 deletions
diff --git a/contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
new file mode 100644
index 000000000000..3619e4c239d7
--- /dev/null
+++ b/contrib/llvm-project/llvm/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -0,0 +1,1874 @@
+//===- HexagonPacketizer.cpp - VLIW packetizer ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This implements a simple VLIW packetizer using DFA. The packetizer works on
+// machine basic blocks. For each instruction I in BB, the packetizer consults
+// the DFA to see if machine resources are available to execute I. If so, the
+// packetizer checks if I depends on any instruction J in the current packet.
+// If no dependency is found, I is added to current packet and machine resource
+// is marked as taken. If any dependency is found, a target API call is made to
+// prune the dependence.
+//
+//===----------------------------------------------------------------------===//
+
+#include "HexagonVLIWPacketizer.h"
+#include "Hexagon.h"
+#include "HexagonInstrInfo.h"
+#include "HexagonRegisterInfo.h"
+#include "HexagonSubtarget.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBundle.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/ScheduleDAG.h"
+#include "llvm/CodeGen/TargetRegisterInfo.h"
+#include "llvm/CodeGen/TargetSubtargetInfo.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/MC/MCInstrDesc.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cassert>
+#include <cstdint>
+#include <iterator>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "packets"
+
+static cl::opt<bool> DisablePacketizer("disable-packetizer", cl::Hidden,
+ cl::ZeroOrMore, cl::init(false),
+ cl::desc("Disable Hexagon packetizer pass"));
+
+cl::opt<bool> Slot1Store("slot1-store-slot0-load", cl::Hidden,
+ cl::ZeroOrMore, cl::init(true),
+ cl::desc("Allow slot1 store and slot0 load"));
+
+static cl::opt<bool> PacketizeVolatiles("hexagon-packetize-volatiles",
+ cl::ZeroOrMore, cl::Hidden, cl::init(true),
+ cl::desc("Allow non-solo packetization of volatile memory references"));
+
+static cl::opt<bool> EnableGenAllInsnClass("enable-gen-insn", cl::init(false),
+ cl::Hidden, cl::ZeroOrMore, cl::desc("Generate all instruction with TC"));
+
+static cl::opt<bool> DisableVecDblNVStores("disable-vecdbl-nv-stores",
+ cl::init(false), cl::Hidden, cl::ZeroOrMore,
+ cl::desc("Disable vector double new-value-stores"));
+
+extern cl::opt<bool> ScheduleInlineAsm;
+
+namespace llvm {
+
+FunctionPass *createHexagonPacketizer(bool Minimal);
+void initializeHexagonPacketizerPass(PassRegistry&);
+
+} // end namespace llvm
+
+namespace {
+
+ class HexagonPacketizer : public MachineFunctionPass {
+ public:
+ static char ID;
+
+ HexagonPacketizer(bool Min = false)
+ : MachineFunctionPass(ID), Minimal(Min) {}
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ AU.addRequired<AAResultsWrapperPass>();
+ AU.addRequired<MachineBranchProbabilityInfo>();
+ AU.addRequired<MachineDominatorTree>();
+ AU.addRequired<MachineLoopInfo>();
+ AU.addPreserved<MachineDominatorTree>();
+ AU.addPreserved<MachineLoopInfo>();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ StringRef getPassName() const override { return "Hexagon Packetizer"; }
+ bool runOnMachineFunction(MachineFunction &Fn) override;
+
+ MachineFunctionProperties getRequiredProperties() const override {
+ return MachineFunctionProperties().set(
+ MachineFunctionProperties::Property::NoVRegs);
+ }
+
+ private:
+ const HexagonInstrInfo *HII;
+ const HexagonRegisterInfo *HRI;
+ const bool Minimal;
+ };
+
+} // end anonymous namespace
+
+char HexagonPacketizer::ID = 0;
+
+INITIALIZE_PASS_BEGIN(HexagonPacketizer, "hexagon-packetizer",
+ "Hexagon Packetizer", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(MachineBranchProbabilityInfo)
+INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_END(HexagonPacketizer, "hexagon-packetizer",
+ "Hexagon Packetizer", false, false)
+
+HexagonPacketizerList::HexagonPacketizerList(MachineFunction &MF,
+ MachineLoopInfo &MLI, AliasAnalysis *AA,
+ const MachineBranchProbabilityInfo *MBPI, bool Minimal)
+ : VLIWPacketizerList(MF, MLI, AA), MBPI(MBPI), MLI(&MLI),
+ Minimal(Minimal) {
+ HII = MF.getSubtarget<HexagonSubtarget>().getInstrInfo();
+ HRI = MF.getSubtarget<HexagonSubtarget>().getRegisterInfo();
+
+ addMutation(llvm::make_unique<HexagonSubtarget::UsrOverflowMutation>());
+ addMutation(llvm::make_unique<HexagonSubtarget::HVXMemLatencyMutation>());
+ addMutation(llvm::make_unique<HexagonSubtarget::BankConflictMutation>());
+}
+
+// Check if FirstI modifies a register that SecondI reads.
+static bool hasWriteToReadDep(const MachineInstr &FirstI,
+ const MachineInstr &SecondI,
+ const TargetRegisterInfo *TRI) {
+ for (auto &MO : FirstI.operands()) {
+ if (!MO.isReg() || !MO.isDef())
+ continue;
+ unsigned R = MO.getReg();
+ if (SecondI.readsRegister(R, TRI))
+ return true;
+ }
+ return false;
+}
+
+
+static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI,
+ MachineBasicBlock::iterator BundleIt, bool Before) {
+ MachineBasicBlock::instr_iterator InsertPt;
+ if (Before)
+ InsertPt = BundleIt.getInstrIterator();
+ else
+ InsertPt = std::next(BundleIt).getInstrIterator();
+
+ MachineBasicBlock &B = *MI.getParent();
+ // The instruction should at least be bundled with the preceding instruction
+ // (there will always be one, i.e. BUNDLE, if nothing else).
+ assert(MI.isBundledWithPred());
+ if (MI.isBundledWithSucc()) {
+ MI.clearFlag(MachineInstr::BundledSucc);
+ MI.clearFlag(MachineInstr::BundledPred);
+ } else {
+ // If it's not bundled with the successor (i.e. it is the last one
+ // in the bundle), then we can simply unbundle it from the predecessor,
+ // which will take care of updating the predecessor's flag.
+ MI.unbundleFromPred();
+ }
+ B.splice(InsertPt, &B, MI.getIterator());
+
+ // Get the size of the bundle without asserting.
+ MachineBasicBlock::const_instr_iterator I = BundleIt.getInstrIterator();
+ MachineBasicBlock::const_instr_iterator E = B.instr_end();
+ unsigned Size = 0;
+ for (++I; I != E && I->isBundledWithPred(); ++I)
+ ++Size;
+
+ // If there are still two or more instructions, then there is nothing
+ // else to be done.
+ if (Size > 1)
+ return BundleIt;
+
+ // Otherwise, extract the single instruction out and delete the bundle.
+ MachineBasicBlock::iterator NextIt = std::next(BundleIt);
+ MachineInstr &SingleI = *BundleIt->getNextNode();
+ SingleI.unbundleFromPred();
+ assert(!SingleI.isBundledWithSucc());
+ BundleIt->eraseFromParent();
+ return NextIt;
+}
+
+bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {
+ auto &HST = MF.getSubtarget<HexagonSubtarget>();
+ HII = HST.getInstrInfo();
+ HRI = HST.getRegisterInfo();
+ auto &MLI = getAnalysis<MachineLoopInfo>();
+ auto *AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+ auto *MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
+
+ if (EnableGenAllInsnClass)
+ HII->genAllInsnTimingClasses(MF);
+
+ // Instantiate the packetizer.
+ bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() ||
+ skipFunction(MF.getFunction());
+ HexagonPacketizerList Packetizer(MF, MLI, AA, MBPI, MinOnly);
+
+ // DFA state table should not be empty.
+ assert(Packetizer.getResourceTracker() && "Empty DFA table!");
+
+ // Loop over all basic blocks and remove KILL pseudo-instructions
+ // These instructions confuse the dependence analysis. Consider:
+ // D0 = ... (Insn 0)
+ // R0 = KILL R0, D0 (Insn 1)
+ // R0 = ... (Insn 2)
+ // Here, Insn 1 will result in the dependence graph not emitting an output
+ // dependence between Insn 0 and Insn 2. This can lead to incorrect
+ // packetization
+ for (MachineBasicBlock &MB : MF) {
+ auto End = MB.end();
+ auto MI = MB.begin();
+ while (MI != End) {
+ auto NextI = std::next(MI);
+ if (MI->isKill()) {
+ MB.erase(MI);
+ End = MB.end();
+ }
+ MI = NextI;
+ }
+ }
+
+ // Loop over all of the basic blocks.
+ for (auto &MB : MF) {
+ auto Begin = MB.begin(), End = MB.end();
+ while (Begin != End) {
+ // Find the first non-boundary starting from the end of the last
+ // scheduling region.
+ MachineBasicBlock::iterator RB = Begin;
+ while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))
+ ++RB;
+ // Find the first boundary starting from the beginning of the new
+ // region.
+ MachineBasicBlock::iterator RE = RB;
+ while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))
+ ++RE;
+ // Add the scheduling boundary if it's not block end.
+ if (RE != End)
+ ++RE;
+ // If RB == End, then RE == End.
+ if (RB != End)
+ Packetizer.PacketizeMIs(&MB, RB, RE);
+
+ Begin = RE;
+ }
+ }
+
+ Packetizer.unpacketizeSoloInstrs(MF);
+ return true;
+}
+
+// Reserve resources for a constant extender. Trigger an assertion if the
+// reservation fails.
+void HexagonPacketizerList::reserveResourcesForConstExt() {
+ if (!tryAllocateResourcesForConstExt(true))
+ llvm_unreachable("Resources not available");
+}
+
+bool HexagonPacketizerList::canReserveResourcesForConstExt() {
+ return tryAllocateResourcesForConstExt(false);
+}
+
+// Allocate resources (i.e. 4 bytes) for constant extender. If succeeded,
+// return true, otherwise, return false.
+bool HexagonPacketizerList::tryAllocateResourcesForConstExt(bool Reserve) {
+ auto *ExtMI = MF.CreateMachineInstr(HII->get(Hexagon::A4_ext), DebugLoc());
+ bool Avail = ResourceTracker->canReserveResources(*ExtMI);
+ if (Reserve && Avail)
+ ResourceTracker->reserveResources(*ExtMI);
+ MF.DeleteMachineInstr(ExtMI);
+ return Avail;
+}
+
+bool HexagonPacketizerList::isCallDependent(const MachineInstr &MI,
+ SDep::Kind DepType, unsigned DepReg) {
+ // Check for LR dependence.
+ if (DepReg == HRI->getRARegister())
+ return true;
+
+ if (HII->isDeallocRet(MI))
+ if (DepReg == HRI->getFrameRegister() || DepReg == HRI->getStackRegister())
+ return true;
+
+ // Call-like instructions can be packetized with preceding instructions
+ // that define registers implicitly used or modified by the call. Explicit
+ // uses are still prohibited, as in the case of indirect calls:
+ // r0 = ...
+ // J2_jumpr r0
+ if (DepType == SDep::Data) {
+ for (const MachineOperand MO : MI.operands())
+ if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())
+ return true;
+ }
+
+ return false;
+}
+
+static bool isRegDependence(const SDep::Kind DepType) {
+ return DepType == SDep::Data || DepType == SDep::Anti ||
+ DepType == SDep::Output;
+}
+
+static bool isDirectJump(const MachineInstr &MI) {
+ return MI.getOpcode() == Hexagon::J2_jump;
+}
+
+static bool isSchedBarrier(const MachineInstr &MI) {
+ switch (MI.getOpcode()) {
+ case Hexagon::Y2_barrier:
+ return true;
+ }
+ return false;
+}
+
+static bool isControlFlow(const MachineInstr &MI) {
+ return MI.getDesc().isTerminator() || MI.getDesc().isCall();
+}
+
+/// Returns true if the instruction modifies a callee-saved register.
+static bool doesModifyCalleeSavedReg(const MachineInstr &MI,
+ const TargetRegisterInfo *TRI) {
+ const MachineFunction &MF = *MI.getParent()->getParent();
+ for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)
+ if (MI.modifiesRegister(*CSR, TRI))
+ return true;
+ return false;
+}
+
+// Returns true if an instruction can be promoted to .new predicate or
+// new-value store.
+bool HexagonPacketizerList::isNewifiable(const MachineInstr &MI,
+ const TargetRegisterClass *NewRC) {
+ // Vector stores can be predicated, and can be new-value stores, but
+ // they cannot be predicated on a .new predicate value.
+ if (NewRC == &Hexagon::PredRegsRegClass) {
+ if (HII->isHVXVec(MI) && MI.mayStore())
+ return false;
+ return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0;
+ }
+ // If the class is not PredRegs, it could only apply to new-value stores.
+ return HII->mayBeNewStore(MI);
+}
+
+// Promote an instructiont to its .cur form.
+// At this time, we have already made a call to canPromoteToDotCur and made
+// sure that it can *indeed* be promoted.
+bool HexagonPacketizerList::promoteToDotCur(MachineInstr &MI,
+ SDep::Kind DepType, MachineBasicBlock::iterator &MII,
+ const TargetRegisterClass* RC) {
+ assert(DepType == SDep::Data);
+ int CurOpcode = HII->getDotCurOp(MI);
+ MI.setDesc(HII->get(CurOpcode));
+ return true;
+}
+
+void HexagonPacketizerList::cleanUpDotCur() {
+ MachineInstr *MI = nullptr;
+ for (auto BI : CurrentPacketMIs) {
+ LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););
+ if (HII->isDotCurInst(*BI)) {
+ MI = BI;
+ continue;
+ }
+ if (MI) {
+ for (auto &MO : BI->operands())
+ if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())
+ return;
+ }
+ }
+ if (!MI)
+ return;
+ // We did not find a use of the CUR, so de-cur it.
+ MI->setDesc(HII->get(HII->getNonDotCurOp(*MI)));
+ LLVM_DEBUG(dbgs() << "Demoted CUR "; MI->dump(););
+}
+
+// Check to see if an instruction can be dot cur.
+bool HexagonPacketizerList::canPromoteToDotCur(const MachineInstr &MI,
+ const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
+ const TargetRegisterClass *RC) {
+ if (!HII->isHVXVec(MI))
+ return false;
+ if (!HII->isHVXVec(*MII))
+ return false;
+
+ // Already a dot new instruction.
+ if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))
+ return false;
+
+ if (!HII->mayBeCurLoad(MI))
+ return false;
+
+ // The "cur value" cannot come from inline asm.
+ if (PacketSU->getInstr()->isInlineAsm())
+ return false;
+
+ // Make sure candidate instruction uses cur.
+ LLVM_DEBUG(dbgs() << "Can we DOT Cur Vector MI\n"; MI.dump();
+ dbgs() << "in packet\n";);
+ MachineInstr &MJ = *MII;
+ LLVM_DEBUG({
+ dbgs() << "Checking CUR against ";
+ MJ.dump();
+ });
+ unsigned DestReg = MI.getOperand(0).getReg();
+ bool FoundMatch = false;
+ for (auto &MO : MJ.operands())
+ if (MO.isReg() && MO.getReg() == DestReg)
+ FoundMatch = true;
+ if (!FoundMatch)
+ return false;
+
+ // Check for existing uses of a vector register within the packet which
+ // would be affected by converting a vector load into .cur formt.
+ for (auto BI : CurrentPacketMIs) {
+ LLVM_DEBUG(dbgs() << "packet has "; BI->dump(););
+ if (BI->readsRegister(DepReg, MF.getSubtarget().getRegisterInfo()))
+ return false;
+ }
+
+ LLVM_DEBUG(dbgs() << "Can Dot CUR MI\n"; MI.dump(););
+ // We can convert the opcode into a .cur.
+ return true;
+}
+
+// Promote an instruction to its .new form. At this time, we have already
+// made a call to canPromoteToDotNew and made sure that it can *indeed* be
+// promoted.
+bool HexagonPacketizerList::promoteToDotNew(MachineInstr &MI,
+ SDep::Kind DepType, MachineBasicBlock::iterator &MII,
+ const TargetRegisterClass* RC) {
+ assert(DepType == SDep::Data);
+ int NewOpcode;
+ if (RC == &Hexagon::PredRegsRegClass)
+ NewOpcode = HII->getDotNewPredOp(MI, MBPI);
+ else
+ NewOpcode = HII->getDotNewOp(MI);
+ MI.setDesc(HII->get(NewOpcode));
+ return true;
+}
+
+bool HexagonPacketizerList::demoteToDotOld(MachineInstr &MI) {
+ int NewOpcode = HII->getDotOldOp(MI);
+ MI.setDesc(HII->get(NewOpcode));
+ return true;
+}
+
+bool HexagonPacketizerList::useCallersSP(MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ switch (Opc) {
+ case Hexagon::S2_storerd_io:
+ case Hexagon::S2_storeri_io:
+ case Hexagon::S2_storerh_io:
+ case Hexagon::S2_storerb_io:
+ break;
+ default:
+ llvm_unreachable("Unexpected instruction");
+ }
+ unsigned FrameSize = MF.getFrameInfo().getStackSize();
+ MachineOperand &Off = MI.getOperand(1);
+ int64_t NewOff = Off.getImm() - (FrameSize + HEXAGON_LRFP_SIZE);
+ if (HII->isValidOffset(Opc, NewOff, HRI)) {
+ Off.setImm(NewOff);
+ return true;
+ }
+ return false;
+}
+
+void HexagonPacketizerList::useCalleesSP(MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ switch (Opc) {
+ case Hexagon::S2_storerd_io:
+ case Hexagon::S2_storeri_io:
+ case Hexagon::S2_storerh_io:
+ case Hexagon::S2_storerb_io:
+ break;
+ default:
+ llvm_unreachable("Unexpected instruction");
+ }
+ unsigned FrameSize = MF.getFrameInfo().getStackSize();
+ MachineOperand &Off = MI.getOperand(1);
+ Off.setImm(Off.getImm() + FrameSize + HEXAGON_LRFP_SIZE);
+}
+
+/// Return true if we can update the offset in MI so that MI and MJ
+/// can be packetized together.
+bool HexagonPacketizerList::updateOffset(SUnit *SUI, SUnit *SUJ) {
+ assert(SUI->getInstr() && SUJ->getInstr());
+ MachineInstr &MI = *SUI->getInstr();
+ MachineInstr &MJ = *SUJ->getInstr();
+
+ unsigned BPI, OPI;
+ if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI))
+ return false;
+ unsigned BPJ, OPJ;
+ if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ))
+ return false;
+ unsigned Reg = MI.getOperand(BPI).getReg();
+ if (Reg != MJ.getOperand(BPJ).getReg())
+ return false;
+ // Make sure that the dependences do not restrict adding MI to the packet.
+ // That is, ignore anti dependences, and make sure the only data dependence
+ // involves the specific register.
+ for (const auto &PI : SUI->Preds)
+ if (PI.getKind() != SDep::Anti &&
+ (PI.getKind() != SDep::Data || PI.getReg() != Reg))
+ return false;
+ int Incr;
+ if (!HII->getIncrementValue(MJ, Incr))
+ return false;
+
+ int64_t Offset = MI.getOperand(OPI).getImm();
+ if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI))
+ return false;
+
+ MI.getOperand(OPI).setImm(Offset + Incr);
+ ChangedOffset = Offset;
+ return true;
+}
+
+/// Undo the changed offset. This is needed if the instruction cannot be
+/// added to the current packet due to a different instruction.
+void HexagonPacketizerList::undoChangedOffset(MachineInstr &MI) {
+ unsigned BP, OP;
+ if (!HII->getBaseAndOffsetPosition(MI, BP, OP))
+ llvm_unreachable("Unable to find base and offset operands.");
+ MI.getOperand(OP).setImm(ChangedOffset);
+}
+
+enum PredicateKind {
+ PK_False,
+ PK_True,
+ PK_Unknown
+};
+
+/// Returns true if an instruction is predicated on p0 and false if it's
+/// predicated on !p0.
+static PredicateKind getPredicateSense(const MachineInstr &MI,
+ const HexagonInstrInfo *HII) {
+ if (!HII->isPredicated(MI))
+ return PK_Unknown;
+ if (HII->isPredicatedTrue(MI))
+ return PK_True;
+ return PK_False;
+}
+
+static const MachineOperand &getPostIncrementOperand(const MachineInstr &MI,
+ const HexagonInstrInfo *HII) {
+ assert(HII->isPostIncrement(MI) && "Not a post increment operation.");
+#ifndef NDEBUG
+ // Post Increment means duplicates. Use dense map to find duplicates in the
+ // list. Caution: Densemap initializes with the minimum of 64 buckets,
+ // whereas there are at most 5 operands in the post increment.
+ DenseSet<unsigned> DefRegsSet;
+ for (auto &MO : MI.operands())
+ if (MO.isReg() && MO.isDef())
+ DefRegsSet.insert(MO.getReg());
+
+ for (auto &MO : MI.operands())
+ if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))
+ return MO;
+#else
+ if (MI.mayLoad()) {
+ const MachineOperand &Op1 = MI.getOperand(1);
+ // The 2nd operand is always the post increment operand in load.
+ assert(Op1.isReg() && "Post increment operand has be to a register.");
+ return Op1;
+ }
+ if (MI.getDesc().mayStore()) {
+ const MachineOperand &Op0 = MI.getOperand(0);
+ // The 1st operand is always the post increment operand in store.
+ assert(Op0.isReg() && "Post increment operand has be to a register.");
+ return Op0;
+ }
+#endif
+ // we should never come here.
+ llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");
+}
+
+// Get the value being stored.
+static const MachineOperand& getStoreValueOperand(const MachineInstr &MI) {
+ // value being stored is always the last operand.
+ return MI.getOperand(MI.getNumOperands()-1);
+}
+
+static bool isLoadAbsSet(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ switch (Opc) {
+ case Hexagon::L4_loadrd_ap:
+ case Hexagon::L4_loadrb_ap:
+ case Hexagon::L4_loadrh_ap:
+ case Hexagon::L4_loadrub_ap:
+ case Hexagon::L4_loadruh_ap:
+ case Hexagon::L4_loadri_ap:
+ return true;
+ }
+ return false;
+}
+
+static const MachineOperand &getAbsSetOperand(const MachineInstr &MI) {
+ assert(isLoadAbsSet(MI));
+ return MI.getOperand(1);
+}
+
+// Can be new value store?
+// Following restrictions are to be respected in convert a store into
+// a new value store.
+// 1. If an instruction uses auto-increment, its address register cannot
+// be a new-value register. Arch Spec 5.4.2.1
+// 2. If an instruction uses absolute-set addressing mode, its address
+// register cannot be a new-value register. Arch Spec 5.4.2.1.
+// 3. If an instruction produces a 64-bit result, its registers cannot be used
+// as new-value registers. Arch Spec 5.4.2.2.
+// 4. If the instruction that sets the new-value register is conditional, then
+// the instruction that uses the new-value register must also be conditional,
+// and both must always have their predicates evaluate identically.
+// Arch Spec 5.4.2.3.
+// 5. There is an implied restriction that a packet cannot have another store,
+// if there is a new value store in the packet. Corollary: if there is
+// already a store in a packet, there can not be a new value store.
+// Arch Spec: 3.4.4.2
+bool HexagonPacketizerList::canPromoteToNewValueStore(const MachineInstr &MI,
+ const MachineInstr &PacketMI, unsigned DepReg) {
+ // Make sure we are looking at the store, that can be promoted.
+ if (!HII->mayBeNewStore(MI))
+ return false;
+
+ // Make sure there is dependency and can be new value'd.
+ const MachineOperand &Val = getStoreValueOperand(MI);
+ if (Val.isReg() && Val.getReg() != DepReg)
+ return false;
+
+ const MCInstrDesc& MCID = PacketMI.getDesc();
+
+ // First operand is always the result.
+ const TargetRegisterClass *PacketRC = HII->getRegClass(MCID, 0, HRI, MF);
+ // Double regs can not feed into new value store: PRM section: 5.4.2.2.
+ if (PacketRC == &Hexagon::DoubleRegsRegClass)
+ return false;
+
+ // New-value stores are of class NV (slot 0), dual stores require class ST
+ // in slot 0 (PRM 5.5).
+ for (auto I : CurrentPacketMIs) {
+ SUnit *PacketSU = MIToSUnit.find(I)->second;
+ if (PacketSU->getInstr()->mayStore())
+ return false;
+ }
+
+ // Make sure it's NOT the post increment register that we are going to
+ // new value.
+ if (HII->isPostIncrement(MI) &&
+ getPostIncrementOperand(MI, HII).getReg() == DepReg) {
+ return false;
+ }
+
+ if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&
+ getPostIncrementOperand(PacketMI, HII).getReg() == DepReg) {
+ // If source is post_inc, or absolute-set addressing, it can not feed
+ // into new value store
+ // r3 = memw(r2++#4)
+ // memw(r30 + #-1404) = r2.new -> can not be new value store
+ // arch spec section: 5.4.2.1.
+ return false;
+ }
+
+ if (isLoadAbsSet(PacketMI) && getAbsSetOperand(PacketMI).getReg() == DepReg)
+ return false;
+
+ // If the source that feeds the store is predicated, new value store must
+ // also be predicated.
+ if (HII->isPredicated(PacketMI)) {
+ if (!HII->isPredicated(MI))
+ return false;
+
+ // Check to make sure that they both will have their predicates
+ // evaluate identically.
+ unsigned predRegNumSrc = 0;
+ unsigned predRegNumDst = 0;
+ const TargetRegisterClass* predRegClass = nullptr;
+
+ // Get predicate register used in the source instruction.
+ for (auto &MO : PacketMI.operands()) {
+ if (!MO.isReg())
+ continue;
+ predRegNumSrc = MO.getReg();
+ predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);
+ if (predRegClass == &Hexagon::PredRegsRegClass)
+ break;
+ }
+ assert((predRegClass == &Hexagon::PredRegsRegClass) &&
+ "predicate register not found in a predicated PacketMI instruction");
+
+ // Get predicate register used in new-value store instruction.
+ for (auto &MO : MI.operands()) {
+ if (!MO.isReg())
+ continue;
+ predRegNumDst = MO.getReg();
+ predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);
+ if (predRegClass == &Hexagon::PredRegsRegClass)
+ break;
+ }
+ assert((predRegClass == &Hexagon::PredRegsRegClass) &&
+ "predicate register not found in a predicated MI instruction");
+
+ // New-value register producer and user (store) need to satisfy these
+ // constraints:
+ // 1) Both instructions should be predicated on the same register.
+ // 2) If producer of the new-value register is .new predicated then store
+ // should also be .new predicated and if producer is not .new predicated
+ // then store should not be .new predicated.
+ // 3) Both new-value register producer and user should have same predicate
+ // sense, i.e, either both should be negated or both should be non-negated.
+ if (predRegNumDst != predRegNumSrc ||
+ HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||
+ getPredicateSense(MI, HII) != getPredicateSense(PacketMI, HII))
+ return false;
+ }
+
+ // Make sure that other than the new-value register no other store instruction
+ // register has been modified in the same packet. Predicate registers can be
+ // modified by they should not be modified between the producer and the store
+ // instruction as it will make them both conditional on different values.
+ // We already know this to be true for all the instructions before and
+ // including PacketMI. Howerver, we need to perform the check for the
+ // remaining instructions in the packet.
+
+ unsigned StartCheck = 0;
+
+ for (auto I : CurrentPacketMIs) {
+ SUnit *TempSU = MIToSUnit.find(I)->second;
+ MachineInstr &TempMI = *TempSU->getInstr();
+
+ // Following condition is true for all the instructions until PacketMI is
+ // reached (StartCheck is set to 0 before the for loop).
+ // StartCheck flag is 1 for all the instructions after PacketMI.
+ if (&TempMI != &PacketMI && !StartCheck) // Start processing only after
+ continue; // encountering PacketMI.
+
+ StartCheck = 1;
+ if (&TempMI == &PacketMI) // We don't want to check PacketMI for dependence.
+ continue;
+
+ for (auto &MO : MI.operands())
+ if (MO.isReg() && TempSU->getInstr()->modifiesRegister(MO.getReg(), HRI))
+ return false;
+ }
+
+ // Make sure that for non-POST_INC stores:
+ // 1. The only use of reg is DepReg and no other registers.
+ // This handles base+index registers.
+ // The following store can not be dot new.
+ // Eg. r0 = add(r0, #3)
+ // memw(r1+r0<<#2) = r0
+ if (!HII->isPostIncrement(MI)) {
+ for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {
+ const MachineOperand &MO = MI.getOperand(opNum);
+ if (MO.isReg() && MO.getReg() == DepReg)
+ return false;
+ }
+ }
+
+ // If data definition is because of implicit definition of the register,
+ // do not newify the store. Eg.
+ // %r9 = ZXTH %r12, implicit %d6, implicit-def %r12
+ // S2_storerh_io %r8, 2, killed %r12; mem:ST2[%scevgep343]
+ for (auto &MO : PacketMI.operands()) {
+ if (MO.isRegMask() && MO.clobbersPhysReg(DepReg))
+ return false;
+ if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())
+ continue;
+ unsigned R = MO.getReg();
+ if (R == DepReg || HRI->isSuperRegister(DepReg, R))
+ return false;
+ }
+
+ // Handle imp-use of super reg case. There is a target independent side
+ // change that should prevent this situation but I am handling it for
+ // just-in-case. For example, we cannot newify R2 in the following case:
+ // %r3 = A2_tfrsi 0;
+ // S2_storeri_io killed %r0, 0, killed %r2, implicit killed %d1;
+ for (auto &MO : MI.operands()) {
+ if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)
+ return false;
+ }
+
+ // Can be dot new store.
+ return true;
+}
+
+// Can this MI to promoted to either new value store or new value jump.
+bool HexagonPacketizerList::canPromoteToNewValue(const MachineInstr &MI,
+ const SUnit *PacketSU, unsigned DepReg,
+ MachineBasicBlock::iterator &MII) {
+ if (!HII->mayBeNewStore(MI))
+ return false;
+
+ // Check to see the store can be new value'ed.
+ MachineInstr &PacketMI = *PacketSU->getInstr();
+ if (canPromoteToNewValueStore(MI, PacketMI, DepReg))
+ return true;
+
+ // Check to see the compare/jump can be new value'ed.
+ // This is done as a pass on its own. Don't need to check it here.
+ return false;
+}
+
+static bool isImplicitDependency(const MachineInstr &I, bool CheckDef,
+ unsigned DepReg) {
+ for (auto &MO : I.operands()) {
+ if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg))
+ return true;
+ if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())
+ continue;
+ if (CheckDef == MO.isDef())
+ return true;
+ }
+ return false;
+}
+
+// Check to see if an instruction can be dot new.
+bool HexagonPacketizerList::canPromoteToDotNew(const MachineInstr &MI,
+ const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII,
+ const TargetRegisterClass* RC) {
+ // Already a dot new instruction.
+ if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))
+ return false;
+
+ if (!isNewifiable(MI, RC))
+ return false;
+
+ const MachineInstr &PI = *PacketSU->getInstr();
+
+ // The "new value" cannot come from inline asm.
+ if (PI.isInlineAsm())
+ return false;
+
+ // IMPLICIT_DEFs won't materialize as real instructions, so .new makes no
+ // sense.
+ if (PI.isImplicitDef())
+ return false;
+
+ // If dependency is trough an implicitly defined register, we should not
+ // newify the use.
+ if (isImplicitDependency(PI, true, DepReg) ||
+ isImplicitDependency(MI, false, DepReg))
+ return false;
+
+ const MCInstrDesc& MCID = PI.getDesc();
+ const TargetRegisterClass *VecRC = HII->getRegClass(MCID, 0, HRI, MF);
+ if (DisableVecDblNVStores && VecRC == &Hexagon::HvxWRRegClass)
+ return false;
+
+ // predicate .new
+ if (RC == &Hexagon::PredRegsRegClass)
+ return HII->predCanBeUsedAsDotNew(PI, DepReg);
+
+ if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))
+ return false;
+
+ // Create a dot new machine instruction to see if resources can be
+ // allocated. If not, bail out now.
+ int NewOpcode = HII->getDotNewOp(MI);
+ const MCInstrDesc &D = HII->get(NewOpcode);
+ MachineInstr *NewMI = MF.CreateMachineInstr(D, DebugLoc());
+ bool ResourcesAvailable = ResourceTracker->canReserveResources(*NewMI);
+ MF.DeleteMachineInstr(NewMI);
+ if (!ResourcesAvailable)
+ return false;
+
+ // New Value Store only. New Value Jump generated as a separate pass.
+ if (!canPromoteToNewValue(MI, PacketSU, DepReg, MII))
+ return false;
+
+ return true;
+}
+
+// Go through the packet instructions and search for an anti dependency between
+// them and DepReg from MI. Consider this case:
+// Trying to add
+// a) %r1 = TFRI_cdNotPt %p3, 2
+// to this packet:
+// {
+// b) %p0 = C2_or killed %p3, killed %p0
+// c) %p3 = C2_tfrrp %r23
+// d) %r1 = C2_cmovenewit %p3, 4
+// }
+// The P3 from a) and d) will be complements after
+// a)'s P3 is converted to .new form
+// Anti-dep between c) and b) is irrelevant for this case
+bool HexagonPacketizerList::restrictingDepExistInPacket(MachineInstr &MI,
+ unsigned DepReg) {
+ SUnit *PacketSUDep = MIToSUnit.find(&MI)->second;
+
+ for (auto I : CurrentPacketMIs) {
+ // We only care for dependencies to predicated instructions
+ if (!HII->isPredicated(*I))
+ continue;
+
+ // Scheduling Unit for current insn in the packet
+ SUnit *PacketSU = MIToSUnit.find(I)->second;
+
+ // Look at dependencies between current members of the packet and
+ // predicate defining instruction MI. Make sure that dependency is
+ // on the exact register we care about.
+ if (PacketSU->isSucc(PacketSUDep)) {
+ for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
+ auto &Dep = PacketSU->Succs[i];
+ if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&
+ Dep.getReg() == DepReg)
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+/// Gets the predicate register of a predicated instruction.
+static unsigned getPredicatedRegister(MachineInstr &MI,
+ const HexagonInstrInfo *QII) {
+ /// We use the following rule: The first predicate register that is a use is
+ /// the predicate register of a predicated instruction.
+ assert(QII->isPredicated(MI) && "Must be predicated instruction");
+
+ for (auto &Op : MI.operands()) {
+ if (Op.isReg() && Op.getReg() && Op.isUse() &&
+ Hexagon::PredRegsRegClass.contains(Op.getReg()))
+ return Op.getReg();
+ }
+
+ llvm_unreachable("Unknown instruction operand layout");
+ return 0;
+}
+
+// Given two predicated instructions, this function detects whether
+// the predicates are complements.
+bool HexagonPacketizerList::arePredicatesComplements(MachineInstr &MI1,
+ MachineInstr &MI2) {
+ // If we don't know the predicate sense of the instructions bail out early, we
+ // need it later.
+ if (getPredicateSense(MI1, HII) == PK_Unknown ||
+ getPredicateSense(MI2, HII) == PK_Unknown)
+ return false;
+
+ // Scheduling unit for candidate.
+ SUnit *SU = MIToSUnit[&MI1];
+
+ // One corner case deals with the following scenario:
+ // Trying to add
+ // a) %r24 = A2_tfrt %p0, %r25
+ // to this packet:
+ // {
+ // b) %r25 = A2_tfrf %p0, %r24
+ // c) %p0 = C2_cmpeqi %r26, 1
+ // }
+ //
+ // On general check a) and b) are complements, but presence of c) will
+ // convert a) to .new form, and then it is not a complement.
+ // We attempt to detect it by analyzing existing dependencies in the packet.
+
+ // Analyze relationships between all existing members of the packet.
+ // Look for Anti dependecy on the same predicate reg as used in the
+ // candidate.
+ for (auto I : CurrentPacketMIs) {
+ // Scheduling Unit for current insn in the packet.
+ SUnit *PacketSU = MIToSUnit.find(I)->second;
+
+ // If this instruction in the packet is succeeded by the candidate...
+ if (PacketSU->isSucc(SU)) {
+ for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {
+ auto Dep = PacketSU->Succs[i];
+ // The corner case exist when there is true data dependency between
+ // candidate and one of current packet members, this dep is on
+ // predicate reg, and there already exist anti dep on the same pred in
+ // the packet.
+ if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&
+ Hexagon::PredRegsRegClass.contains(Dep.getReg())) {
+ // Here I know that I is predicate setting instruction with true
+ // data dep to candidate on the register we care about - c) in the
+ // above example. Now I need to see if there is an anti dependency
+ // from c) to any other instruction in the same packet on the pred
+ // reg of interest.
+ if (restrictingDepExistInPacket(*I, Dep.getReg()))
+ return false;
+ }
+ }
+ }
+ }
+
+ // If the above case does not apply, check regular complement condition.
+ // Check that the predicate register is the same and that the predicate
+ // sense is different We also need to differentiate .old vs. .new: !p0
+ // is not complementary to p0.new.
+ unsigned PReg1 = getPredicatedRegister(MI1, HII);
+ unsigned PReg2 = getPredicatedRegister(MI2, HII);
+ return PReg1 == PReg2 &&
+ Hexagon::PredRegsRegClass.contains(PReg1) &&
+ Hexagon::PredRegsRegClass.contains(PReg2) &&
+ getPredicateSense(MI1, HII) != getPredicateSense(MI2, HII) &&
+ HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);
+}
+
+// Initialize packetizer flags.
+void HexagonPacketizerList::initPacketizerState() {
+ Dependence = false;
+ PromotedToDotNew = false;
+ GlueToNewValueJump = false;
+ GlueAllocframeStore = false;
+ FoundSequentialDependence = false;
+ ChangedOffset = INT64_MAX;
+}
+
+// Ignore bundling of pseudo instructions.
+bool HexagonPacketizerList::ignorePseudoInstruction(const MachineInstr &MI,
+ const MachineBasicBlock *) {
+ if (MI.isDebugInstr())
+ return true;
+
+ if (MI.isCFIInstruction())
+ return false;
+
+ // We must print out inline assembly.
+ if (MI.isInlineAsm())
+ return false;
+
+ if (MI.isImplicitDef())
+ return false;
+
+ // We check if MI has any functional units mapped to it. If it doesn't,
+ // we ignore the instruction.
+ const MCInstrDesc& TID = MI.getDesc();
+ auto *IS = ResourceTracker->getInstrItins()->beginStage(TID.getSchedClass());
+ unsigned FuncUnits = IS->getUnits();
+ return !FuncUnits;
+}
+
+bool HexagonPacketizerList::isSoloInstruction(const MachineInstr &MI) {
+ // Ensure any bundles created by gather packetize remain seperate.
+ if (MI.isBundle())
+ return true;
+
+ if (MI.isEHLabel() || MI.isCFIInstruction())
+ return true;
+
+ // Consider inline asm to not be a solo instruction by default.
+ // Inline asm will be put in a packet temporarily, but then it will be
+ // removed, and placed outside of the packet (before or after, depending
+ // on dependencies). This is to reduce the impact of inline asm as a
+ // "packet splitting" instruction.
+ if (MI.isInlineAsm() && !ScheduleInlineAsm)
+ return true;
+
+ if (isSchedBarrier(MI))
+ return true;
+
+ if (HII->isSolo(MI))
+ return true;
+
+ if (MI.getOpcode() == Hexagon::A2_nop)
+ return true;
+
+ return false;
+}
+
+// Quick check if instructions MI and MJ cannot coexist in the same packet.
+// Limit the tests to be "one-way", e.g. "if MI->isBranch and MJ->isInlineAsm",
+// but not the symmetric case: "if MJ->isBranch and MI->isInlineAsm".
+// For full test call this function twice:
+// cannotCoexistAsymm(MI, MJ) || cannotCoexistAsymm(MJ, MI)
+// Doing the test only one way saves the amount of code in this function,
+// since every test would need to be repeated with the MI and MJ reversed.
+static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ,
+ const HexagonInstrInfo &HII) {
+ const MachineFunction *MF = MI.getParent()->getParent();
+ if (MF->getSubtarget<HexagonSubtarget>().hasV60OpsOnly() &&
+ HII.isHVXMemWithAIndirect(MI, MJ))
+ return true;
+
+ // An inline asm cannot be together with a branch, because we may not be
+ // able to remove the asm out after packetizing (i.e. if the asm must be
+ // moved past the bundle). Similarly, two asms cannot be together to avoid
+ // complications when determining their relative order outside of a bundle.
+ if (MI.isInlineAsm())
+ return MJ.isInlineAsm() || MJ.isBranch() || MJ.isBarrier() ||
+ MJ.isCall() || MJ.isTerminator();
+
+ // New-value stores cannot coexist with any other stores.
+ if (HII.isNewValueStore(MI) && MJ.mayStore())
+ return true;
+
+ switch (MI.getOpcode()) {
+ case Hexagon::S2_storew_locked:
+ case Hexagon::S4_stored_locked:
+ case Hexagon::L2_loadw_locked:
+ case Hexagon::L4_loadd_locked:
+ case Hexagon::Y2_dccleana:
+ case Hexagon::Y2_dccleaninva:
+ case Hexagon::Y2_dcinva:
+ case Hexagon::Y2_dczeroa:
+ case Hexagon::Y4_l2fetch:
+ case Hexagon::Y5_l2fetch: {
+ // These instructions can only be grouped with ALU32 or non-floating-point
+ // XTYPE instructions. Since there is no convenient way of identifying fp
+ // XTYPE instructions, only allow grouping with ALU32 for now.
+ unsigned TJ = HII.getType(MJ);
+ if (TJ != HexagonII::TypeALU32_2op &&
+ TJ != HexagonII::TypeALU32_3op &&
+ TJ != HexagonII::TypeALU32_ADDI)
+ return true;
+ break;
+ }
+ default:
+ break;
+ }
+
+ // "False" really means that the quick check failed to determine if
+ // I and J cannot coexist.
+ return false;
+}
+
+// Full, symmetric check.
+bool HexagonPacketizerList::cannotCoexist(const MachineInstr &MI,
+ const MachineInstr &MJ) {
+ return cannotCoexistAsymm(MI, MJ, *HII) || cannotCoexistAsymm(MJ, MI, *HII);
+}
+
+void HexagonPacketizerList::unpacketizeSoloInstrs(MachineFunction &MF) {
+ for (auto &B : MF) {
+ MachineBasicBlock::iterator BundleIt;
+ MachineBasicBlock::instr_iterator NextI;
+ for (auto I = B.instr_begin(), E = B.instr_end(); I != E; I = NextI) {
+ NextI = std::next(I);
+ MachineInstr &MI = *I;
+ if (MI.isBundle())
+ BundleIt = I;
+ if (!MI.isInsideBundle())
+ continue;
+
+ // Decide on where to insert the instruction that we are pulling out.
+ // Debug instructions always go before the bundle, but the placement of
+ // INLINE_ASM depends on potential dependencies. By default, try to
+ // put it before the bundle, but if the asm writes to a register that
+ // other instructions in the bundle read, then we need to place it
+ // after the bundle (to preserve the bundle semantics).
+ bool InsertBeforeBundle;
+ if (MI.isInlineAsm())
+ InsertBeforeBundle = !hasWriteToReadDep(MI, *BundleIt, HRI);
+ else if (MI.isDebugValue())
+ InsertBeforeBundle = true;
+ else
+ continue;
+
+ BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);
+ }
+ }
+}
+
+// Check if a given instruction is of class "system".
+static bool isSystemInstr(const MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ switch (Opc) {
+ case Hexagon::Y2_barrier:
+ case Hexagon::Y2_dcfetchbo:
+ case Hexagon::Y4_l2fetch:
+ case Hexagon::Y5_l2fetch:
+ return true;
+ }
+ return false;
+}
+
+bool HexagonPacketizerList::hasDeadDependence(const MachineInstr &I,
+ const MachineInstr &J) {
+ // The dependence graph may not include edges between dead definitions,
+ // so without extra checks, we could end up packetizing two instruction
+ // defining the same (dead) register.
+ if (I.isCall() || J.isCall())
+ return false;
+ if (HII->isPredicated(I) || HII->isPredicated(J))
+ return false;
+
+ BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);
+ for (auto &MO : I.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.isDead())
+ continue;
+ DeadDefs[MO.getReg()] = true;
+ }
+
+ for (auto &MO : J.operands()) {
+ if (!MO.isReg() || !MO.isDef() || !MO.isDead())
+ continue;
+ unsigned R = MO.getReg();
+ if (R != Hexagon::USR_OVF && DeadDefs[R])
+ return true;
+ }
+ return false;
+}
+
+bool HexagonPacketizerList::hasControlDependence(const MachineInstr &I,
+ const MachineInstr &J) {
+ // A save callee-save register function call can only be in a packet
+ // with instructions that don't write to the callee-save registers.
+ if ((HII->isSaveCalleeSavedRegsCall(I) &&
+ doesModifyCalleeSavedReg(J, HRI)) ||
+ (HII->isSaveCalleeSavedRegsCall(J) &&
+ doesModifyCalleeSavedReg(I, HRI)))
+ return true;
+
+ // Two control flow instructions cannot go in the same packet.
+ if (isControlFlow(I) && isControlFlow(J))
+ return true;
+
+ // \ref-manual (7.3.4) A loop setup packet in loopN or spNloop0 cannot
+ // contain a speculative indirect jump,
+ // a new-value compare jump or a dealloc_return.
+ auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {
+ if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))
+ return true;
+ if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))
+ return true;
+ return false;
+ };
+
+ if (HII->isLoopN(I) && isBadForLoopN(J))
+ return true;
+ if (HII->isLoopN(J) && isBadForLoopN(I))
+ return true;
+
+ // dealloc_return cannot appear in the same packet as a conditional or
+ // unconditional jump.
+ return HII->isDeallocRet(I) &&
+ (J.isBranch() || J.isCall() || J.isBarrier());
+}
+
+bool HexagonPacketizerList::hasRegMaskDependence(const MachineInstr &I,
+ const MachineInstr &J) {
+ // Adding I to a packet that has J.
+
+ // Regmasks are not reflected in the scheduling dependency graph, so
+ // we need to check them manually. This code assumes that regmasks only
+ // occur on calls, and the problematic case is when we add an instruction
+ // defining a register R to a packet that has a call that clobbers R via
+ // a regmask. Those cannot be packetized together, because the call will
+ // be executed last. That's also a reson why it is ok to add a call
+ // clobbering R to a packet that defines R.
+
+ // Look for regmasks in J.
+ for (const MachineOperand &OpJ : J.operands()) {
+ if (!OpJ.isRegMask())
+ continue;
+ assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");
+ for (const MachineOperand &OpI : I.operands()) {
+ if (OpI.isReg()) {
+ if (OpJ.clobbersPhysReg(OpI.getReg()))
+ return true;
+ } else if (OpI.isRegMask()) {
+ // Both are regmasks. Assume that they intersect.
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+bool HexagonPacketizerList::hasDualStoreDependence(const MachineInstr &I,
+ const MachineInstr &J) {
+ bool SysI = isSystemInstr(I), SysJ = isSystemInstr(J);
+ bool StoreI = I.mayStore(), StoreJ = J.mayStore();
+ if ((SysI && StoreJ) || (SysJ && StoreI))
+ return true;
+
+ if (StoreI && StoreJ) {
+ if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))
+ return true;
+ } else {
+ // A memop cannot be in the same packet with another memop or a store.
+ // Two stores can be together, but here I and J cannot both be stores.
+ bool MopStI = HII->isMemOp(I) || StoreI;
+ bool MopStJ = HII->isMemOp(J) || StoreJ;
+ if (MopStI && MopStJ)
+ return true;
+ }
+
+ return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));
+}
+
+// SUI is the current instruction that is out side of the current packet.
+// SUJ is the current instruction inside the current packet against which that
+// SUI will be packetized.
+bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
+ assert(SUI->getInstr() && SUJ->getInstr());
+ MachineInstr &I = *SUI->getInstr();
+ MachineInstr &J = *SUJ->getInstr();
+
+ // Clear IgnoreDepMIs when Packet starts.
+ if (CurrentPacketMIs.size() == 1)
+ IgnoreDepMIs.clear();
+
+ MachineBasicBlock::iterator II = I.getIterator();
+
+ // Solo instructions cannot go in the packet.
+ assert(!isSoloInstruction(I) && "Unexpected solo instr!");
+
+ if (cannotCoexist(I, J))
+ return false;
+
+ Dependence = hasDeadDependence(I, J) || hasControlDependence(I, J);
+ if (Dependence)
+ return false;
+
+ // Regmasks are not accounted for in the scheduling graph, so we need
+ // to explicitly check for dependencies caused by them. They should only
+ // appear on calls, so it's not too pessimistic to reject all regmask
+ // dependencies.
+ Dependence = hasRegMaskDependence(I, J);
+ if (Dependence)
+ return false;
+
+ // Dual-store does not allow second store, if the first store is not
+ // in SLOT0. New value store, new value jump, dealloc_return and memop
+ // always take SLOT0. Arch spec 3.4.4.2.
+ Dependence = hasDualStoreDependence(I, J);
+ if (Dependence)
+ return false;
+
+ // If an instruction feeds new value jump, glue it.
+ MachineBasicBlock::iterator NextMII = I.getIterator();
+ ++NextMII;
+ if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {
+ MachineInstr &NextMI = *NextMII;
+
+ bool secondRegMatch = false;
+ const MachineOperand &NOp0 = NextMI.getOperand(0);
+ const MachineOperand &NOp1 = NextMI.getOperand(1);
+
+ if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg())
+ secondRegMatch = true;
+
+ for (MachineInstr *PI : CurrentPacketMIs) {
+ // NVJ can not be part of the dual jump - Arch Spec: section 7.8.
+ if (PI->isCall()) {
+ Dependence = true;
+ break;
+ }
+ // Validate:
+ // 1. Packet does not have a store in it.
+ // 2. If the first operand of the nvj is newified, and the second
+ // operand is also a reg, it (second reg) is not defined in
+ // the same packet.
+ // 3. If the second operand of the nvj is newified, (which means
+ // first operand is also a reg), first reg is not defined in
+ // the same packet.
+ if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||
+ HII->isLoopN(*PI)) {
+ Dependence = true;
+ break;
+ }
+ // Check #2/#3.
+ const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;
+ if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {
+ Dependence = true;
+ break;
+ }
+ }
+
+ GlueToNewValueJump = true;
+ if (Dependence)
+ return false;
+ }
+
+ // There no dependency between a prolog instruction and its successor.
+ if (!SUJ->isSucc(SUI))
+ return true;
+
+ for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {
+ if (FoundSequentialDependence)
+ break;
+
+ if (SUJ->Succs[i].getSUnit() != SUI)
+ continue;
+
+ SDep::Kind DepType = SUJ->Succs[i].getKind();
+ // For direct calls:
+ // Ignore register dependences for call instructions for packetization
+ // purposes except for those due to r31 and predicate registers.
+ //
+ // For indirect calls:
+ // Same as direct calls + check for true dependences to the register
+ // used in the indirect call.
+ //
+ // We completely ignore Order dependences for call instructions.
+ //
+ // For returns:
+ // Ignore register dependences for return instructions like jumpr,
+ // dealloc return unless we have dependencies on the explicit uses
+ // of the registers used by jumpr (like r31) or dealloc return
+ // (like r29 or r30).
+ unsigned DepReg = 0;
+ const TargetRegisterClass *RC = nullptr;
+ if (DepType == SDep::Data) {
+ DepReg = SUJ->Succs[i].getReg();
+ RC = HRI->getMinimalPhysRegClass(DepReg);
+ }
+
+ if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {
+ if (!isRegDependence(DepType))
+ continue;
+ if (!isCallDependent(I, DepType, SUJ->Succs[i].getReg()))
+ continue;
+ }
+
+ if (DepType == SDep::Data) {
+ if (canPromoteToDotCur(J, SUJ, DepReg, II, RC))
+ if (promoteToDotCur(J, DepType, II, RC))
+ continue;
+ }
+
+ // Data dpendence ok if we have load.cur.
+ if (DepType == SDep::Data && HII->isDotCurInst(J)) {
+ if (HII->isHVXVec(I))
+ continue;
+ }
+
+ // For instructions that can be promoted to dot-new, try to promote.
+ if (DepType == SDep::Data) {
+ if (canPromoteToDotNew(I, SUJ, DepReg, II, RC)) {
+ if (promoteToDotNew(I, DepType, II, RC)) {
+ PromotedToDotNew = true;
+ if (cannotCoexist(I, J))
+ FoundSequentialDependence = true;
+ continue;
+ }
+ }
+ if (HII->isNewValueJump(I))
+ continue;
+ }
+
+ // For predicated instructions, if the predicates are complements then
+ // there can be no dependence.
+ if (HII->isPredicated(I) && HII->isPredicated(J) &&
+ arePredicatesComplements(I, J)) {
+ // Not always safe to do this translation.
+ // DAG Builder attempts to reduce dependence edges using transitive
+ // nature of dependencies. Here is an example:
+ //
+ // r0 = tfr_pt ... (1)
+ // r0 = tfr_pf ... (2)
+ // r0 = tfr_pt ... (3)
+ //
+ // There will be an output dependence between (1)->(2) and (2)->(3).
+ // However, there is no dependence edge between (1)->(3). This results
+ // in all 3 instructions going in the same packet. We ignore dependce
+ // only once to avoid this situation.
+ auto Itr = find(IgnoreDepMIs, &J);
+ if (Itr != IgnoreDepMIs.end()) {
+ Dependence = true;
+ return false;
+ }
+ IgnoreDepMIs.push_back(&I);
+ continue;
+ }
+
+ // Ignore Order dependences between unconditional direct branches
+ // and non-control-flow instructions.
+ if (isDirectJump(I) && !J.isBranch() && !J.isCall() &&
+ DepType == SDep::Order)
+ continue;
+
+ // Ignore all dependences for jumps except for true and output
+ // dependences.
+ if (I.isConditionalBranch() && DepType != SDep::Data &&
+ DepType != SDep::Output)
+ continue;
+
+ if (DepType == SDep::Output) {
+ FoundSequentialDependence = true;
+ break;
+ }
+
+ // For Order dependences:
+ // 1. Volatile loads/stores can be packetized together, unless other
+ // rules prevent is.
+ // 2. Store followed by a load is not allowed.
+ // 3. Store followed by a store is valid.
+ // 4. Load followed by any memory operation is allowed.
+ if (DepType == SDep::Order) {
+ if (!PacketizeVolatiles) {
+ bool OrdRefs = I.hasOrderedMemoryRef() || J.hasOrderedMemoryRef();
+ if (OrdRefs) {
+ FoundSequentialDependence = true;
+ break;
+ }
+ }
+ // J is first, I is second.
+ bool LoadJ = J.mayLoad(), StoreJ = J.mayStore();
+ bool LoadI = I.mayLoad(), StoreI = I.mayStore();
+ bool NVStoreJ = HII->isNewValueStore(J);
+ bool NVStoreI = HII->isNewValueStore(I);
+ bool IsVecJ = HII->isHVXVec(J);
+ bool IsVecI = HII->isHVXVec(I);
+
+ if (Slot1Store && MF.getSubtarget<HexagonSubtarget>().hasV65Ops() &&
+ ((LoadJ && StoreI && !NVStoreI) ||
+ (StoreJ && LoadI && !NVStoreJ)) &&
+ (J.getOpcode() != Hexagon::S2_allocframe &&
+ I.getOpcode() != Hexagon::S2_allocframe) &&
+ (J.getOpcode() != Hexagon::L2_deallocframe &&
+ I.getOpcode() != Hexagon::L2_deallocframe) &&
+ (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI))
+ setmemShufDisabled(true);
+ else
+ if (StoreJ && LoadI && alias(J, I)) {
+ FoundSequentialDependence = true;
+ break;
+ }
+
+ if (!StoreJ)
+ if (!LoadJ || (!LoadI && !StoreI)) {
+ // If J is neither load nor store, assume a dependency.
+ // If J is a load, but I is neither, also assume a dependency.
+ FoundSequentialDependence = true;
+ break;
+ }
+ // Store followed by store: not OK on V2.
+ // Store followed by load: not OK on all.
+ // Load followed by store: OK on all.
+ // Load followed by load: OK on all.
+ continue;
+ }
+
+ // Special case for ALLOCFRAME: even though there is dependency
+ // between ALLOCFRAME and subsequent store, allow it to be packetized
+ // in a same packet. This implies that the store is using the caller's
+ // SP. Hence, offset needs to be updated accordingly.
+ if (DepType == SDep::Data && J.getOpcode() == Hexagon::S2_allocframe) {
+ unsigned Opc = I.getOpcode();
+ switch (Opc) {
+ case Hexagon::S2_storerd_io:
+ case Hexagon::S2_storeri_io:
+ case Hexagon::S2_storerh_io:
+ case Hexagon::S2_storerb_io:
+ if (I.getOperand(0).getReg() == HRI->getStackRegister()) {
+ // Since this store is to be glued with allocframe in the same
+ // packet, it will use SP of the previous stack frame, i.e.
+ // caller's SP. Therefore, we need to recalculate offset
+ // according to this change.
+ GlueAllocframeStore = useCallersSP(I);
+ if (GlueAllocframeStore)
+ continue;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ // There are certain anti-dependencies that cannot be ignored.
+ // Specifically:
+ // J2_call ... implicit-def %r0 ; SUJ
+ // R0 = ... ; SUI
+ // Those cannot be packetized together, since the call will observe
+ // the effect of the assignment to R0.
+ if ((DepType == SDep::Anti || DepType == SDep::Output) && J.isCall()) {
+ // Check if I defines any volatile register. We should also check
+ // registers that the call may read, but these happen to be a
+ // subset of the volatile register set.
+ for (const MachineOperand &Op : I.operands()) {
+ if (Op.isReg() && Op.isDef()) {
+ unsigned R = Op.getReg();
+ if (!J.readsRegister(R, HRI) && !J.modifiesRegister(R, HRI))
+ continue;
+ } else if (!Op.isRegMask()) {
+ // If I has a regmask assume dependency.
+ continue;
+ }
+ FoundSequentialDependence = true;
+ break;
+ }
+ }
+
+ // Skip over remaining anti-dependences. Two instructions that are
+ // anti-dependent can share a packet, since in most such cases all
+ // operands are read before any modifications take place.
+ // The exceptions are branch and call instructions, since they are
+ // executed after all other instructions have completed (at least
+ // conceptually).
+ if (DepType != SDep::Anti) {
+ FoundSequentialDependence = true;
+ break;
+ }
+ }
+
+ if (FoundSequentialDependence) {
+ Dependence = true;
+ return false;
+ }
+
+ return true;
+}
+
+bool HexagonPacketizerList::isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) {
+ assert(SUI->getInstr() && SUJ->getInstr());
+ MachineInstr &I = *SUI->getInstr();
+ MachineInstr &J = *SUJ->getInstr();
+
+ bool Coexist = !cannotCoexist(I, J);
+
+ if (Coexist && !Dependence)
+ return true;
+
+ // Check if the instruction was promoted to a dot-new. If so, demote it
+ // back into a dot-old.
+ if (PromotedToDotNew)
+ demoteToDotOld(I);
+
+ cleanUpDotCur();
+ // Check if the instruction (must be a store) was glued with an allocframe
+ // instruction. If so, restore its offset to its original value, i.e. use
+ // current SP instead of caller's SP.
+ if (GlueAllocframeStore) {
+ useCalleesSP(I);
+ GlueAllocframeStore = false;
+ }
+
+ if (ChangedOffset != INT64_MAX)
+ undoChangedOffset(I);
+
+ if (GlueToNewValueJump) {
+ // Putting I and J together would prevent the new-value jump from being
+ // packetized with the producer. In that case I and J must be separated.
+ GlueToNewValueJump = false;
+ return false;
+ }
+
+ if (!Coexist)
+ return false;
+
+ if (ChangedOffset == INT64_MAX && updateOffset(SUI, SUJ)) {
+ FoundSequentialDependence = false;
+ Dependence = false;
+ return true;
+ }
+
+ return false;
+}
+
+
+bool HexagonPacketizerList::foundLSInPacket() {
+ bool FoundLoad = false;
+ bool FoundStore = false;
+
+ for (auto MJ : CurrentPacketMIs) {
+ unsigned Opc = MJ->getOpcode();
+ if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe)
+ continue;
+ if (HII->isMemOp(*MJ))
+ continue;
+ if (MJ->mayLoad())
+ FoundLoad = true;
+ if (MJ->mayStore() && !HII->isNewValueStore(*MJ))
+ FoundStore = true;
+ }
+ return FoundLoad && FoundStore;
+}
+
+
+MachineBasicBlock::iterator
+HexagonPacketizerList::addToPacket(MachineInstr &MI) {
+ MachineBasicBlock::iterator MII = MI.getIterator();
+ MachineBasicBlock *MBB = MI.getParent();
+
+ if (CurrentPacketMIs.empty())
+ PacketStalls = false;
+ PacketStalls |= producesStall(MI);
+
+ if (MI.isImplicitDef()) {
+ // Add to the packet to allow subsequent instructions to be checked
+ // properly.
+ CurrentPacketMIs.push_back(&MI);
+ return MII;
+ }
+ assert(ResourceTracker->canReserveResources(MI));
+
+ bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);
+ bool Good = true;
+
+ if (GlueToNewValueJump) {
+ MachineInstr &NvjMI = *++MII;
+ // We need to put both instructions in the same packet: MI and NvjMI.
+ // Either of them can require a constant extender. Try to add both to
+ // the current packet, and if that fails, end the packet and start a
+ // new one.
+ ResourceTracker->reserveResources(MI);
+ if (ExtMI)
+ Good = tryAllocateResourcesForConstExt(true);
+
+ bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);
+ if (Good) {
+ if (ResourceTracker->canReserveResources(NvjMI))
+ ResourceTracker->reserveResources(NvjMI);
+ else
+ Good = false;
+ }
+ if (Good && ExtNvjMI)
+ Good = tryAllocateResourcesForConstExt(true);
+
+ if (!Good) {
+ endPacket(MBB, MI);
+ assert(ResourceTracker->canReserveResources(MI));
+ ResourceTracker->reserveResources(MI);
+ if (ExtMI) {
+ assert(canReserveResourcesForConstExt());
+ tryAllocateResourcesForConstExt(true);
+ }
+ assert(ResourceTracker->canReserveResources(NvjMI));
+ ResourceTracker->reserveResources(NvjMI);
+ if (ExtNvjMI) {
+ assert(canReserveResourcesForConstExt());
+ reserveResourcesForConstExt();
+ }
+ }
+ CurrentPacketMIs.push_back(&MI);
+ CurrentPacketMIs.push_back(&NvjMI);
+ return MII;
+ }
+
+ ResourceTracker->reserveResources(MI);
+ if (ExtMI && !tryAllocateResourcesForConstExt(true)) {
+ endPacket(MBB, MI);
+ if (PromotedToDotNew)
+ demoteToDotOld(MI);
+ if (GlueAllocframeStore) {
+ useCalleesSP(MI);
+ GlueAllocframeStore = false;
+ }
+ ResourceTracker->reserveResources(MI);
+ reserveResourcesForConstExt();
+ }
+
+ CurrentPacketMIs.push_back(&MI);
+ return MII;
+}
+
+void HexagonPacketizerList::endPacket(MachineBasicBlock *MBB,
+ MachineBasicBlock::iterator EndMI) {
+ // Replace VLIWPacketizerList::endPacket(MBB, EndMI).
+
+ bool memShufDisabled = getmemShufDisabled();
+ if (memShufDisabled && !foundLSInPacket()) {
+ setmemShufDisabled(false);
+ LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n");
+ }
+ memShufDisabled = getmemShufDisabled();
+
+ OldPacketMIs.clear();
+ for (MachineInstr *MI : CurrentPacketMIs) {
+ MachineBasicBlock::instr_iterator NextMI = std::next(MI->getIterator());
+ for (auto &I : make_range(HII->expandVGatherPseudo(*MI), NextMI))
+ OldPacketMIs.push_back(&I);
+ }
+ CurrentPacketMIs.clear();
+
+ if (OldPacketMIs.size() > 1) {
+ MachineBasicBlock::instr_iterator FirstMI(OldPacketMIs.front());
+ MachineBasicBlock::instr_iterator LastMI(EndMI.getInstrIterator());
+ finalizeBundle(*MBB, FirstMI, LastMI);
+ auto BundleMII = std::prev(FirstMI);
+ if (memShufDisabled)
+ HII->setBundleNoShuf(BundleMII);
+
+ setmemShufDisabled(false);
+ }
+
+ ResourceTracker->clearResources();
+ LLVM_DEBUG(dbgs() << "End packet\n");
+}
+
+bool HexagonPacketizerList::shouldAddToPacket(const MachineInstr &MI) {
+ if (Minimal)
+ return false;
+ return !producesStall(MI);
+}
+
+// V60 forward scheduling.
+bool HexagonPacketizerList::producesStall(const MachineInstr &I) {
+ // If the packet already stalls, then ignore the stall from a subsequent
+ // instruction in the same packet.
+ if (PacketStalls)
+ return false;
+
+ // Check whether the previous packet is in a different loop. If this is the
+ // case, there is little point in trying to avoid a stall because that would
+ // favor the rare case (loop entry) over the common case (loop iteration).
+ //
+ // TODO: We should really be able to check all the incoming edges if this is
+ // the first packet in a basic block, so we can avoid stalls from the loop
+ // backedge.
+ if (!OldPacketMIs.empty()) {
+ auto *OldBB = OldPacketMIs.front()->getParent();
+ auto *ThisBB = I.getParent();
+ if (MLI->getLoopFor(OldBB) != MLI->getLoopFor(ThisBB))
+ return false;
+ }
+
+ SUnit *SUI = MIToSUnit[const_cast<MachineInstr *>(&I)];
+
+ // If the latency is 0 and there is a data dependence between this
+ // instruction and any instruction in the current packet, we disregard any
+ // potential stalls due to the instructions in the previous packet. Most of
+ // the instruction pairs that can go together in the same packet have 0
+ // latency between them. The exceptions are
+ // 1. NewValueJumps as they're generated much later and the latencies can't
+ // be changed at that point.
+ // 2. .cur instructions, if its consumer has a 0 latency successor (such as
+ // .new). In this case, the latency between .cur and the consumer stays
+ // non-zero even though we can have both .cur and .new in the same packet.
+ // Changing the latency to 0 is not an option as it causes software pipeliner
+ // to not pipeline in some cases.
+
+ // For Example:
+ // {
+ // I1: v6.cur = vmem(r0++#1)
+ // I2: v7 = valign(v6,v4,r2)
+ // I3: vmem(r5++#1) = v7.new
+ // }
+ // Here I2 and I3 has 0 cycle latency, but I1 and I2 has 2.
+
+ for (auto J : CurrentPacketMIs) {
+ SUnit *SUJ = MIToSUnit[J];
+ for (auto &Pred : SUI->Preds)
+ if (Pred.getSUnit() == SUJ)
+ if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) ||
+ HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I))
+ return false;
+ }
+
+ // Check if the latency is greater than one between this instruction and any
+ // instruction in the previous packet.
+ for (auto J : OldPacketMIs) {
+ SUnit *SUJ = MIToSUnit[J];
+ for (auto &Pred : SUI->Preds)
+ if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)
+ return true;
+ }
+
+ return false;
+}
+
+//===----------------------------------------------------------------------===//
+// Public Constructor Functions
+//===----------------------------------------------------------------------===//
+
+FunctionPass *llvm::createHexagonPacketizer(bool Minimal) {
+ return new HexagonPacketizer(Minimal);
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-08 15:46:08 +0000